Compare commits

..

4 Commits

Author SHA1 Message Date
Andrew Kane
22a4ed0302 Removed extra file [skip ci] 2024-03-30 02:27:20 -07:00
Andrew Kane
cbcbebe52e Exclude zero vectors for cosine distance to be consistent with other types [skip ci] 2024-03-29 22:54:23 -07:00
Andrew Kane
5da3bdf98e Fixed CI 2024-03-29 17:35:47 -07:00
Andrew Kane
0d82124bca Added intvec type 2024-03-29 17:19:07 -07:00
163 changed files with 3730 additions and 14683 deletions

8
.dockerignore Normal file
View File

@@ -0,0 +1,8 @@
/.git/
/dist/
/results/
/tmp_check/
/sql/vector--?.?.?.sql
regression.*
*.o
*.so

View File

@@ -8,29 +8,27 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 19
os: ubuntu-24.04
- postgres: 18
os: ubuntu-24.04
- postgres: 17
os: ubuntu-24.04
os: ubuntu-22.04
- postgres: 16
os: ubuntu-24.04-arm
os: ubuntu-22.04
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-22.04-arm
- postgres: 13
os: ubuntu-22.04
- postgres: 13
os: ubuntu-20.04
- postgres: 12
os: ubuntu-20.04
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
@@ -42,70 +40,46 @@ jobs:
sudo apt-get install libipc-run-perl
- run: make prove_installcheck
mac:
runs-on: ${{ matrix.os }}
runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 18
os: macos-26
- postgres: 14
os: macos-15-intel
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
postgres-version: 14
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
# Homebrew Postgres does not enable TAP tests, so need to download
- run: |
brew install cpanm
cpanm --notest IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/$TAG.tar.gz
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
tar xf REL_14_10.tar.gz
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
env:
TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
env:
LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: ${{ matrix.os }}
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: windows-2025
- postgres: 14
os: windows-2022
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
cd %TEMP% && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
- if: ${{ failure() }}
run: cat regression.diffs
i386:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
@@ -127,17 +101,14 @@ jobs:
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- if: ${{ failure() }}
run: cat pgvector/regression.diffs
valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 18
check-ub: yes
- run: make OPTFLAGS=""
postgres-version: 16
- run: make
- run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck

View File

@@ -1,69 +1,6 @@
## 0.8.4 (2026-06-30)
## 0.7.0 (unreleased)
- Fixed `hnsw graph not repaired` error with HNSW vacuuming
- Fixed possible error with inserts during HNSW vacuuming
## 0.8.3 (2026-06-17)
- Fixed possible index corruption with HNSW vacuuming
- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)
- Improved `install` target on Windows
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
## 0.8.1 (2025-09-04)
- Added support for Postgres 18 rc1
- Improved performance of `binary_quantize` function
## 0.8.0 (2024-10-30)
- Added support for iterative index scans
- Added casts for arrays to `sparsevec`
- Improved cost estimation for better index selection when filtering
- Improved performance of HNSW index scans
- Improved performance of HNSW inserts and on-disk index builds
- Dropped support for Postgres 12
## 0.7.4 (2024-08-05)
- Fixed locking for parallel HNSW index builds
- Fixed compilation error with GCC 14 on i386 when SSE2 is not enabled
## 0.7.3 (2024-07-22)
- Fixed `failed to add index item` error with `sparsevec`
- Fixed compilation error with FreeBSD ARM
- Fixed compilation warning with MSVC and Postgres 16
## 0.7.2 (2024-06-11)
- Fixed initialization fork for indexes on unlogged tables
## 0.7.1 (2024-06-03)
- Improved performance of on-disk HNSW index builds
- Fixed `undefined symbol` error with GCC 8
- Fixed compilation error with universal binaries on Mac
- Fixed compilation warning with Clang < 14
## 0.7.0 (2024-04-29)
- Added `halfvec` type
- Added `sparsevec` type
- Added support for indexing `bit` type
- Added support for indexing L1 distance with HNSW
- Added `binary_quantize` function
- Added `hamming_distance` function
- Added `jaccard_distance` function
- Added `l2_normalize` function
- Added `subvector` function
- Added concatenate operator for vectors
- Added CPU dispatching for distance functions on Linux x86-64
- Updated comparison operators to support vectors with different dimensions
- Added `intvec` type
## 0.6.2 (2024-03-18)

View File

@@ -1,11 +1,8 @@
# syntax=docker/dockerfile:1
ARG PG_MAJOR=17
ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR=16
FROM postgres:$PG_MAJOR
ARG PG_MAJOR
ADD https://github.com/pgvector/pgvector.git#v0.8.4 /tmp/pgvector
COPY . /tmp/pgvector
RUN apt-get update && \
apt-mark hold locales && \

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,17 +2,17 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.4",
"version": "0.6.2",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
"license": {
"PostgreSQL": "https://www.postgresql.org/about/licence"
"PostgreSQL": "http://www.postgresql.org/about/licence"
},
"prereqs": {
"runtime": {
"requires": {
"PostgreSQL": "13.0.0"
"PostgreSQL": "12.0.0"
}
}
},
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.8.4",
"version": "0.6.2",
"abstract": "Open-source vector similarity search for Postgres"
}
},
@@ -38,7 +38,7 @@
"generated_by": "Andrew Kane",
"meta-spec": {
"version": "1.0.0",
"url": "https://pgxn.org/meta/spec.txt"
"url": "http://pgxn.org/meta/spec.txt"
},
"tags": [
"vectors",

View File

@@ -1,20 +1,18 @@
EXTENSION = vector
EXTVERSION = 0.8.4
EXTVERSION = 0.6.2
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)
DATA_built = sql/$(EXTENSION)--$(EXTVERSION).sql
OBJS = src/bitutils.o src/bitvec.o src/halfutils.o src/halfvec.o src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/sparsevec.o src/vector.o
HEADERS = src/halfvec.h src/sparsevec.h src/vector.h
DATA = $(wildcard sql/*--*.sql)
OBJS = src/hnsw.o src/hnswbuild.o src/hnswinsert.o src/hnswscan.o src/hnswutils.o src/hnswvacuum.o src/intvec.o src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.o src/ivfutils.o src/ivfvacuum.o src/vector.o
HEADERS = src/intvec.h src/vector.h
TESTS = $(wildcard test/sql/*.sql)
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# To compile for portability, run: make OPTFLAGS=""
OPTFLAGS = -march=native
# Mac ARM doesn't always support -march=native
# Mac ARM doesn't support -march=native
ifeq ($(shell uname -s), Darwin)
ifeq ($(shell uname -p), arm)
# no difference with -march=armv8.5-a
@@ -27,11 +25,6 @@ ifneq ($(filter ppc64%, $(shell uname -m)), )
OPTFLAGS =
endif
# RISC-V64 doesn't support -march=native
ifeq ($(shell uname -m), riscv64)
OPTFLAGS =
endif
# For auto-vectorization:
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -48,6 +41,8 @@ all: sql/$(EXTENSION)--$(EXTVERSION).sql
sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql
cp $< $@
EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql
PG_CONFIG ?= pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
@@ -57,7 +52,7 @@ ifeq ($(PROVE),)
PROVE = prove
endif
# for Postgres < 15
# for Postgres 15
PROVE_FLAGS += -I ./test/perl
prove_installcheck:
@@ -71,7 +66,7 @@ dist:
git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master
# for Docker
PG_MAJOR ?= 17
PG_MAJOR ?= 16
.PHONY: docker
@@ -81,9 +76,4 @@ docker:
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
.PHONY: docker-release-trixie
docker-release-trixie:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

View File

@@ -1,11 +1,10 @@
EXTENSION = vector
EXTVERSION = 0.8.4
EXTVERSION = 0.6.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
HEADERS = src\halfvec.h src\sparsevec.h src\vector.h
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\intvec.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
HEADERS = src\intvec.h src\vector.h
REGRESS = bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
# For /arch flags
@@ -20,6 +19,11 @@ PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
all: sql\$(EXTENSION)--$(EXTVERSION).sql
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
# TODO use pg_config
!ifndef PGROOT
!error PGROOT is not set
@@ -31,9 +35,6 @@ LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
# Use $(PGROOT)\bin\pg_regress for Postgres < 17
PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
@@ -42,26 +43,23 @@ SHLIB = $(EXTENSION).dll
LIBS = "$(LIBDIR)\postgres.lib"
all: $(SHLIB) $(DATA_built)
.c.obj:
$(CC) $(CFLAGS) /c $< /Fo$@
$(SHLIB): $(OBJS)
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
all: $(SHLIB)
install: all
install:
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
installcheck:
"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
uninstall:
del /f "$(PKGLIBDIR)\$(SHLIB)"
@@ -72,6 +70,6 @@ uninstall:
clean:
del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp
del /f $(DATA_built)
del /f $(OBJS)
del /f sql\$(EXTENSION)--$(EXTVERSION).sql
del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_iso

587
README.md
View File

@@ -5,25 +5,22 @@ Open-source vector similarity search for Postgres
Store your vectors with the rest of your data. Supports:
- exact and approximate nearest neighbor search
- single-precision, half-precision, binary, and sparse vectors
- L2 distance, inner product, cosine distance, L1 distance, Hamming distance, and Jaccard distance
- L2 distance, inner product, and cosine distance
- any [language](#languages) with a Postgres client
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
Have a lot of vectors? Use [quantization](#scaling) to scale
[![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions)
## Installation
### Linux and Mac
Compile and install the extension (supports Postgres 13+)
Compile and install the extension (supports Postgres 12+)
```sh
cd /tmp
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -31,16 +28,24 @@ make install # may need sudo
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\18"
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
```
Note: The exact path will vary depending on your Visual Studio version and edition
Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
cd %TEMP%
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -76,7 +81,7 @@ Get the nearest neighbors by L2 distance
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`)
Also supports inner product (`<#>`) and cosine distance (`<=>`)
Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators
@@ -94,15 +99,13 @@ Or add a vector column to an existing table
ALTER TABLE items ADD COLUMN embedding vector(3);
```
Also supports [half-precision](#half-precision-vectors), [binary](#binary-vectors), and [sparse](#sparse-vectors) vectors
Insert vectors
```sql
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
```
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py))
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -135,15 +138,6 @@ Get the nearest neighbors to a vector
SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
Supported distance functions are:
- `<->` - L2 distance
- `<#>` - (negative) inner product
- `<=>` - cosine distance
- `<+>` - L1 distance
- `<~>` - Hamming distance (binary vectors)
- `<%>` - Jaccard distance (binary vectors)
Get the nearest neighbors to a row
```sql
@@ -200,7 +194,7 @@ You can add an index to use approximate nearest neighbor search, which trades so
Supported index types are:
- [HNSW](#hnsw)
- [HNSW](#hnsw) - added in 0.5.0
- [IVFFlat](#ivfflat)
## HNSW
@@ -215,8 +209,6 @@ L2 distance
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
Note: Use `halfvec_l2_ops` for `halfvec` and `sparsevec_l2_ops` for `sparsevec` (and similar with the other distance functions)
Inner product
```sql
@@ -229,30 +221,7 @@ Cosine distance
CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops);
```
L1 distance
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l1_ops);
```
Hamming distance
```sql
CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops);
```
Jaccard distance
```sql
CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops);
```
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions
- `bit` - up to 64,000 dimensions
- `sparsevec` - up to 1,000 non-zero elements
Vectors with up to 2,000 dimensions can be indexed.
### Index Options
@@ -306,21 +275,17 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t
Like other index types, its faster to create an index after loading your initial data
You can also speed up index creation by increasing the number of parallel workers (2 by default)
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql
SET max_parallel_maintenance_workers = 7; -- plus leader
```
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
Use [binary quantization](#binary-quantization) for faster build times at scale
For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql
SELECT phase, round(100.0 * blocks_done / nullif(blocks_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -349,8 +314,6 @@ L2 distance
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100);
```
Note: Use `halfvec_l2_ops` for `halfvec` (and similar with the other distance functions)
Inner product
```sql
@@ -363,17 +326,7 @@ Cosine distance
CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
```
Hamming distance
```sql
CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100);
```
Supported types are:
- `vector` - up to 2,000 dimensions
- `halfvec` - up to 4,000 dimensions
- `bit` - up to 64,000 dimensions
Vectors with up to 2,000 dimensions can be indexed.
### Query Options
@@ -406,7 +359,7 @@ For a large number of workers, you may also need to increase `max_parallel_worke
### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) with Postgres 12+
```sql
SELECT phase, round(100.0 * tuples_done / nullif(tuples_total, 0), 1) AS "%" FROM pg_stat_progress_create_index;
@@ -423,203 +376,30 @@ Note: `%` is only populated during the `loading tuples` phase
## Filtering
There are a few ways to index nearest neighbor queries with a `WHERE` clause.
There are a few ways to index nearest neighbor queries with a `WHERE` clause
```sql
SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
A good place to start is creating an index on the filter column. This can provide fast, exact nearest neighbor search in many cases. Postgres has a number of [index types](https://www.postgresql.org/docs/current/indexes-types.html) for this: B-tree (default), hash, GiST, SP-GiST, GIN, and BRIN.
Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
```sql
CREATE INDEX ON items (category_id);
```
For multiple columns, consider a [multicolumn index](https://www.postgresql.org/docs/current/indexes-multicolumn.html).
```sql
CREATE INDEX ON items (location_id, category_id);
```
Exact indexes work well for conditions that match a low percentage of rows. Otherwise, [approximate indexes](#indexing) can work better.
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
```sql
SET hnsw.iterative_scan = strict_order;
```
If filtering by only a few distinct values, consider [partial indexing](https://www.postgresql.org/docs/current/indexes-partial.html).
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
```sql
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WHERE (category_id = 123);
```
If filtering by many different values, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html).
Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
```sql
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
```
## Iterative Index Scans
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
Iterative scans can use strict or relaxed ordering.
Strict ensures results are in the exact order by distance
```sql
SET hnsw.iterative_scan = strict_order;
```
Relaxed allows results to be slightly out of order by distance, but provides better recall
```sql
SET hnsw.iterative_scan = relaxed_order;
# or
SET ivfflat.iterative_scan = relaxed_order;
```
With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-CTE-MATERIALIZATION) to get strict ordering
```sql
WITH relaxed_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
) SELECT * FROM relaxed_results ORDER BY distance + 0;
```
Note: `+ 0` is needed for Postgres 17+
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
```sql
WITH nearest_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items ORDER BY distance LIMIT 5
) SELECT * FROM nearest_results WHERE distance < 5 ORDER BY distance;
```
Note: Place any other filters inside the CTE
### Iterative Scan Options
Since scanning a large portion of an approximate index is expensive, there are options to control when a scan ends.
#### HNSW
Specify the max number of tuples to visit (20,000 by default)
```sql
SET hnsw.max_scan_tuples = 20000;
```
Note: This is approximate and does not affect the initial scan
Specify the max amount of memory to use, as a multiple of `work_mem` (1 by default)
```sql
SET hnsw.scan_mem_multiplier = 2;
```
Note: Try increasing this if increasing `hnsw.max_scan_tuples` does not improve recall
#### IVFFlat
Specify the max number of probes
```sql
SET ivfflat.max_probes = 100;
```
Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used
## Half-Precision Vectors
Use the `halfvec` type to store half-precision vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3));
```
## Half-Precision Indexing
Index vectors at half precision for smaller indexes
```sql
CREATE INDEX ON items USING hnsw ((embedding::halfvec(3)) halfvec_l2_ops);
```
Get the nearest neighbors
```sql
SELECT * FROM items ORDER BY embedding::halfvec(3) <-> '[1,2,3]' LIMIT 5;
```
## Binary Vectors
Use the `bit` type to store binary vectors ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/imagehash/example.py))
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES ('000'), ('111');
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5;
```
Also supports Jaccard distance (`<%>`)
## Binary Quantization
Use expression indexing for binary quantization
```sql
CREATE INDEX ON items USING hnsw ((binary_quantize(embedding)::bit(3)) bit_hamming_ops);
```
Get the nearest neighbors by Hamming distance
```sql
SELECT * FROM items ORDER BY binary_quantize(embedding)::bit(3) <~> binary_quantize('[1,-2,3]') LIMIT 5;
```
Re-rank by the original vectors for better recall
```sql
SELECT * FROM (
SELECT * FROM items ORDER BY binary_quantize(embedding)::bit(3) <~> binary_quantize('[1,-2,3]') LIMIT 20
) ORDER BY embedding <=> '[1,-2,3]' LIMIT 5;
```
## Sparse Vectors
Use the `sparsevec` type to store sparse vectors
```sql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding sparsevec(5));
```
Insert vectors
```sql
INSERT INTO items (embedding) VALUES ('{1:1,3:2,5:3}/5'), ('{1:4,3:5,5:6}/5');
```
The format is `{index1:value1,index2:value2}/dimensions` and indices start at 1 like SQL arrays
Get the nearest neighbors by L2 distance
```sql
SELECT * FROM items ORDER BY embedding <-> '{1:3,3:1,5:2}/5' LIMIT 5;
```
## Hybrid Search
Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search.
@@ -629,29 +409,7 @@ SELECT id, content FROM items, plainto_tsquery('hello search') query
WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
```
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search/cross_encoder.py) to combine results.
## Indexing Subvectors
Use expression indexing to index subvectors
```sql
CREATE INDEX ON items USING hnsw ((subvector(embedding, 1, 3)::vector(3)) vector_cosine_ops);
```
Get the nearest neighbors by cosine distance
```sql
SELECT * FROM items ORDER BY subvector(embedding, 1, 3)::vector(3) <=> subvector('[1,2,3,4,5]'::vector, 1, 3) LIMIT 5;
```
Re-rank by the full vectors for better recall
```sql
SELECT * FROM (
SELECT * FROM items ORDER BY subvector(embedding, 1, 3)::vector(3) <=> subvector('[1,2,3,4,5]'::vector, 1, 3) LIMIT 20
) ORDER BY embedding <=> '[1,2,3,4,5]' LIMIT 5;
```
You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search_rrf.py) or a [cross-encoder](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py) to combine results.
## Performance
@@ -671,13 +429,9 @@ SHOW shared_buffers;
Be sure to restart Postgres for changes to take effect.
### Storing
Use the `halfvec` type instead of `vector` for a smaller working set.
### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py)).
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
@@ -689,8 +443,6 @@ Add any indexes *after* loading the initial data for best performance.
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
In production environments, create indexes concurrently to avoid blocking writes.
```sql
@@ -699,10 +451,10 @@ CREATE INDEX CONCURRENTLY ...
### Querying
Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
Use `EXPLAIN ANALYZE` to debug performance.
```sql
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
#### Exact Search
@@ -721,8 +473,6 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
#### Approximate Search
Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
```sql
@@ -738,20 +488,23 @@ REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name;
```
## Scaling
For a smaller working set:
1. Use the `halfvec` type instead of `vector` for tables
2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Monitoring
Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance.
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
```sql
CREATE EXTENSION pg_stat_statements;
```
Get the most time-consuming queries with:
```sql
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Monitor recall by comparing results from approximate search with exact search.
@@ -762,46 +515,42 @@ SELECT ...
COMMIT;
```
## Scaling
Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus.py)).
## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
Language | Libraries / Examples
--- | ---
Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol)
C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
Elixir | [pgvector-elixir](https://github.com/pgvector/pgvector-elixir)
Erlang | [pgvector-erlang](https://github.com/pgvector/pgvector-erlang)
Fortran | [pgvector-fortran](https://github.com/pgvector/pgvector-fortran)
Gleam | [pgvector-gleam](https://github.com/pgvector/pgvector-gleam)
Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl)
Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig)
## Frequently Asked Questions
@@ -816,11 +565,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
Youll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
#### Can I store vectors with different dimensions in the same column?
You can use `vector` as the type (instead of `vector(n)`).
You can use `vector` as the type (instead of `vector(3)`).
```sql
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -867,6 +616,18 @@ and query with:
SELECT * FROM items ORDER BY embedding::vector(3) <-> '[3,1,2]' LIMIT 5;
```
#### Are binary vectors supported?
You can store binary vectors and perform exact nearest neighbor search by Hamming distance in Postgres without an extension ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/hash_image_search.py)).
```tsql
CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3));
INSERT INTO items (embedding) VALUES (B'000'), (B'111');
SELECT * FROM items ORDER BY bit_count(embedding # B'101') LIMIT 5;
```
Indexing is not currently supported.
#### Do indexes need to fit into memory?
No, but like other index types, youll likely see better performance if they do. You can get the size of an index with:
@@ -875,13 +636,11 @@ No, but like other index types, youll likely see better performance if they d
SELECT pg_size_pretty(pg_relation_size('index_name'));
```
Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
## Troubleshooting
#### Why isnt a query using an index?
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator (not an expression) in ascending order.
The query needs to have an `ORDER BY` and `LIMIT`, and the `ORDER BY` must be the result of a distance operator, not an expression.
```sql
-- index
@@ -922,7 +681,7 @@ ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
#### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`), which is 40 by default. There may be even less results due to dead tuples or filtering conditions in the query. Enabling [iterative index scans](#iterative-index-scans) can help address this.
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
@@ -934,17 +693,12 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name;
```
Results can also be limited by the number of probes (`ivfflat.probes`). Enabling [iterative index scans](#iterative-index-scans) can address this.
Results can also be limited by the number of probes (`ivfflat.probes`).
Also, note that `NULL` vectors are not indexed (as well as zero vectors for cosine distance).
## Reference
- [Vector](#vector-type)
- [Halfvec](#halfvec-type)
- [Bit](#bit-type)
- [Sparsevec](#sparsevec-type)
### Vector Type
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single-precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 16,000 dimensions.
@@ -956,23 +710,18 @@ Operator | Description | Added
\+ | element-wise addition |
\- | element-wise subtraction |
\* | element-wise multiplication | 0.5.0
\|\| | concatenate | 0.7.0
<-> | Euclidean distance |
<#> | negative inner product |
<=> | cosine distance |
<+> | taxicab distance | 0.7.0
### Vector Functions
Function | Description | Added
--- | --- | ---
binary_quantize(vector) → bit | binary quantize | 0.7.0
cosine_distance(vector, vector) → double precision | cosine distance |
inner_product(vector, vector) → double precision | inner product |
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
l2_distance(vector, vector) → double precision | Euclidean distance |
l2_normalize(vector) → vector | Normalize with Euclidean norm | 0.7.0
subvector(vector, integer, integer) → vector | subvector | 0.7.0
l1_distance(vector, vector) → double precision | taxicab distance | 0.5.0
vector_dims(vector) → integer | number of dimensions |
vector_norm(vector) → double precision | Euclidean norm |
@@ -983,85 +732,26 @@ Function | Description | Added
avg(vector) → vector | average |
sum(vector) → vector | sum | 0.5.0
### Halfvec Type
### Intvec Type
Each half vector takes `2 * dimensions + 8` bytes of storage. Each element is a half-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Half vectors can have up to 16,000 dimensions.
Each int vector takes `dimensions + 8` bytes of storage. Each element is a single byte signed integer. Int vectors can have up to 16,000 dimensions.
### Halfvec Operators
Operator | Description | Added
--- | --- | ---
\+ | element-wise addition | 0.7.0
\- | element-wise subtraction | 0.7.0
\* | element-wise multiplication | 0.7.0
\|\| | concatenate | 0.7.0
<-> | Euclidean distance | 0.7.0
<#> | negative inner product | 0.7.0
<=> | cosine distance | 0.7.0
<+> | taxicab distance | 0.7.0
### Halfvec Functions
Function | Description | Added
--- | --- | ---
binary_quantize(halfvec) → bit | binary quantize | 0.7.0
cosine_distance(halfvec, halfvec) → double precision | cosine distance | 0.7.0
inner_product(halfvec, halfvec) → double precision | inner product | 0.7.0
l1_distance(halfvec, halfvec) → double precision | taxicab distance | 0.7.0
l2_distance(halfvec, halfvec) → double precision | Euclidean distance | 0.7.0
l2_norm(halfvec) → double precision | Euclidean norm | 0.7.0
l2_normalize(halfvec) → halfvec | Normalize with Euclidean norm | 0.7.0
subvector(halfvec, integer, integer) → halfvec | subvector | 0.7.0
vector_dims(halfvec) → integer | number of dimensions | 0.7.0
### Halfvec Aggregate Functions
Function | Description | Added
--- | --- | ---
avg(halfvec) → halfvec | average | 0.7.0
sum(halfvec) → halfvec | sum | 0.7.0
### Bit Type
Each bit vector takes `dimensions / 8 + 8` bytes of storage. See the [Postgres docs](https://www.postgresql.org/docs/current/datatype-bit.html) for more info.
### Bit Operators
Operator | Description | Added
--- | --- | ---
<~> | Hamming distance | 0.7.0
<%> | Jaccard distance | 0.7.0
### Bit Functions
Function | Description | Added
--- | --- | ---
hamming_distance(bit, bit) → double precision | Hamming distance | 0.7.0
jaccard_distance(bit, bit) → double precision | Jaccard distance | 0.7.0
### Sparsevec Type
Each sparse vector takes `8 * non-zero elements + 16` bytes of storage. Each element is a single-precision floating-point number, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Sparse vectors can have up to 16,000 non-zero elements.
### Sparsevec Operators
### Intvec Operators
Operator | Description | Added
--- | --- | ---
<-> | Euclidean distance | 0.7.0
<#> | negative inner product | 0.7.0
<=> | cosine distance | 0.7.0
<+> | taxicab distance | 0.7.0
### Sparsevec Functions
### Intvec Functions
Function | Description | Added
--- | --- | ---
cosine_distance(sparsevec, sparsevec) → double precision | cosine distance | 0.7.0
inner_product(sparsevec, sparsevec) → double precision | inner product | 0.7.0
l1_distance(sparsevec, sparsevec) → double precision | taxicab distance | 0.7.0
l2_distance(sparsevec, sparsevec) → double precision | Euclidean distance | 0.7.0
l2_norm(sparsevec) → double precision | Euclidean norm | 0.7.0
l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
cosine_distance(intvec, intvec) → double precision | cosine distance | 0.7.0
inner_product(intvec, intvec) → double precision | inner product | 0.7.0
l2_distance(intvec, intvec) → double precision | Euclidean distance | 0.7.0
l1_distance(intvec, intvec) → double precision | taxicab distance | 0.7.0
## Installation Notes - Linux and Mac
@@ -1070,7 +760,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh
export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
export PG_CONFIG=/Library/PostgreSQL/16/bin/pg_config
```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1081,11 +771,11 @@ sudo --preserve-env=PG_CONFIG make install
A few common paths on Mac are:
- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
- EDB installer - `/Library/PostgreSQL/16/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@16/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@16/bin/pg_config`
Note: Replace `18` with your Postgres server version
Note: Replace `16` with your Postgres server version
### Missing Header
@@ -1094,20 +784,14 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use:
```sh
sudo apt install postgresql-server-dev-18
sudo apt install postgresql-server-dev-16
```
Note: Replace `18` with your Postgres server version
Note: Replace `16` with your Postgres server version
### Missing SDK
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists.
```sh
pg_config --cppflags
```
Reinstall Postgres to fix this.
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
### Portability
@@ -1125,14 +809,6 @@ make OPTFLAGS=""
If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct.
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure youre using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+.
### Permissions
If installation fails with `Access is denied`, re-run the installation instructions as an administrator.
@@ -1144,38 +820,17 @@ If installation fails with `Access is denied`, re-run the installation instructi
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh
docker pull pgvector/pgvector:pg18-trixie
docker pull pgvector/pgvector:pg16
```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
Supported tags are:
- `pg18-trixie`, `0.8.4-pg18-trixie`
- `pg18-bookworm`, `0.8.4-pg18-bookworm`, `pg18`, `0.8.4-pg18`
- `pg17-trixie`, `0.8.4-pg17-trixie`
- `pg17-bookworm`, `0.8.4-pg17-bookworm`, `pg17`, `0.8.4-pg17`
- `pg16-trixie`, `0.8.4-pg16-trixie`
- `pg16-bookworm`, `0.8.4-pg16-bookworm`, `pg16`, `0.8.4-pg16`
- `pg15-trixie`, `0.8.4-pg15-trixie`
- `pg15-bookworm`, `0.8.4-pg15-bookworm`, `pg15`, `0.8.4-pg15`
- `pg14-trixie`, `0.8.4-pg14-trixie`
- `pg14-bookworm`, `0.8.4-pg14-bookworm`, `pg14`, `0.8.4-pg14`
- `pg13-trixie`, `0.8.4-pg13-trixie`
- `pg13-bookworm`, `0.8.4-pg13-bookworm`, `pg13`, `0.8.4-pg13`
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `16` with your Postgres server version, and run it the same way).
You can also build the image manually:
```sh
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.2 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
```
### Homebrew
@@ -1186,7 +841,7 @@ With Homebrew Postgres, you can use:
brew install pgvector
```
Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
Note: This only adds it to the `postgresql@14` formula
### PGXN
@@ -1201,29 +856,29 @@ pgxn install vector
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh
sudo apt install postgresql-18-pgvector
sudo apt install postgresql-16-pgvector
```
Note: Replace `18` with your Postgres server version
Note: Replace `16` with your Postgres server version
### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh
sudo yum install pgvector_18
sudo yum install pgvector_16
# or
sudo dnf install pgvector_18
sudo dnf install pgvector_16
```
Note: Replace `18` with your Postgres server version
Note: Replace `16` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql17-pgvector
pkg install postgresql15-pg_vector
```
or the port with:
@@ -1233,14 +888,6 @@ cd /usr/ports/databases/pgvector
make install
```
### APK
Install the Alpine package with:
```sh
apk add postgresql-pgvector
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -1273,6 +920,36 @@ You can check the version in the current database with:
SELECT extversion FROM pg_extension WHERE extname = 'vector';
```
## Upgrade Notes
### 0.6.0
#### Postgres 12
If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`:
```sql
ALTER TYPE vector SET (STORAGE = external);
```
Then run `make install` and `ALTER EXTENSION vector UPDATE;`.
#### Docker
The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag).
```sh
docker pull pgvector/pgvector:pg16
# or
docker pull pgvector/pgvector:0.6.0-pg16
```
Also, if youve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
```sh
docker run --shm-size=1g ...
```
## Thanks
Thanks to:
@@ -1329,7 +1006,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
To enable benchmarking:
```sh
make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install
make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
```
To show memory usage:

View File

@@ -1,569 +1,92 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.0'" to load this file. \quit
CREATE FUNCTION l2_normalize(vector) RETURNS vector
CREATE TYPE intvec;
CREATE FUNCTION intvec_in(cstring, oid, integer) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION binary_quantize(vector) RETURNS bit
CREATE FUNCTION intvec_out(intvec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
CREATE FUNCTION intvec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_concat(vector, vector) RETURNS vector
CREATE FUNCTION intvec_recv(internal, oid, integer) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <+> (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR || (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
);
CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_sparsevec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE OPERATOR CLASS vector_l1_ops
FOR TYPE vector USING hnsw AS
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(vector, vector);
CREATE TYPE halfvec;
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
CREATE FUNCTION intvec_send(intvec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE halfvec (
INPUT = halfvec_in,
OUTPUT = halfvec_out,
TYPMOD_IN = halfvec_typmod_in,
RECEIVE = halfvec_recv,
SEND = halfvec_send,
CREATE TYPE intvec (
INPUT = intvec_in,
OUTPUT = intvec_out,
TYPMOD_IN = intvec_typmod_in,
RECEIVE = intvec_recv,
SEND = intvec_send,
STORAGE = external
);
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_dims(halfvec) RETURNS integer
AS 'MODULE_PATHNAME', 'halfvec_vector_dims' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_normalize(halfvec) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION binary_quantize(halfvec) RETURNS bit
AS 'MODULE_PATHNAME', 'halfvec_binary_quantize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_add(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION intvec_l2_squared_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_sub(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION intvec_negative_inner_product(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_mul(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION intvec(intvec, integer, boolean) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_concat(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION array_to_intvec(integer[], integer, boolean) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_lt(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (intvec AS intvec)
WITH FUNCTION intvec(intvec, integer, boolean) AS IMPLICIT;
CREATE FUNCTION halfvec_le(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_eq(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_ne(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_ge(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_gt(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_cmp(halfvec, halfvec) RETURNS int4
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_spherical_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precision[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME', 'vector_combine' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE AGGREGATE avg(halfvec) (
SFUNC = halfvec_accum,
STYPE = double precision[],
FINALFUNC = halfvec_avg,
COMBINEFUNC = halfvec_combine,
INITCOND = '{0}',
PARALLEL = SAFE
);
CREATE AGGREGATE sum(halfvec) (
SFUNC = halfvec_add,
STYPE = halfvec,
COMBINEFUNC = halfvec_add,
PARALLEL = SAFE
);
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (halfvec AS halfvec)
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS vector)
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (vector AS halfvec)
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS real[])
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (integer[] AS halfvec)
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS halfvec)
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS halfvec)
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS halfvec)
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (integer[] AS intvec)
WITH FUNCTION array_to_intvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE OPERATOR <-> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = intvec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR <+> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR + (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_add,
COMMUTATOR = +
);
CREATE OPERATOR - (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_sub
);
CREATE OPERATOR * (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_mul,
COMMUTATOR = *
);
CREATE OPERATOR || (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_concat
);
CREATE OPERATOR < (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_lt,
COMMUTATOR = > , NEGATOR = >= ,
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
CREATE OPERATOR <= (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
CREATE OPERATOR = (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_eq,
COMMUTATOR = = , NEGATOR = <> ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR <> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_ne,
COMMUTATOR = <> , NEGATOR = = ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR >= (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargesel, JOIN = scalargejoinsel
);
CREATE OPERATOR > (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_gt,
COMMUTATOR = < , NEGATOR = <= ,
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
);
CREATE OPERATOR CLASS halfvec_ops
DEFAULT FOR TYPE halfvec USING btree AS
OPERATOR 1 < ,
OPERATOR 2 <= ,
OPERATOR 3 = ,
OPERATOR 4 >= ,
OPERATOR 5 > ,
FUNCTION 1 halfvec_cmp(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 l2_distance(halfvec, halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_l1_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 5 ivfflat_bit_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
CREATE TYPE sparsevec;
CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE sparsevec (
INPUT = sparsevec_in,
OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv,
SEND = sparsevec_send,
STORAGE = external
);
CREATE FUNCTION l2_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_normalize(sparsevec) RETURNS sparsevec
AS 'MODULE_PATHNAME', 'sparsevec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_lt(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_le(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_eq(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_ne(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_ge(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_gt(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_cmp(sparsevec, sparsevec) RETURNS int4
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_negative_inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec(sparsevec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (sparsevec AS sparsevec)
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS vector)
WITH FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (vector AS sparsevec)
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS halfvec)
WITH FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (halfvec AS sparsevec)
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
CREATE OPERATOR <-> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR <+> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR < (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_lt,
COMMUTATOR = > , NEGATOR = >= ,
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
CREATE OPERATOR <= (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
CREATE OPERATOR = (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_eq,
COMMUTATOR = = , NEGATOR = <> ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR <> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_ne,
COMMUTATOR = <> , NEGATOR = = ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR >= (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargesel, JOIN = scalargejoinsel
);
CREATE OPERATOR > (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_gt,
COMMUTATOR = < , NEGATOR = <= ,
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
);
CREATE OPERATOR CLASS sparsevec_ops
DEFAULT FOR TYPE sparsevec USING btree AS
OPERATOR 1 < ,
OPERATOR 2 <= ,
OPERATOR 3 = ,
OPERATOR 4 >= ,
OPERATOR 5 > ,
FUNCTION 1 sparsevec_cmp(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 l2_norm(sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_l1_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS intvec_l2_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <-> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 intvec_l2_squared_distance(intvec, intvec);
CREATE OPERATOR CLASS intvec_ip_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <#> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 intvec_negative_inner_product(intvec, intvec);
CREATE OPERATOR CLASS intvec_cosine_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <=> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 cosine_distance(intvec, intvec),
FUNCTION 2 l2_norm(intvec);

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.1'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.2'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.7.4'" to load this file. \quit

View File

@@ -1,26 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.0'" to load this file. \quit
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

View File

@@ -1,7 +1,7 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION vector" to load this file. \quit
-- vector type
-- type
CREATE TYPE vector;
@@ -29,7 +29,7 @@ CREATE TYPE vector (
STORAGE = external
);
-- vector functions
-- functions
CREATE FUNCTION l2_distance(vector, vector) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -49,17 +49,6 @@ CREATE FUNCTION vector_dims(vector) RETURNS integer
CREATE FUNCTION vector_norm(vector) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_normalize(vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION binary_quantize(vector) RETURNS bit
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(vector, int, int) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector private functions
CREATE FUNCTION vector_add(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -69,8 +58,7 @@ CREATE FUNCTION vector_sub(vector, vector) RETURNS vector
CREATE FUNCTION vector_mul(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_concat(vector, vector) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- private functions
CREATE FUNCTION vector_lt(vector, vector) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -111,7 +99,7 @@ CREATE FUNCTION vector_avg(double precision[]) RETURNS vector
CREATE FUNCTION vector_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector aggregates
-- aggregates
CREATE AGGREGATE avg(vector) (
SFUNC = vector_accum,
@@ -129,7 +117,7 @@ CREATE AGGREGATE sum(vector) (
PARALLEL = SAFE
);
-- vector cast functions
-- cast functions
CREATE FUNCTION vector(vector, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
@@ -149,7 +137,7 @@ CREATE FUNCTION array_to_vector(numeric[], integer, boolean) RETURNS vector
CREATE FUNCTION vector_to_float4(vector, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- vector casts
-- casts
CREATE CAST (vector AS vector)
WITH FUNCTION vector(vector, integer, boolean) AS IMPLICIT;
@@ -169,7 +157,7 @@ CREATE CAST (double precision[] AS vector)
CREATE CAST (numeric[] AS vector)
WITH FUNCTION array_to_vector(numeric[], integer, boolean) AS ASSIGNMENT;
-- vector operators
-- operators
CREATE OPERATOR <-> (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l2_distance,
@@ -186,11 +174,6 @@ CREATE OPERATOR <=> (
COMMUTATOR = '<=>'
);
CREATE OPERATOR <+> (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR + (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_add,
COMMUTATOR = +
@@ -205,10 +188,6 @@ CREATE OPERATOR * (
COMMUTATOR = *
);
CREATE OPERATOR || (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_concat
);
CREATE OPERATOR < (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_lt,
COMMUTATOR = > , NEGATOR = >= ,
@@ -261,24 +240,7 @@ CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnswhandler;
COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method';
-- access method private functions
CREATE FUNCTION ivfflat_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION ivfflat_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_halfvec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_bit_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
CREATE FUNCTION hnsw_sparsevec_support(internal) RETURNS internal
AS 'MODULE_PATHNAME' LANGUAGE C;
-- vector opclasses
-- opclasses
CREATE OPERATOR CLASS vector_ops
DEFAULT FOR TYPE vector USING btree AS
@@ -326,593 +288,106 @@ CREATE OPERATOR CLASS vector_cosine_ops
FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 2 vector_norm(vector);
CREATE OPERATOR CLASS vector_l1_ops
FOR TYPE vector USING hnsw AS
OPERATOR 1 <+> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(vector, vector);
-- intvec type
-- halfvec type
CREATE TYPE intvec;
CREATE TYPE halfvec;
CREATE FUNCTION halfvec_in(cstring, oid, integer) RETURNS halfvec
CREATE FUNCTION intvec_in(cstring, oid, integer) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_out(halfvec) RETURNS cstring
CREATE FUNCTION intvec_out(intvec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_typmod_in(cstring[]) RETURNS integer
CREATE FUNCTION intvec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_recv(internal, oid, integer) RETURNS halfvec
CREATE FUNCTION intvec_recv(internal, oid, integer) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_send(halfvec) RETURNS bytea
CREATE FUNCTION intvec_send(intvec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE halfvec (
INPUT = halfvec_in,
OUTPUT = halfvec_out,
TYPMOD_IN = halfvec_typmod_in,
RECEIVE = halfvec_recv,
SEND = halfvec_send,
CREATE TYPE intvec (
INPUT = intvec_in,
OUTPUT = intvec_out,
TYPMOD_IN = intvec_typmod_in,
RECEIVE = intvec_recv,
SEND = intvec_send,
STORAGE = external
);
-- halfvec functions
-- intvec functions
CREATE FUNCTION l2_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_dims(halfvec) RETURNS integer
AS 'MODULE_PATHNAME', 'halfvec_vector_dims' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(intvec) RETURNS float8
AS 'MODULE_PATHNAME', 'intvec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(halfvec) RETURNS float8
AS 'MODULE_PATHNAME', 'halfvec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- intvec private functions
CREATE FUNCTION l2_normalize(halfvec) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION binary_quantize(halfvec) RETURNS bit
AS 'MODULE_PATHNAME', 'halfvec_binary_quantize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION subvector(halfvec, int, int) RETURNS halfvec
AS 'MODULE_PATHNAME', 'halfvec_subvector' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec private functions
CREATE FUNCTION halfvec_add(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION intvec_l2_squared_distance(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_sub(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION intvec_negative_inner_product(intvec, intvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_mul(halfvec, halfvec) RETURNS halfvec
-- intvec cast functions
CREATE FUNCTION intvec(intvec, integer, boolean) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_concat(halfvec, halfvec) RETURNS halfvec
CREATE FUNCTION array_to_intvec(integer[], integer, boolean) RETURNS intvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_lt(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- intvec casts
CREATE FUNCTION halfvec_le(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (intvec AS intvec)
WITH FUNCTION intvec(intvec, integer, boolean) AS IMPLICIT;
CREATE FUNCTION halfvec_eq(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE CAST (integer[] AS intvec)
WITH FUNCTION array_to_intvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE FUNCTION halfvec_ne(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_ge(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_gt(halfvec, halfvec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_cmp(halfvec, halfvec) RETURNS int4
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_l2_squared_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_negative_inner_product(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_spherical_distance(halfvec, halfvec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_accum(double precision[], halfvec) RETURNS double precision[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_avg(double precision[]) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_combine(double precision[], double precision[]) RETURNS double precision[]
AS 'MODULE_PATHNAME', 'vector_combine' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec aggregates
CREATE AGGREGATE avg(halfvec) (
SFUNC = halfvec_accum,
STYPE = double precision[],
FINALFUNC = halfvec_avg,
COMBINEFUNC = halfvec_combine,
INITCOND = '{0}',
PARALLEL = SAFE
);
CREATE AGGREGATE sum(halfvec) (
SFUNC = halfvec_add,
STYPE = halfvec,
COMBINEFUNC = halfvec_add,
PARALLEL = SAFE
);
-- halfvec cast functions
CREATE FUNCTION halfvec(halfvec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_vector(halfvec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_halfvec(vector, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(integer[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(real[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(double precision[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_halfvec(numeric[], integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_float4(halfvec, integer, boolean) RETURNS real[]
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- halfvec casts
CREATE CAST (halfvec AS halfvec)
WITH FUNCTION halfvec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS vector)
WITH FUNCTION halfvec_to_vector(halfvec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (vector AS halfvec)
WITH FUNCTION vector_to_halfvec(vector, integer, boolean) AS IMPLICIT;
CREATE CAST (halfvec AS real[])
WITH FUNCTION halfvec_to_float4(halfvec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (integer[] AS halfvec)
WITH FUNCTION array_to_halfvec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS halfvec)
WITH FUNCTION array_to_halfvec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS halfvec)
WITH FUNCTION array_to_halfvec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS halfvec)
WITH FUNCTION array_to_halfvec(numeric[], integer, boolean) AS ASSIGNMENT;
-- halfvec operators
-- intvec operators
CREATE OPERATOR <-> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l2_distance,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_negative_inner_product,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = intvec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = cosine_distance,
LEFTARG = intvec, RIGHTARG = intvec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR <+> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR + (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_add,
COMMUTATOR = +
);
CREATE OPERATOR - (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_sub
);
CREATE OPERATOR * (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_mul,
COMMUTATOR = *
);
CREATE OPERATOR || (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_concat
);
CREATE OPERATOR < (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_lt,
COMMUTATOR = > , NEGATOR = >= ,
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
CREATE OPERATOR <= (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
CREATE OPERATOR = (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_eq,
COMMUTATOR = = , NEGATOR = <> ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR <> (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_ne,
COMMUTATOR = <> , NEGATOR = = ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR >= (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargesel, JOIN = scalargejoinsel
);
CREATE OPERATOR > (
LEFTARG = halfvec, RIGHTARG = halfvec, PROCEDURE = halfvec_gt,
COMMUTATOR = < , NEGATOR = <= ,
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
);
-- halfvec opclasses
CREATE OPERATOR CLASS halfvec_ops
DEFAULT FOR TYPE halfvec USING btree AS
OPERATOR 1 < ,
OPERATOR 2 <= ,
OPERATOR 3 = ,
OPERATOR 4 >= ,
OPERATOR 5 > ,
FUNCTION 1 halfvec_cmp(halfvec, halfvec);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 l2_distance(halfvec, halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING ivfflat AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 halfvec_spherical_distance(halfvec, halfvec),
FUNCTION 4 l2_norm(halfvec),
FUNCTION 5 ivfflat_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_l2_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <-> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_l2_squared_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_ip_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <#> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_cosine_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <=> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 halfvec_negative_inner_product(halfvec, halfvec),
FUNCTION 2 l2_norm(halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
CREATE OPERATOR CLASS halfvec_l1_ops
FOR TYPE halfvec USING hnsw AS
OPERATOR 1 <+> (halfvec, halfvec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(halfvec, halfvec),
FUNCTION 3 hnsw_halfvec_support(internal);
-- bit functions
CREATE FUNCTION hamming_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION jaccard_distance(bit, bit) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- bit operators
CREATE OPERATOR <~> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = hamming_distance,
COMMUTATOR = '<~>'
);
CREATE OPERATOR <%> (
LEFTARG = bit, RIGHTARG = bit, PROCEDURE = jaccard_distance,
COMMUTATOR = '<%>'
);
-- bit opclasses
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING ivfflat AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hamming_distance(bit, bit),
FUNCTION 5 ivfflat_bit_support(internal);
CREATE OPERATOR CLASS bit_hamming_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <~> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 hamming_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
CREATE OPERATOR CLASS bit_jaccard_ops
FOR TYPE bit USING hnsw AS
OPERATOR 1 <%> (bit, bit) FOR ORDER BY float_ops,
FUNCTION 1 jaccard_distance(bit, bit),
FUNCTION 3 hnsw_bit_support(internal);
--- sparsevec type
CREATE TYPE sparsevec;
CREATE FUNCTION sparsevec_in(cstring, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_out(sparsevec) RETURNS cstring
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_typmod_in(cstring[]) RETURNS integer
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_recv(internal, oid, integer) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_send(sparsevec) RETURNS bytea
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE TYPE sparsevec (
INPUT = sparsevec_in,
OUTPUT = sparsevec_out,
TYPMOD_IN = sparsevec_typmod_in,
RECEIVE = sparsevec_recv,
SEND = sparsevec_send,
STORAGE = external
);
-- sparsevec functions
CREATE FUNCTION l2_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_inner_product' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION cosine_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_cosine_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l1_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l1_distance' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_norm(sparsevec) RETURNS float8
AS 'MODULE_PATHNAME', 'sparsevec_l2_norm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION l2_normalize(sparsevec) RETURNS sparsevec
AS 'MODULE_PATHNAME', 'sparsevec_l2_normalize' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec private functions
CREATE FUNCTION sparsevec_lt(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_le(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_eq(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_ne(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_ge(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_gt(sparsevec, sparsevec) RETURNS bool
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_cmp(sparsevec, sparsevec) RETURNS int4
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_l2_squared_distance(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_negative_inner_product(sparsevec, sparsevec) RETURNS float8
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec cast functions
CREATE FUNCTION sparsevec(sparsevec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION vector_to_sparsevec(vector, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) RETURNS vector
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) RETURNS halfvec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(integer[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(real[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(double precision[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION array_to_sparsevec(numeric[], integer, boolean) RETURNS sparsevec
AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
-- sparsevec casts
CREATE CAST (sparsevec AS sparsevec)
WITH FUNCTION sparsevec(sparsevec, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS vector)
WITH FUNCTION sparsevec_to_vector(sparsevec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (vector AS sparsevec)
WITH FUNCTION vector_to_sparsevec(vector, integer, boolean) AS IMPLICIT;
CREATE CAST (sparsevec AS halfvec)
WITH FUNCTION sparsevec_to_halfvec(sparsevec, integer, boolean) AS ASSIGNMENT;
CREATE CAST (halfvec AS sparsevec)
WITH FUNCTION halfvec_to_sparsevec(halfvec, integer, boolean) AS IMPLICIT;
CREATE CAST (integer[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(integer[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (real[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(real[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (double precision[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(double precision[], integer, boolean) AS ASSIGNMENT;
CREATE CAST (numeric[] AS sparsevec)
WITH FUNCTION array_to_sparsevec(numeric[], integer, boolean) AS ASSIGNMENT;
-- sparsevec operators
CREATE OPERATOR <-> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR <#> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_negative_inner_product,
COMMUTATOR = '<#>'
);
CREATE OPERATOR <=> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = cosine_distance,
COMMUTATOR = '<=>'
);
CREATE OPERATOR <+> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = l1_distance,
COMMUTATOR = '<+>'
);
CREATE OPERATOR < (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_lt,
COMMUTATOR = > , NEGATOR = >= ,
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
CREATE OPERATOR <= (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
CREATE OPERATOR = (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_eq,
COMMUTATOR = = , NEGATOR = <> ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR <> (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_ne,
COMMUTATOR = <> , NEGATOR = = ,
RESTRICT = eqsel, JOIN = eqjoinsel
);
CREATE OPERATOR >= (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargesel, JOIN = scalargejoinsel
);
CREATE OPERATOR > (
LEFTARG = sparsevec, RIGHTARG = sparsevec, PROCEDURE = sparsevec_gt,
COMMUTATOR = < , NEGATOR = <= ,
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
);
-- sparsevec opclasses
CREATE OPERATOR CLASS sparsevec_ops
DEFAULT FOR TYPE sparsevec USING btree AS
OPERATOR 1 < ,
OPERATOR 2 <= ,
OPERATOR 3 = ,
OPERATOR 4 >= ,
OPERATOR 5 > ,
FUNCTION 1 sparsevec_cmp(sparsevec, sparsevec);
CREATE OPERATOR CLASS sparsevec_l2_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <-> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_l2_squared_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_ip_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <#> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_cosine_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <=> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 sparsevec_negative_inner_product(sparsevec, sparsevec),
FUNCTION 2 l2_norm(sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
CREATE OPERATOR CLASS sparsevec_l1_ops
FOR TYPE sparsevec USING hnsw AS
OPERATOR 1 <+> (sparsevec, sparsevec) FOR ORDER BY float_ops,
FUNCTION 1 l1_distance(sparsevec, sparsevec),
FUNCTION 3 hnsw_sparsevec_support(internal);
-- intvec opclasses
CREATE OPERATOR CLASS intvec_l2_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <-> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 intvec_l2_squared_distance(intvec, intvec);
CREATE OPERATOR CLASS intvec_ip_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <#> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 intvec_negative_inner_product(intvec, intvec);
CREATE OPERATOR CLASS intvec_cosine_ops
FOR TYPE intvec USING hnsw AS
OPERATOR 1 <=> (intvec, intvec) FOR ORDER BY float_ops,
FUNCTION 1 cosine_distance(intvec, intvec),
FUNCTION 2 l2_norm(intvec);

View File

@@ -1,224 +0,0 @@
#include "postgres.h"
#include "bitutils.h"
#include "halfvec.h" /* for USE_DISPATCH and USE_TARGET_CLONES */
#include "port/pg_bitutils.h"
#if defined(USE_DISPATCH)
#define BIT_DISPATCH
#endif
#ifdef BIT_DISPATCH
#include <immintrin.h>
#if defined(USE__GET_CPUID)
#include <cpuid.h>
#else
#include <intrin.h>
#endif
#ifdef _MSC_VER
#define TARGET_AVX512_POPCOUNT
#else
#define TARGET_AVX512_POPCOUNT __attribute__((target("avx512f,avx512vpopcntdq")))
#endif
#endif
/* Disable for LLVM due to crash with bitcode generation */
#if defined(USE_TARGET_CLONES) && !defined(__POPCNT__) && !defined(__llvm__)
#define BIT_TARGET_CLONES __attribute__((target_clones("default", "popcnt")))
#else
#define BIT_TARGET_CLONES
#endif
/* Use built-ins when possible for Postgres < 19 for inlining */
#if PG_VERSION_NUM >= 190000
#define popcount64(x) pg_popcount64(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
#define popcount64(x) __builtin_popcountl(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8)
#define popcount64(x) __builtin_popcountll(x)
#elif !defined(_MSC_VER)
/* Fails to resolve with MSVC */
#define popcount64(x) pg_popcount64(x)
#endif
uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);
double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb);
BIT_TARGET_CLONES static uint64
BitHammingDistanceDefault(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance)
{
#ifdef popcount64
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
uint64 axs;
uint64 bxs;
/* Ensure aligned */
memcpy(&axs, ax, sizeof(uint64));
memcpy(&bxs, bx, sizeof(uint64));
distance += popcount64(axs ^ bxs);
ax += sizeof(uint64);
bx += sizeof(uint64);
}
#endif
for (uint32 i = 0; i < bytes; i++)
distance += pg_number_of_ones[ax[i] ^ bx[i]];
return distance;
}
#ifdef BIT_DISPATCH
TARGET_AVX512_POPCOUNT static uint64
BitHammingDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance)
{
__m512i dist = _mm512_setzero_si512();
for (; bytes >= sizeof(__m512i); bytes -= sizeof(__m512i))
{
__m512i axs = _mm512_loadu_si512((const __m512i *) ax);
__m512i bxs = _mm512_loadu_si512((const __m512i *) bx);
dist = _mm512_add_epi64(dist, _mm512_popcnt_epi64(_mm512_xor_si512(axs, bxs)));
ax += sizeof(__m512i);
bx += sizeof(__m512i);
}
distance += _mm512_reduce_add_epi64(dist);
return BitHammingDistanceDefault(bytes, ax, bx, distance);
}
#endif
BIT_TARGET_CLONES static double
BitJaccardDistanceDefault(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb)
{
#ifdef popcount64
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
uint64 axs;
uint64 bxs;
/* Ensure aligned */
memcpy(&axs, ax, sizeof(uint64));
memcpy(&bxs, bx, sizeof(uint64));
ab += popcount64(axs & bxs);
aa += popcount64(axs);
bb += popcount64(bxs);
ax += sizeof(uint64);
bx += sizeof(uint64);
}
#endif
for (uint32 i = 0; i < bytes; i++)
{
ab += pg_number_of_ones[ax[i] & bx[i]];
aa += pg_number_of_ones[ax[i]];
bb += pg_number_of_ones[bx[i]];
}
if (ab == 0)
return 1;
else
return 1 - (ab / ((double) (aa + bb - ab)));
}
#ifdef BIT_DISPATCH
TARGET_AVX512_POPCOUNT static double
BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb)
{
__m512i abx = _mm512_setzero_si512();
__m512i aax = _mm512_setzero_si512();
__m512i bbx = _mm512_setzero_si512();
for (; bytes >= sizeof(__m512i); bytes -= sizeof(__m512i))
{
__m512i axs = _mm512_loadu_si512((const __m512i *) ax);
__m512i bxs = _mm512_loadu_si512((const __m512i *) bx);
abx = _mm512_add_epi64(abx, _mm512_popcnt_epi64(_mm512_and_si512(axs, bxs)));
aax = _mm512_add_epi64(aax, _mm512_popcnt_epi64(axs));
bbx = _mm512_add_epi64(bbx, _mm512_popcnt_epi64(bxs));
ax += sizeof(__m512i);
bx += sizeof(__m512i);
}
ab += _mm512_reduce_add_epi64(abx);
aa += _mm512_reduce_add_epi64(aax);
bb += _mm512_reduce_add_epi64(bbx);
return BitJaccardDistanceDefault(bytes, ax, bx, ab, aa, bb);
}
#endif
#ifdef BIT_DISPATCH
#define CPU_FEATURE_OSXSAVE (1 << 27) /* F1 ECX */
#define CPU_FEATURE_AVX512F (1 << 16) /* F7,0 EBX */
#define CPU_FEATURE_AVX512VPOPCNTDQ (1 << 14) /* F7,0 ECX */
#ifdef _MSC_VER
#define TARGET_XSAVE
#else
#define TARGET_XSAVE __attribute__((target("xsave")))
#endif
TARGET_XSAVE static bool
SupportsAvx512Popcount(void)
{
unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuid(exx, 1);
#endif
/* Check OS supports XSAVE */
if ((exx[2] & CPU_FEATURE_OSXSAVE) != CPU_FEATURE_OSXSAVE)
return false;
/* Check XMM, YMM, and ZMM registers are enabled */
if ((_xgetbv(0) & 0xe6) != 0xe6)
return false;
#if defined(USE__GET_CPUID)
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuidex(exx, 7, 0);
#endif
/* Check AVX512F */
if ((exx[1] & CPU_FEATURE_AVX512F) != CPU_FEATURE_AVX512F)
return false;
/* Check AVX512VPOPCNTDQ */
return (exx[2] & CPU_FEATURE_AVX512VPOPCNTDQ) == CPU_FEATURE_AVX512VPOPCNTDQ;
}
#endif
void
BitvecInit(void)
{
/*
* Could skip pointer when single function, but no difference in
* performance
*/
BitHammingDistance = BitHammingDistanceDefault;
BitJaccardDistance = BitJaccardDistanceDefault;
#ifdef BIT_DISPATCH
if (SupportsAvx512Popcount())
{
BitHammingDistance = BitHammingDistanceAvx512Popcount;
BitJaccardDistance = BitJaccardDistanceAvx512Popcount;
}
#endif
}

View File

@@ -1,16 +0,0 @@
#ifndef BITUTILS_H
#define BITUTILS_H
#include "postgres.h"
/* Check version in first header */
#if PG_VERSION_NUM < 130000
#error "Requires PostgreSQL 13+"
#endif
extern uint64 (*BitHammingDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 distance);
extern double (*BitJaccardDistance) (uint32 bytes, unsigned char *ax, unsigned char *bx, uint64 ab, uint64 aa, uint64 bb);
void BitvecInit(void);
#endif

View File

@@ -1,70 +0,0 @@
#include "postgres.h"
#include "bitutils.h"
#include "bitvec.h"
#include "fmgr.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate and initialize a new bit vector
*/
VarBit *
InitBitVector(int dim)
{
VarBit *result;
int size;
size = VARBITTOTALLEN(dim);
result = (VarBit *) palloc0(size);
SET_VARSIZE(result, size);
VARBITLEN(result) = dim;
return result;
}
/*
* Ensure same dimensions
*/
static inline void
CheckDims(VarBit *a, VarBit *b)
{
if (VARBITLEN(a) != VARBITLEN(b))
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different bit lengths %u and %u", VARBITLEN(a), VARBITLEN(b))));
}
/*
* Get the Hamming distance between two bit vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hamming_distance);
Datum
hamming_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8((double) BitHammingDistance(VARBITBYTES(a), VARBITS(a), VARBITS(b), 0));
}
/*
* Get the Jaccard distance between two bit vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(jaccard_distance);
Datum
jaccard_distance(PG_FUNCTION_ARGS)
{
VarBit *a = PG_GETARG_VARBIT_P(0);
VarBit *b = PG_GETARG_VARBIT_P(1);
CheckDims(a, b);
PG_RETURN_FLOAT8(BitJaccardDistance(VARBITBYTES(a), VARBITS(a), VARBITS(b), 0, 0, 0));
}

View File

@@ -1,8 +0,0 @@
#ifndef BITVEC_H
#define BITVEC_H
#include "utils/varbit.h"
VarBit *InitBitVector(int dim);
#endif

View File

@@ -1,300 +0,0 @@
#include "postgres.h"
#include <math.h>
#include "halfutils.h"
#include "halfvec.h"
#ifdef HALFVEC_DISPATCH
#include <immintrin.h>
#if defined(USE__GET_CPUID)
#include <cpuid.h>
#else
#include <intrin.h>
#endif
#ifdef _MSC_VER
#define TARGET_F16C
#else
#define TARGET_F16C __attribute__((target("avx,f16c,fma")))
#endif
#endif
float (*HalfvecL2SquaredDistance) (int dim, half * ax, half * bx);
float (*HalfvecInnerProduct) (int dim, half * ax, half * bx);
double (*HalfvecCosineSimilarity) (int dim, half * ax, half * bx);
float (*HalfvecL1Distance) (int dim, half * ax, half * bx);
static float
HalfvecL2SquaredDistanceDefault(int dim, half * ax, half * bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
{
float diff = HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]);
distance += diff * diff;
}
return distance;
}
#ifdef HALFVEC_DISPATCH
TARGET_F16C static float
HalfvecL2SquaredDistanceF16c(int dim, half * ax, half * bx)
{
float distance;
int i;
float s[8];
int count = (dim / 8) * 8;
__m256 dist = _mm256_setzero_ps();
for (i = 0; i < count; i += 8)
{
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
__m256 axs = _mm256_cvtph_ps(axi);
__m256 bxs = _mm256_cvtph_ps(bxi);
__m256 diff = _mm256_sub_ps(axs, bxs);
dist = _mm256_fmadd_ps(diff, diff, dist);
}
_mm256_storeu_ps(s, dist);
distance = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
for (; i < dim; i++)
{
float diff = HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]);
distance += diff * diff;
}
return distance;
}
#endif
static float
HalfvecInnerProductDefault(int dim, half * ax, half * bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]);
return distance;
}
#ifdef HALFVEC_DISPATCH
TARGET_F16C static float
HalfvecInnerProductF16c(int dim, half * ax, half * bx)
{
float distance;
int i;
float s[8];
int count = (dim / 8) * 8;
__m256 dist = _mm256_setzero_ps();
for (i = 0; i < count; i += 8)
{
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
__m256 axs = _mm256_cvtph_ps(axi);
__m256 bxs = _mm256_cvtph_ps(bxi);
dist = _mm256_fmadd_ps(axs, bxs, dist);
}
_mm256_storeu_ps(s, dist);
distance = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
for (; i < dim; i++)
distance += HalfToFloat4(ax[i]) * HalfToFloat4(bx[i]);
return distance;
}
#endif
static double
HalfvecCosineSimilarityDefault(int dim, half * ax, half * bx)
{
float similarity = 0.0;
float norma = 0.0;
float normb = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
{
float axi = HalfToFloat4(ax[i]);
float bxi = HalfToFloat4(bx[i]);
similarity += axi * bxi;
norma += axi * axi;
normb += bxi * bxi;
}
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
return (double) similarity / sqrt((double) norma * (double) normb);
}
#ifdef HALFVEC_DISPATCH
TARGET_F16C static double
HalfvecCosineSimilarityF16c(int dim, half * ax, half * bx)
{
float similarity;
float norma;
float normb;
int i;
float s[8];
int count = (dim / 8) * 8;
__m256 sim = _mm256_setzero_ps();
__m256 na = _mm256_setzero_ps();
__m256 nb = _mm256_setzero_ps();
for (i = 0; i < count; i += 8)
{
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
__m256 axs = _mm256_cvtph_ps(axi);
__m256 bxs = _mm256_cvtph_ps(bxi);
sim = _mm256_fmadd_ps(axs, bxs, sim);
na = _mm256_fmadd_ps(axs, axs, na);
nb = _mm256_fmadd_ps(bxs, bxs, nb);
}
_mm256_storeu_ps(s, sim);
similarity = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
_mm256_storeu_ps(s, na);
norma = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
_mm256_storeu_ps(s, nb);
normb = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
/* Auto-vectorized */
for (; i < dim; i++)
{
float axi = HalfToFloat4(ax[i]);
float bxi = HalfToFloat4(bx[i]);
similarity += axi * bxi;
norma += axi * axi;
normb += bxi * bxi;
}
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
return (double) similarity / sqrt((double) norma * (double) normb);
}
#endif
static float
HalfvecL1DistanceDefault(int dim, half * ax, half * bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
distance += fabsf(HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]));
return distance;
}
#ifdef HALFVEC_DISPATCH
/* Does not require FMA, but keep logic simple */
TARGET_F16C static float
HalfvecL1DistanceF16c(int dim, half * ax, half * bx)
{
float distance;
int i;
float s[8];
int count = (dim / 8) * 8;
__m256 dist = _mm256_setzero_ps();
__m256 sign = _mm256_set1_ps(-0.0);
for (i = 0; i < count; i += 8)
{
__m128i axi = _mm_loadu_si128((__m128i *) (ax + i));
__m128i bxi = _mm_loadu_si128((__m128i *) (bx + i));
__m256 axs = _mm256_cvtph_ps(axi);
__m256 bxs = _mm256_cvtph_ps(bxi);
dist = _mm256_add_ps(dist, _mm256_andnot_ps(sign, _mm256_sub_ps(axs, bxs)));
}
_mm256_storeu_ps(s, dist);
distance = s[0] + s[1] + s[2] + s[3] + s[4] + s[5] + s[6] + s[7];
for (; i < dim; i++)
distance += fabsf(HalfToFloat4(ax[i]) - HalfToFloat4(bx[i]));
return distance;
}
#endif
#ifdef HALFVEC_DISPATCH
#define CPU_FEATURE_FMA (1 << 12)
#define CPU_FEATURE_OSXSAVE (1 << 27)
#define CPU_FEATURE_AVX (1 << 28)
#define CPU_FEATURE_F16C (1 << 29)
#ifdef _MSC_VER
#define TARGET_XSAVE
#else
#define TARGET_XSAVE __attribute__((target("xsave")))
#endif
TARGET_XSAVE static bool
SupportsCpuFeature(unsigned int feature)
{
unsigned int exx[4] = {0, 0, 0, 0};
#if defined(USE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#else
__cpuid(exx, 1);
#endif
/* Check OS supports XSAVE */
if ((exx[2] & CPU_FEATURE_OSXSAVE) != CPU_FEATURE_OSXSAVE)
return false;
/* Check XMM and YMM registers are enabled */
if ((_xgetbv(0) & 6) != 6)
return false;
/* Now check features */
return (exx[2] & feature) == feature;
}
#endif
void
HalfvecInit(void)
{
/*
* Could skip pointer when single function, but no difference in
* performance
*/
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceDefault;
HalfvecInnerProduct = HalfvecInnerProductDefault;
HalfvecCosineSimilarity = HalfvecCosineSimilarityDefault;
HalfvecL1Distance = HalfvecL1DistanceDefault;
#ifdef HALFVEC_DISPATCH
if (SupportsCpuFeature(CPU_FEATURE_AVX | CPU_FEATURE_F16C | CPU_FEATURE_FMA))
{
HalfvecL2SquaredDistance = HalfvecL2SquaredDistanceF16c;
HalfvecInnerProduct = HalfvecInnerProductF16c;
HalfvecCosineSimilarity = HalfvecCosineSimilarityF16c;
/* Does not require FMA, but keep logic simple */
HalfvecL1Distance = HalfvecL1DistanceF16c;
}
#endif
}

View File

@@ -1,263 +0,0 @@
#ifndef HALFUTILS_H
#define HALFUTILS_H
#include <math.h>
#include "common/shortest_dec.h"
#include "halfvec.h"
#ifdef F16C_SUPPORT
#include <immintrin.h>
#endif
extern float (*HalfvecL2SquaredDistance) (int dim, half * ax, half * bx);
extern float (*HalfvecInnerProduct) (int dim, half * ax, half * bx);
extern double (*HalfvecCosineSimilarity) (int dim, half * ax, half * bx);
extern float (*HalfvecL1Distance) (int dim, half * ax, half * bx);
void HalfvecInit(void);
/*
* Check if half is NaN
*/
static inline bool
HalfIsNan(half num)
{
#ifdef FLT16_SUPPORT
return isnan(num);
#else
return (num & 0x7C00) == 0x7C00 && (num & 0x7FFF) != 0x7C00;
#endif
}
/*
* Check if half is infinite
*/
static inline bool
HalfIsInf(half num)
{
#ifdef FLT16_SUPPORT
return isinf(num);
#else
return (num & 0x7FFF) == 0x7C00;
#endif
}
/*
* Check if half is zero
*/
static inline bool
HalfIsZero(half num)
{
#ifdef FLT16_SUPPORT
return num == 0;
#else
return (num & 0x7FFF) == 0x0000;
#endif
}
/*
* Convert a half to a float4
*/
static inline float
HalfToFloat4(half num)
{
#if defined(F16C_SUPPORT)
return _cvtsh_ss(num);
#elif defined(FLT16_SUPPORT)
return (float) num;
#else
union
{
float f;
uint32 i;
} swapfloat;
union
{
half h;
uint16 i;
} swaphalf;
uint16 bin;
uint32 exponent;
uint32 mantissa;
uint32 result;
swaphalf.h = num;
bin = swaphalf.i;
exponent = (bin & 0x7C00) >> 10;
mantissa = bin & 0x03FF;
/* Sign */
result = (bin & 0x8000) << 16;
if (unlikely(exponent == 31))
{
if (mantissa == 0)
{
/* Infinite */
result |= 0x7F800000;
}
else
{
/* NaN */
result |= 0x7FC00000;
}
}
else if (unlikely(exponent == 0))
{
/* Subnormal */
if (mantissa != 0)
{
exponent = -14;
for (int i = 0; i < 10; i++)
{
mantissa <<= 1;
exponent -= 1;
if ((mantissa >> 10) % 2 == 1)
{
mantissa &= 0x03ff;
break;
}
}
result |= (exponent + 127) << 23;
}
}
else
{
/* Normal */
result |= (exponent - 15 + 127) << 23;
}
result |= mantissa << 13;
swapfloat.i = result;
return swapfloat.f;
#endif
}
/*
* Convert a float4 to a half
*/
static inline half
Float4ToHalfUnchecked(float num)
{
#if defined(F16C_SUPPORT)
return _cvtss_sh(num, 0);
#elif defined(FLT16_SUPPORT)
return (_Float16) num;
#else
union
{
float f;
uint32 i;
} swapfloat;
union
{
half h;
uint16 i;
} swaphalf;
uint32 bin;
int exponent;
int mantissa;
uint16 result;
swapfloat.f = num;
bin = swapfloat.i;
exponent = (bin & 0x7F800000) >> 23;
mantissa = bin & 0x007FFFFF;
/* Sign */
result = (bin & 0x80000000) >> 16;
if (isinf(num))
{
/* Infinite */
result |= 0x7C00;
}
else if (isnan(num))
{
/* NaN */
result |= 0x7E00;
result |= mantissa >> 13;
}
else if (exponent > 98)
{
int m;
int gr;
int s;
exponent -= 127;
s = mantissa & 0x00000FFF;
/* Subnormal */
if (exponent < -14)
{
int diff = -exponent - 14;
mantissa >>= diff;
mantissa += 1 << (23 - diff);
s |= mantissa & 0x00000FFF;
}
m = mantissa >> 13;
/* Round */
gr = (mantissa >> 12) % 4;
if (gr == 3 || (gr == 1 && s != 0))
m += 1;
if (m == 1024)
{
m = 0;
exponent += 1;
}
if (exponent > 15)
{
/* Infinite */
result |= 0x7C00;
}
else
{
if (exponent >= -14)
result |= (exponent + 15) << 10;
result |= m;
}
}
swaphalf.i = result;
return swaphalf.h;
#endif
}
/*
* Convert a float4 to a half
*/
static inline half
Float4ToHalf(float num)
{
half result = Float4ToHalfUnchecked(num);
if (unlikely(HalfIsInf(result)) && !isinf(num))
{
char *buf = palloc(FLOAT_SHORTEST_DECIMAL_LEN);
float_to_shortest_decimal_buf(num, buf);
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type halfvec", buf)));
}
return result;
}
#endif

File diff suppressed because it is too large Load Diff

View File

@@ -1,70 +0,0 @@
#ifndef HALFVEC_H
#define HALFVEC_H
#define __STDC_WANT_IEC_60559_TYPES_EXT__
#include <float.h>
/* We use two types of dispatching: intrinsics and target_clones */
/* TODO Move to better place */
#ifndef DISABLE_DISPATCH
/* Only enable for more recent compilers to keep build process simple */
#if defined(__x86_64__) && defined(__GNUC__) && __GNUC__ >= 9
#define USE_DISPATCH
#elif defined(__x86_64__) && defined(__clang_major__) && __clang_major__ >= 7
#define USE_DISPATCH
#elif defined(_M_AMD64) && defined(_MSC_VER) && _MSC_VER >= 1920
#define USE_DISPATCH
#endif
#endif
/* target_clones requires glibc */
#if defined(USE_DISPATCH) && defined(__gnu_linux__) && defined(__has_attribute)
/* Use separate line for portability */
#if __has_attribute(target_clones)
#define USE_TARGET_CLONES
#endif
#endif
/* Apple clang check needed for universal binaries on Mac */
#if defined(USE_DISPATCH) && (defined(HAVE__GET_CPUID) || defined(__apple_build_version__))
#define USE__GET_CPUID
#endif
#if defined(USE_DISPATCH)
#define HALFVEC_DISPATCH
#endif
/* F16C has better performance than _Float16 (on x86-64) */
#if defined(__F16C__)
#define F16C_SUPPORT
#elif defined(__FLT16_MAX__) && !defined(HALFVEC_DISPATCH) && !defined(__FreeBSD__) && (!defined(__i386__) || defined(__SSE2__))
#define FLT16_SUPPORT
#endif
#ifdef FLT16_SUPPORT
#define half _Float16
#define HALF_MAX FLT16_MAX
#else
#define half uint16
#define HALF_MAX 65504
#endif
#define HALFVEC_MAX_DIM 16000
#define HALFVEC_SIZE(_dim) (offsetof(HalfVector, x) + sizeof(half)*(_dim))
#define DatumGetHalfVector(x) ((HalfVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_HALFVEC_P(x) DatumGetHalfVector(PG_GETARG_DATUM(x))
#define PG_RETURN_HALFVEC_P(x) PG_RETURN_POINTER(x)
typedef struct HalfVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused; /* reserved for future use, always zero */
half x[FLEXIBLE_ARRAY_MEMBER];
} HalfVector;
HalfVector *InitHalfVector(int dim);
#endif

View File

@@ -1,41 +1,22 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "storage/lwlock.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
#endif
static const struct config_enum_entry hnsw_iterative_scan_options[] = {
{"off", HNSW_ITERATIVE_SCAN_OFF, false},
{"relaxed_order", HNSW_ITERATIVE_SCAN_RELAXED, false},
{"strict_order", HNSW_ITERATIVE_SCAN_STRICT, false},
{NULL, 0, false}
};
int hnsw_ef_search;
int hnsw_iterative_scan;
int hnsw_max_scan_tuples;
double hnsw_scan_mem_multiplier;
int hnsw_lock_tranche_id;
static relopt_kind hnsw_relopt_kind;
@@ -59,20 +40,12 @@ HnswInitLockTranche(void)
sizeof(int) * 1,
&found);
if (!found)
{
#if PG_VERSION_NUM >= 190000
tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
#else
tranche_ids[0] = LWLockNewTrancheId();
#endif
}
hnsw_lock_tranche_id = tranche_ids[0];
LWLockRelease(AddinShmemInitLock);
#if PG_VERSION_NUM < 190000
/* Per-backend registration of the tranche ID */
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
#endif
}
/*
@@ -86,28 +59,22 @@ HnswInit(void)
hnsw_relopt_kind = add_reloption_kind();
add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",
HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M, AccessExclusiveLock);
HNSW_DEFAULT_M, HNSW_MIN_M, HNSW_MAX_M
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
add_int_reloption(hnsw_relopt_kind, "ef_construction", "Size of the dynamic candidate list for construction",
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION, AccessExclusiveLock);
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("hnsw.ef_search", "Sets the size of the dynamic candidate list for search",
"Valid range is 1..1000.", &hnsw_ef_search,
HNSW_DEFAULT_EF_SEARCH, HNSW_MIN_EF_SEARCH, HNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
DefineCustomEnumVariable("hnsw.iterative_scan", "Sets the mode for iterative scans",
NULL, &hnsw_iterative_scan,
HNSW_ITERATIVE_SCAN_OFF, hnsw_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
/* This is approximate and does not affect the initial scan */
DefineCustomIntVariable("hnsw.max_scan_tuples", "Sets the max number of tuples to visit for iterative scans",
NULL, &hnsw_max_scan_tuples,
20000, 1, INT_MAX, PGC_USERSET, 0, NULL, NULL, NULL);
/* Same range as hash_mem_multiplier */
DefineCustomRealVariable("hnsw.scan_mem_multiplier", "Sets the multiple of work_mem to use for iterative scans",
NULL, &hnsw_scan_mem_multiplier,
1, 1, 1000, PGC_USERSET, 0, NULL, NULL, NULL);
MarkGUCPrefixReserved("hnsw");
}
@@ -139,93 +106,37 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
{
GenericCosts costs;
int m;
double ratio;
double startupPages;
double spc_seq_page_cost;
int entryLevel;
Relation index;
/* Never use index without order */
if (path->indexorderbys == NIL)
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
*indexStartupCost = DBL_MAX;
*indexTotalCost = DBL_MAX;
*indexSelectivity = 0;
*indexCorrelation = 0;
*indexPages = 0;
#if PG_VERSION_NUM >= 180000
/* See "On disable_cost" thread on pgsql-hackers */
path->path.disabled_nodes = 2;
#endif
return;
}
MemSet(&costs, 0, sizeof(costs));
genericcostestimate(root, path, loop_count, &costs);
index = index_open(path->indexinfo->indexoid, NoLock);
HnswGetMetaPageInfo(index, &m, NULL);
index_close(index, NoLock);
/*
* HNSW cost estimation follows a formula that accounts for the total
* number of tuples indexed combined with the parameters that most
* influence the duration of the index scan, namely: m - the number of
* tuples that are scanned in each step of the HNSW graph traversal
* ef_search - which influences the total number of steps taken at layer 0
*
* The source of the vector data can impact how many steps it takes to
* converge on the set of vectors to return to the executor. Currently, we
* use a hardcoded scaling factor (HNSWScanScalingFactor) to help
* influence that, but this could later become a configurable parameter
* based on the cost estimations.
*
* The tuple estimator formula is below:
*
* numIndexTuples = entryLevel * m + layer0TuplesMax * layer0Selectivity
*
* "entryLevel * m" represents the floor of tuples we need to scan to get
* to layer 0 (L0).
*
* "layer0TuplesMax" is the estimated total number of tuples we'd scan at
* L0 if we weren't discarding already visited tuples as part of the scan.
*
* "layer0Selectivity" estimates the percentage of tuples that are scanned
* at L0, accounting for previously visited tuples, multiplied by the
* "scalingFactor" (currently hardcoded).
*/
if (path->indexinfo->tuples > 0)
{
double scalingFactor = 0.55;
int entryLevel = (int) (log(path->indexinfo->tuples) * HnswGetMl(m));
int layer0TuplesMax = HnswGetLayerM(m, 0) * hnsw_ef_search;
double layer0Selectivity = scalingFactor * log(path->indexinfo->tuples) / (log(m) * (1 + log(hnsw_ef_search)));
/* Approximate entry level */
entryLevel = (int) -log(1.0 / path->indexinfo->tuples) * HnswGetMl(m);
ratio = (entryLevel * m + layer0TuplesMax * layer0Selectivity) / path->indexinfo->tuples;
/* TODO Improve estimate of visited tuples (currently underestimates) */
/* Account for number of tuples (or entry level), m, and ef_search */
costs.numIndexTuples = (entryLevel + 2) * m;
if (ratio > 1)
ratio = 1;
}
else
ratio = 1;
genericcostestimate(root, path, loop_count, &costs);
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
/* Startup cost is cost before returning the first row */
costs.indexStartupCost = costs.indexTotalCost * ratio;
/* Adjust cost if needed since TOAST not included in seq scan cost */
startupPages = costs.numIndexPages * ratio;
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
{
/* Change all page cost from random to sequential */
costs.indexStartupCost -= startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
*indexStartupCost = costs.indexStartupCost;
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost;
*indexSelectivity = costs.indexSelectivity;
*indexCorrelation = costs.indexCorrelation;
@@ -243,10 +154,23 @@ hnswoptions(Datum reloptions, bool validate)
{"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)},
};
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate,
hnsw_relopt_kind,
sizeof(HnswOptions),
tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
HnswOptions *rdopts;
options = parseRelOptions(reloptions, validate, hnsw_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(HnswOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(HnswOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
}
/*
@@ -263,80 +187,19 @@ hnswvalidate(Oid opclassoid)
*
* See https://www.postgresql.org/docs/current/index-api.html
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
PGDLLEXPORT PG_FUNCTION_INFO_V1(hnswhandler);
Datum
hnswhandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 3,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = hnswbuild,
.ambuildempty = hnswbuildempty,
.aminsert = hnswinsert,
.aminsertcleanup = NULL,
.ambulkdelete = hnswbulkdelete,
.amvacuumcleanup = hnswvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = hnswcostestimate,
.amgettreeheight = NULL,
.amoptions = hnswoptions,
.amproperty = NULL,
.ambuildphasename = hnswbuildphasename,
.amvalidate = hnswvalidate,
.amadjustmembers = NULL,
.ambeginscan = hnswbeginscan,
.amrescan = hnswrescan,
.amgettuple = hnswgettuple,
.amgetbitmap = NULL,
.amendscan = hnswendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 3;
amroutine->amsupport = 2;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -347,31 +210,21 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false;
amroutine->ampredlocks = false;
amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid;
/* Interface functions */
amroutine->ambuild = hnswbuild;
amroutine->ambuildempty = hnswbuildempty;
amroutine->aminsert = hnswinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = hnswbulkdelete;
amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL;
amroutine->amcostestimate = hnswcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = hnswoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = hnswbuildphasename;
@@ -392,11 +245,5 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -3,36 +3,24 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/bufpage.h"
#include "storage/condition_variable.h"
#include "storage/lwlock.h"
#include "storage/s_lock.h"
#include "utils/relptr.h"
#include "utils/sampling.h"
#include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#if PG_VERSION_NUM < 120000
#error "Requires PostgreSQL 12+"
#endif
#define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000
/* Support functions */
#define HNSW_DISTANCE_PROC 1
#define HNSW_NORM_PROC 2
#define HNSW_TYPE_INFO_PROC 3
#define HNSW_VERSION 1
#define HNSW_MAGIC_NUMBER 0xA953A953
@@ -67,6 +55,12 @@ typedef Pointer Item;
#define HNSW_UPDATE_ENTRY_GREATER 1
#define HNSW_UPDATE_ENTRY_ALWAYS 2
typedef enum HnswType
{
HNSW_TYPE_VECTOR,
HNSW_TYPE_INTVEC
} HnswType;
/* Build phases */
/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 */
#define PROGRESS_HNSW_PHASE_LOAD 2
@@ -82,21 +76,6 @@ typedef Pointer Item;
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -105,6 +84,11 @@ typedef Pointer Item;
#define SeedRandom(seed) srandom(seed)
#endif
#if PG_VERSION_NUM < 130000
#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
#endif
#define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
#define HnswIsNeighborTuple(tup) ((tup)->type == HNSW_NEIGHBOR_TUPLE_TYPE)
@@ -117,9 +101,6 @@ typedef Pointer Item;
/* Ensure fits on page and in uint8 */
#define HnswGetMaxLevel(m) Min(((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - offsetof(HnswNeighborTupleData, indextids) - sizeof(ItemIdData)) / (sizeof(ItemPointerData)) / (m)) - 2, 255)
#define HnswGetSearchCandidate(membername, ptr) pairingheap_container(HnswSearchCandidate, membername, ptr)
#define HnswGetSearchCandidateConst(membername, ptr) pairingheap_const_container(HnswSearchCandidate, membername, ptr)
#define HnswGetValue(base, element) PointerGetDatum(HnswPtrAccess(base, (element)->value))
#if PG_VERSION_NUM < 140005
@@ -138,24 +119,14 @@ typedef Pointer Item;
/* Variables */
extern int hnsw_ef_search;
extern int hnsw_iterative_scan;
extern int hnsw_max_scan_tuples;
extern double hnsw_scan_mem_multiplier;
extern int hnsw_lock_tranche_id;
typedef enum HnswIterativeScanMode
{
HNSW_ITERATIVE_SCAN_OFF,
HNSW_ITERATIVE_SCAN_RELAXED,
HNSW_ITERATIVE_SCAN_STRICT
} HnswIterativeScanMode;
typedef struct HnswElementData HnswElementData;
typedef struct HnswNeighborArray HnswNeighborArray;
#define HnswPtrDeclare(type, relptrtype, ptrtype) \
relptr_declare(type, relptrtype); \
typedef union { type *ptr; relptrtype relptr; } ptrtype
typedef union { type *ptr; relptrtype relptr; } ptrtype;
/* Pointers that can be absolute or relative */
/* Use char for DatumPtr so works with Pointer */
@@ -171,7 +142,6 @@ struct HnswElementData
uint8 heaptidsLength;
uint8 level;
uint8 deleted;
uint8 version;
uint32 hash;
HnswNeighborsPtr neighbors;
BlockNumber blkno;
@@ -198,13 +168,11 @@ struct HnswNeighborArray
HnswCandidate items[FLEXIBLE_ARRAY_MEMBER];
};
typedef struct HnswSearchCandidate
typedef struct HnswPairingHeapNode
{
pairingheap_node c_node;
pairingheap_node w_node;
HnswElementPtr element;
double distance;
} HnswSearchCandidate;
pairingheap_node ph_node;
HnswCandidate *inner;
} HnswPairingHeapNode;
/* HNSW index options */
typedef struct HnswOptions
@@ -228,8 +196,8 @@ typedef struct HnswGraph
/* Allocations state */
LWLock allocatorLock;
Size memoryUsed;
Size memoryTotal;
long memoryUsed;
long memoryTotal;
/* Flushed state */
LWLock flushLock;
@@ -273,25 +241,6 @@ typedef struct HnswAllocator
void *state;
} HnswAllocator;
typedef struct HnswTypeInfo
{
int maxDimensions;
Datum (*normalize) (PG_FUNCTION_ARGS);
void (*checkValue) (Pointer v);
} HnswTypeInfo;
typedef struct HnswSupport
{
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
} HnswSupport;
typedef struct HnswQuery
{
Datum value;
} HnswQuery;
typedef struct HnswBuildState
{
/* Info */
@@ -299,7 +248,7 @@ typedef struct HnswBuildState
Relation index;
IndexInfo *indexInfo;
ForkNumber forkNum;
const HnswTypeInfo *typeInfo;
HnswType type;
/* Settings */
int dimensions;
@@ -311,7 +260,9 @@ typedef struct HnswBuildState
double reltuples;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
/* Variables */
HnswGraph graphData;
@@ -359,10 +310,10 @@ typedef struct HnswElementTupleData
uint8 type;
uint8 level;
uint8 deleted;
uint8 version;
uint8 unused;
ItemPointerData heaptids[HNSW_HEAPTIDS];
ItemPointerData neighbortid;
uint16 unused;
uint16 unused2;
Vector data;
} HnswElementTupleData;
@@ -371,42 +322,23 @@ typedef HnswElementTupleData * HnswElementTuple;
typedef struct HnswNeighborTupleData
{
uint8 type;
uint8 version;
uint8 unused;
uint16 count;
ItemPointerData indextids[FLEXIBLE_ARRAY_MEMBER];
} HnswNeighborTupleData;
typedef HnswNeighborTupleData * HnswNeighborTuple;
typedef union
{
struct pointerhash_hash *pointers;
struct offsethash_hash *offsets;
struct tidhash_hash *tids;
} visited_hash;
typedef union
{
HnswElement element;
ItemPointerData indextid;
} HnswUnvisited;
typedef struct HnswScanOpaqueData
{
const HnswTypeInfo *typeInfo;
bool first;
List *w;
visited_hash v;
pairingheap *discarded;
HnswQuery q;
int m;
int64 tuples;
double previousDistance;
Size maxMemory;
MemoryContext tmpCtx;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
} HnswScanOpaqueData;
typedef HnswScanOpaqueData * HnswScanOpaque;
@@ -424,14 +356,14 @@ typedef struct HnswVacuumState
int efConstruction;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
Oid collation;
/* Variables */
struct tidhash_hash *deleting;
struct tidhash_hash *deleted;
BufferAccessStrategy bas;
HnswNeighborTuple ntup;
HnswElementData highestPoint;
HnswElementData fallbackPoint;
/* Memory */
MemoryContext tmpCtx;
@@ -441,35 +373,31 @@ typedef struct HnswVacuumState
int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
void HnswInitSupport(HnswSupport * support, Relation index);
Datum HnswNormValue(const HnswTypeInfo * typeInfo, Oid collation, Datum value);
bool HnswCheckNorm(HnswSupport * support, Datum value);
HnswType HnswGetType(Relation index);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, HnswType type);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);
void HnswInit(void);
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement, visited_hash * v, pairingheap **discarded, bool initVisited, int64 *tuples);
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
HnswNeighborArray *HnswInitNeighborArray(int lm, HnswAllocator * allocator);
void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc);
bool HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPointer heaptid, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building);
bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
void HnswLoadElement(HnswElement element, double *distance, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec, double *maxDistance);
bool HnswFormIndexValue(Datum *out, Datum *values, bool *isnull, const HnswTypeInfo * typeInfo, HnswSupport * support);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
void HnswUpdateConnection(char *base, HnswNeighborArray * neighbors, HnswElement newElement, float distance, int lm, int *updateIdx, Relation index, HnswSupport * support);
bool HnswLoadNeighborTids(HnswElement element, ItemPointerData *indextids, Relation index, int m, int lm, int lc);
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
void HnswInitLockTranche(void);
const HnswTypeInfo *HnswGetTypeInfo(Relation index);
PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -36,34 +36,22 @@
*/
#include "postgres.h"
#include <limits.h>
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -71,6 +59,12 @@
#include "pgstat.h"
#endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h"
#include "utils/wait_event.h"
@@ -80,7 +74,9 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
#if PG_VERSION_NUM < 130000
#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T + SIZEOF_VOID_P * 2)
#endif
/*
* Create the metapage
@@ -195,9 +191,7 @@ CreateGraphPages(HnswBuildState * buildstate)
/* Initial size check */
if (etupSize > HNSW_TUPLE_ALLOC_SIZE)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("index tuple too large")));
elog(ERROR, "index tuple too large");
HnswSetElementTuple(base, etup, element);
@@ -379,18 +373,12 @@ AddElementInMemory(char *base, HnswGraph * graph, HnswElement element)
* Update neighbors
*/
static void
UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswElement e, int m)
{
for (int lc = e->level; lc >= 0; lc--)
{
int lm = HnswGetLayerM(m, lc);
Size neighborsSize = HNSW_NEIGHBOR_ARRAY_SIZE(lm);
HnswNeighborArray *neighbors = palloc(neighborsSize);
/* Copy neighbors to local memory */
LWLockAcquire(&e->lock, LW_SHARED);
memcpy(neighbors, HnswGetNeighbors(base, e, lc), neighborsSize);
LWLockRelease(&e->lock);
HnswNeighborArray *neighbors = HnswGetNeighbors(base, e, lc);
for (int i = 0; i < neighbors->length; i++)
{
@@ -400,8 +388,9 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
/* Keep scan-build happy on Mac x86-64 */
Assert(neighborElement);
/* Use element for lock instead of hc since hc can be replaced */
LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
HnswUpdateConnection(base, HnswGetNeighbors(base, neighborElement, lc), e, hc->distance, lm, NULL, NULL, support);
HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
LWLockRelease(&neighborElement->lock);
}
}
@@ -411,7 +400,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
* Update graph in memory
*/
static void
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
{
HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea;
@@ -424,7 +413,7 @@ UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswEleme
AddElementInMemory(base, graph, element);
/* Update neighbors */
UpdateNeighborsInMemory(base, support, element, m);
UpdateNeighborsInMemory(base, procinfo, collation, element, m);
/* Update entry point if needed (already have lock) */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -437,8 +426,9 @@ UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswEleme
static void
InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
{
FmgrInfo *procinfo = buildstate->procinfo;
Oid collation = buildstate->collation;
HnswGraph *graph = buildstate->graph;
HnswSupport *support = &buildstate->support;
HnswElement entryPoint;
LWLock *entryLock = &graph->entryLock;
LWLock *entryWaitLock = &graph->entryWaitLock;
@@ -470,10 +460,10 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
HnswFindElementNeighbors(base, element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
/* Update graph in memory */
UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate);
/* Release entry lock */
LWLockRelease(entryLock);
@@ -488,24 +478,24 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
HnswGraph *graph = buildstate->graph;
HnswElement element;
HnswAllocator *allocator = &buildstate->allocator;
HnswSupport *support = &buildstate->support;
Size valueSize;
Pointer valuePtr;
LWLock *flushLock = &graph->flushLock;
char *base = buildstate->hnswarea;
Datum value;
Size memoryMargin;
/* Form index value */
if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
return false;
/* Detoast once for all calls */
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Normalize if needed */
if (buildstate->normprocinfo != NULL)
{
if (!HnswNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->type))
return false;
}
/* Get datum size */
valueSize = VARSIZE_ANY(DatumGetPointer(value));
/* In a parallel build, add a margin so allocations never fail */
memoryMargin = base == NULL ? 0 : 1024 * 1024;
/* Ensure graph not flushed when inserting */
LWLockAcquire(flushLock, LW_SHARED);
@@ -514,7 +504,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
{
LWLockRelease(flushLock);
return HnswInsertTupleOnDisk(index, support, value, heaptid, true);
return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, true);
}
/*
@@ -527,7 +517,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Check that we have enough memory available for the new element now that
* we have the allocator lock, and flush pages if needed.
*/
if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
if (graph->memoryUsed >= graph->memoryTotal)
{
LWLockRelease(&graph->allocatorLock);
@@ -546,7 +536,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
LWLockRelease(flushLock);
return HnswInsertTupleOnDisk(index, support, value, heaptid, true);
return HnswInsertTupleOnDisk(index, value, values, isnull, heaptid, true);
}
/* Ok, we can proceed to allocate the element */
@@ -562,7 +552,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Copy the datum */
memcpy(valuePtr, DatumGetPointer(value), valueSize);
HnswPtrStore(base, element->value, (char *) valuePtr);
HnswPtrStore(base, element->value, valuePtr);
/* Create a lock for the element */
LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -580,13 +570,17 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Callback for table_index_build_scan
*/
static void
BuildCallback(Relation index, ItemPointer tid, Datum *values,
BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
HnswGraph *graph = buildstate->graph;
MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */
if (isnull[0])
return;
@@ -612,7 +606,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
* Initialize the graph
*/
static void
InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
InitGraph(HnswGraph * graph, char *base, long memoryTotal)
{
/* Initialize the lock tranche if needed */
HnswInitLockTranche();
@@ -620,7 +614,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
graph->memoryTotal = memoryTotal;
graph->flushed = false;
graph->indtuples = 0;
SpinLockInit(&graph->lock);
@@ -649,7 +643,11 @@ HnswMemoryContextAlloc(Size size, void *state)
HnswBuildState *buildstate = (HnswBuildState *) state;
void *chunk = MemoryContextAlloc(buildstate->graphCtx, size);
#if PG_VERSION_NUM >= 130000
buildstate->graphData.memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
#else
buildstate->graphData.memoryUsed += MAXALIGN(size);
#endif
return chunk;
}
@@ -661,17 +659,9 @@ static void *
HnswSharedMemoryAlloc(Size size, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
Size alignedSize = MAXALIGN(size);
void *chunk;
void *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
if (alignedSize > 1024 * 1024)
elog(ERROR, "hnsw allocation too large");
if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
elog(ERROR, "hnsw allocator out of memory");
chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
buildstate->graph->memoryUsed += alignedSize;
buildstate->graph->memoryUsed += MAXALIGN(size);
return chunk;
}
@@ -681,45 +671,40 @@ HnswSharedMemoryAlloc(Size size, void *state)
static void
InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, IndexInfo *indexInfo, ForkNumber forkNum)
{
int maxDimensions = HNSW_MAX_DIM;
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->forkNum = forkNum;
buildstate->typeInfo = HnswGetTypeInfo(index);
buildstate->type = HnswGetType(index);
buildstate->m = HnswGetM(index);
buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for hnsw index")));
if (buildstate->type == HNSW_TYPE_INTVEC)
maxDimensions *= 4;
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for hnsw index", buildstate->typeInfo->maxDimensions)));
if (buildstate->dimensions > maxDimensions)
elog(ERROR, "column cannot have more than %d dimensions for hnsw index", maxDimensions);
if (buildstate->efConstruction < 2 * buildstate->m)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("ef_construction must be greater than or equal to 2 * m")));
elog(ERROR, "ef_construction must be greater than or equal to 2 * m");
buildstate->reltuples = 0;
buildstate->indtuples = 0;
/* Get support functions */
HnswInitSupport(&buildstate->support, index);
buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
buildstate->collation = index->rd_indcollation[0];
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024);
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * 1024L);
buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -803,11 +788,7 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
buildstate.hnswarea = hnswarea;
InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
scan = table_beginscan_parallel(heapRel,
ParallelTableScanFromHnswShared(hnswshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
ParallelTableScanFromHnswShared(hnswshared));
reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -956,13 +937,11 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Leave space for other objects in shared memory */
/* Docker has a default limit of 64 MB for shm_size */
/* which happens to be the default value of maintenance_work_mem */
esthnswarea = maintenance_work_mem * (Size) 1024;
esthnswarea = maintenance_work_mem * 1024L;
estother = 3 * 1024 * 1024;
if (esthnswarea > estother)
esthnswarea -= estother;
esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
shm_toc_estimate_keys(&pcxt->estimator, 2);
@@ -1005,7 +984,8 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
snapshot);
hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
/*
* Avoid base address for relptr for Postgres < 14.5
@@ -1084,7 +1064,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
* Build graph
*/
static void
BuildGraph(HnswBuildState * buildstate)
BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
{
int parallel_workers = 0;
@@ -1132,10 +1112,10 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
InitBuildState(buildstate, heap, index, indexInfo, forkNum);
BuildGraph(buildstate);
BuildGraph(buildstate, forkNum);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
if (RelationNeedsWAL(index))
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocks(index), true);
FreeBuildState(buildstate);
}

View File

@@ -1,19 +1,13 @@
#include "postgres.h"
#include "access/genam.h"
#include <math.h>
#include "access/generic_xlog.h"
#include "hnsw.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "storage/lwlock.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Get the insert page
@@ -42,15 +36,14 @@ GetInsertPage(Relation index)
* Check for a free offset
*/
static bool
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size etupSize, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage, uint8 *tupleVersion)
HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size ntupSize, Buffer *nbuf, Page *npage, OffsetNumber *freeOffno, OffsetNumber *freeNeighborOffno, BlockNumber *newInsertPage)
{
OffsetNumber offno;
OffsetNumber maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
ItemId eitemid = PageGetItemId(page, offno);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, eitemid);
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
@@ -61,9 +54,7 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
BlockNumber elementPage = BufferGetBlockNumber(buf);
BlockNumber neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
OffsetNumber neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
ItemId nitemid;
Size pageFree;
Size npageFree;
ItemId itemid;
if (!BlockNumberIsValid(*newInsertPage))
*newInsertPage = elementPage;
@@ -82,29 +73,13 @@ HnswFreeOffset(Relation index, Buffer buf, Page page, HnswElement element, Size
*npage = BufferGetPage(*nbuf);
}
nitemid = PageGetItemId(*npage, neighborOffno);
itemid = PageGetItemId(*npage, neighborOffno);
/* Ensure aligned for space check */
Assert(etupSize == MAXALIGN(etupSize));
Assert(ntupSize == MAXALIGN(ntupSize));
/*
* Calculate free space individually since tuples are overwritten
* individually (in separate calls to PageIndexTupleOverwrite)
*/
pageFree = ItemIdGetLength(eitemid) + PageGetExactFreeSpace(page);
npageFree = ItemIdGetLength(nitemid);
if (neighborPage != elementPage)
npageFree += PageGetExactFreeSpace(*npage);
else if (pageFree >= etupSize)
npageFree += pageFree - etupSize;
/* Check for space */
if (pageFree >= etupSize && npageFree >= ntupSize)
/* Check for space on neighbor tuple page */
if (PageGetFreeSpace(*npage) + ItemIdGetLength(itemid) - sizeof(ItemIdData) >= ntupSize)
{
*freeOffno = offno;
*freeNeighborOffno = neighborOffno;
*tupleVersion = etup->version;
return true;
}
else if (*nbuf != buf)
@@ -160,7 +135,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
OffsetNumber freeOffno = InvalidOffsetNumber;
OffsetNumber freeNeighborOffno = InvalidOffsetNumber;
BlockNumber newInsertPage = InvalidBlockNumber;
uint8 tupleVersion;
char *base = NULL;
/* Calculate sizes */
@@ -210,7 +184,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
}
/* Next, try space from a deleted element */
if (HnswFreeOffset(index, buf, page, e, etupSize, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage, &tupleVersion))
if (HnswFreeOffset(index, buf, page, e, ntupSize, &nbuf, &npage, &freeOffno, &freeNeighborOffno, &newInsertPage))
{
if (nbuf != buf)
{
@@ -220,10 +194,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
npage = GenericXLogRegisterBuffer(state, nbuf, 0);
}
/* Set tuple version */
etup->version = tupleVersion;
ntup->version = tupleVersion;
break;
}
@@ -346,107 +316,6 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
*updatedInsertPage = newInsertPage;
}
/*
* Load neighbors
*/
static HnswNeighborArray *
HnswLoadNeighbors(HnswElement element, Relation index, int m, int lm, int lc)
{
char *base = NULL;
HnswNeighborArray *neighbors = HnswInitNeighborArray(lm, NULL);
ItemPointerData indextids[HNSW_MAX_M * 2];
if (!HnswLoadNeighborTids(element, indextids, index, m, lm, lc))
return neighbors;
for (int i = 0; i < lm; i++)
{
ItemPointer indextid = &indextids[i];
HnswElement e;
HnswCandidate *hc;
if (!ItemPointerIsValid(indextid))
break;
e = HnswInitElementFromBlock(ItemPointerGetBlockNumber(indextid), ItemPointerGetOffsetNumber(indextid));
hc = &neighbors->items[neighbors->length++];
HnswPtrStore(base, hc->element, e);
}
return neighbors;
}
/*
* Load elements for insert
*/
static void
LoadElementsForInsert(HnswNeighborArray * neighbors, HnswQuery * q, int *idx, Relation index, HnswSupport * support)
{
char *base = NULL;
for (int i = 0; i < neighbors->length; i++)
{
HnswCandidate *hc = &neighbors->items[i];
HnswElement element = HnswPtrAccess(base, hc->element);
double distance;
HnswLoadElement(element, &distance, q, index, support, true, NULL);
hc->distance = distance;
/* Prune element if being deleted */
if (element->heaptidsLength == 0)
{
*idx = i;
break;
}
}
}
/*
* Get update index
*/
static int
GetUpdateIndex(HnswElement element, HnswElement newElement, float distance, int m, int lm, int lc, Relation index, HnswSupport * support, MemoryContext updateCtx)
{
char *base = NULL;
int idx = -1;
HnswNeighborArray *neighbors;
MemoryContext oldCtx = MemoryContextSwitchTo(updateCtx);
/*
* Get latest neighbors since they may have changed. Do not lock yet since
* selecting neighbors can take time. Could use optimistic locking to
* retry if another update occurs before getting exclusive lock.
*/
neighbors = HnswLoadNeighbors(element, index, m, lm, lc);
/*
* Could improve performance for vacuuming by checking neighbors against
* list of elements being deleted to find index. It's important to exclude
* already deleted elements for this since they can be replaced at any
* time.
*/
if (neighbors->length < lm)
idx = -2;
else
{
HnswQuery q;
q.value = HnswGetValue(base, element);
LoadElementsForInsert(neighbors, &q, &idx, index, support);
if (idx == -1)
HnswUpdateConnection(base, neighbors, newElement, distance, lm, &idx, index, support);
}
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(updateCtx);
return idx;
}
/*
* Check if connection already exists
*/
@@ -467,94 +336,14 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
return false;
}
/*
* Update neighbor
*/
static void
UpdateNeighborOnDisk(HnswElement element, HnswElement newElement, int idx, int m, int lm, int lc, Relation index, bool checkExisting, bool building)
{
Buffer buf;
Page page;
GenericXLogState *state;
HnswNeighborTuple ntup;
int startIdx;
OffsetNumber offno = element->neighborOffno;
/* Register page */
buf = ReadBuffer(index, element->neighborPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
if (building)
{
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Get tuple */
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Calculate index for update */
startIdx = (element->level - lc) * m;
/* Check for existing connection */
if (checkExisting && ConnectionExists(newElement, ntup, startIdx, lm))
idx = -1;
else if (idx == -2)
{
/* Find free offset if still exists */
/* TODO Retry updating connections if not */
for (int j = 0; j < lm; j++)
{
if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
{
idx = startIdx + j;
break;
}
}
}
else
idx += startIdx;
/* Make robust to issues */
if (idx >= 0 && idx < ntup->count)
{
ItemPointer indextid = &ntup->indextids[idx];
/* Update neighbor on the buffer */
ItemPointerSet(indextid, newElement->blkno, newElement->offno);
/* Commit */
if (building)
MarkBufferDirty(buf);
else
GenericXLogFinish(state);
}
else if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf);
}
/*
* Update neighbors
*/
void
HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building)
HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
{
char *base = NULL;
/* Use separate memory context to improve performance for larger vectors */
MemoryContext updateCtx = GenerationContextCreate(CurrentMemoryContext,
"Hnsw insert update context",
#if PG_VERSION_NUM >= 150000
128 * 1024, 128 * 1024,
#endif
128 * 1024);
for (int lc = e->level; lc >= 0; lc--)
{
int lm = HnswGetLayerM(m, lc);
@@ -563,20 +352,92 @@ HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e,
for (int i = 0; i < neighbors->length; i++)
{
HnswCandidate *hc = &neighbors->items[i];
Buffer buf;
Page page;
GenericXLogState *state;
HnswNeighborTuple ntup;
int idx = -1;
int startIdx;
HnswElement neighborElement = HnswPtrAccess(base, hc->element);
int idx;
OffsetNumber offno = neighborElement->neighborOffno;
idx = GetUpdateIndex(neighborElement, e, hc->distance, m, lm, lc, index, support, updateCtx);
/* Get latest neighbors since they may have changed */
/* Do not lock yet since selecting neighbors can take time */
HnswLoadNeighbors(neighborElement, index, m);
/*
* Could improve performance for vacuuming by checking neighbors
* against list of elements being deleted to find index. It's
* important to exclude already deleted elements for this since
* they can be replaced at any time.
*/
/* Select neighbors */
HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, procinfo, collation);
/* New element was not selected as a neighbor */
if (idx == -1)
continue;
UpdateNeighborOnDisk(neighborElement, e, idx, m, lm, lc, index, checkExisting, building);
/* Register page */
buf = ReadBuffer(index, neighborElement->neighborPage);
LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
if (building)
{
state = NULL;
page = BufferGetPage(buf);
}
else
{
state = GenericXLogStart(index);
page = GenericXLogRegisterBuffer(state, buf, 0);
}
/* Get tuple */
ntup = (HnswNeighborTuple) PageGetItem(page, PageGetItemId(page, offno));
/* Calculate index for update */
startIdx = (neighborElement->level - lc) * m;
/* Check for existing connection */
if (checkExisting && ConnectionExists(e, ntup, startIdx, lm))
idx = -1;
else if (idx == -2)
{
/* Find free offset if still exists */
/* TODO Retry updating connections if not */
for (int j = 0; j < lm; j++)
{
if (!ItemPointerIsValid(&ntup->indextids[startIdx + j]))
{
idx = startIdx + j;
break;
}
}
}
else
idx += startIdx;
/* Make robust to issues */
if (idx >= 0 && idx < ntup->count)
{
ItemPointer indextid = &ntup->indextids[idx];
/* Update neighbor on the buffer */
ItemPointerSet(indextid, e->blkno, e->offno);
/* Commit */
if (building)
MarkBufferDirty(buf);
else
GenericXLogFinish(state);
}
else if (!building)
GenericXLogAbort(state);
UnlockReleaseBuffer(buf);
}
}
MemoryContextDelete(updateCtx);
}
/*
@@ -666,7 +527,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk
*/
static void
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
{
BlockNumber newInsertPage = InvalidBlockNumber;
@@ -682,7 +543,7 @@ UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, in
HnswUpdateMetaPage(index, 0, NULL, newInsertPage, MAIN_FORKNUM, building);
/* Update neighbors */
HnswUpdateNeighborsOnDisk(index, support, element, m, false, building);
HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, false, building);
/* Update entry point if needed */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -693,12 +554,14 @@ UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, in
* Insert a tuple into the index
*/
bool
HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPointer heaptid, bool building)
HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building)
{
HnswElement entryPoint;
HnswElement element;
int m;
int efConstruction = HnswGetEfConstruction(index);
FmgrInfo *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
Oid collation = index->rd_indcollation[0];
LOCKMODE lockmode = ShareLock;
char *base = NULL;
@@ -713,8 +576,8 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
HnswGetMetaPageInfo(index, &m, &entryPoint);
/* Create an element */
element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
element = HnswInitElement(base, heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
HnswPtrStore(base, element->value, DatumGetPointer(value));
/* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -731,10 +594,10 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false);
/* Update graph on disk */
UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building);
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);
@@ -746,19 +609,24 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
* Insert a tuple into the index
*/
static void
HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid)
HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
{
Datum value;
const HnswTypeInfo *typeInfo = HnswGetTypeInfo(index);
HnswSupport support;
FmgrInfo *normprocinfo;
Oid collation = index->rd_indcollation[0];
HnswInitSupport(&support, index);
/* Detoast once for all calls */
value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Form index value */
if (!HnswFormIndexValue(&value, values, isnull, typeInfo, &support))
return;
/* Normalize if needed */
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
if (normprocinfo != NULL)
{
if (!HnswNormValue(normprocinfo, collation, &value, HnswGetType(index)))
return;
}
HnswInsertTupleOnDisk(index, &support, value, heaptid, false);
HnswInsertTupleOnDisk(index, value, values, isnull, heap_tid, false);
}
/*

View File

@@ -1,89 +1,43 @@
#include "postgres.h"
#include <limits.h>
#include "access/genam.h"
#include "access/relscan.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/float.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Algorithm 5 from paper
*/
static List *
GetScanItems(IndexScanDesc scan, Datum value)
GetScanItems(IndexScanDesc scan, Datum q)
{
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
Relation index = scan->indexRelation;
HnswSupport *support = &so->support;
FmgrInfo *procinfo = so->procinfo;
Oid collation = so->collation;
List *ep;
List *w;
int m;
HnswElement entryPoint;
char *base = NULL;
HnswQuery *q = &so->q;
/* Get m and entry point */
HnswGetMetaPageInfo(index, &m, &entryPoint);
q->value = value;
so->m = m;
if (entryPoint == NULL)
return NIL;
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, support, false));
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
for (int lc = entryPoint->level; lc >= 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, false, NULL, NULL, NULL, true, NULL);
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
ep = w;
}
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, support, m, false, NULL, &so->v, hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF ? &so->discarded : NULL, true, &so->tuples);
}
/*
* Resume scan at ground level with discarded candidates
*/
static List *
ResumeScanItems(IndexScanDesc scan)
{
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
Relation index = scan->indexRelation;
List *ep = NIL;
char *base = NULL;
int batch_size = hnsw_ef_search;
if (pairingheap_is_empty(so->discarded))
return NIL;
/* Get next batch of candidates */
for (int i = 0; i < batch_size; i++)
{
HnswSearchCandidate *sc;
if (pairingheap_is_empty(so->discarded))
break;
sc = HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded));
ep = lappend(ep, sc);
}
return HnswSearchLayer(base, &so->q, ep, batch_size, 0, index, &so->support, so->m, false, NULL, &so->v, &so->discarded, false, &so->tuples);
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
}
/*
@@ -105,25 +59,14 @@ GetScanValue(IndexScanDesc scan)
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Normalize if needed */
if (so->support.normprocinfo != NULL)
value = HnswNormValue(so->typeInfo, so->support.collation, value);
/* Fine if normalization fails */
if (so->normprocinfo != NULL)
HnswNormValue(so->normprocinfo, so->collation, &value, HnswGetType(scan->indexRelation));
}
return value;
}
#if defined(HNSW_MEMORY)
/*
* Show memory usage
*/
static void
ShowMemoryUsage(HnswScanOpaque so)
{
elog(INFO, "memory: %zu KB, tuples: " INT64_FORMAT, MemoryContextMemAllocated(so->tmpCtx, false) / 1024, so->tuples);
}
#endif
/*
* Prepare for an index scan
*/
@@ -132,28 +75,19 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
{
IndexScanDesc scan;
HnswScanOpaque so;
double maxMemory;
scan = RelationGetIndexScan(index, nkeys, norderbys);
so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
so->typeInfo = HnswGetTypeInfo(index);
/* Set support functions */
HnswInitSupport(&so->support, index);
/*
* Use a lower max allocation size than default to allow scanning more
* tuples for iterative search before exceeding work_mem
*/
so->first = true;
so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw scan temporary context",
0, 8 * 1024, 256 * 1024);
ALLOCSET_DEFAULT_SIZES);
/* Calculate max memory */
/* Add 256 extra bytes to fill last block when close */
maxMemory = (double) work_mem * hnsw_scan_mem_multiplier * 1024.0 + 256;
so->maxMemory = Min(maxMemory, (double) (SIZE_MAX / 2));
/* Set support functions */
so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
so->collation = index->rd_indcollation[0];
scan->opaque = so;
@@ -169,11 +103,6 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
so->first = true;
/* v and discarded are allocated in tmpCtx */
so->v.tids = NULL;
so->discarded = NULL;
so->tuples = 0;
so->previousDistance = -get_float8_infinity();
MemoryContextReset(so->tmpCtx);
if (keys && scan->numberOfKeys > 0)
@@ -204,10 +133,6 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)
@@ -233,91 +158,24 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
so->first = false;
#if defined(HNSW_MEMORY)
ShowMemoryUsage(so);
#endif
}
for (;;)
while (list_length(so->w) > 0)
{
char *base = NULL;
HnswSearchCandidate *sc;
HnswElement element;
HnswCandidate *hc = llast(so->w);
HnswElement element = HnswPtrAccess(base, hc->element);
ItemPointer heaptid;
if (list_length(so->w) == 0)
{
if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_OFF)
break;
/* Empty index */
if (so->discarded == NULL)
break;
/* Reached max number of tuples or memory limit */
if (so->tuples >= hnsw_max_scan_tuples || MemoryContextMemAllocated(so->tmpCtx, false) > so->maxMemory)
{
if (pairingheap_is_empty(so->discarded))
break;
/* Return remaining tuples */
so->w = lappend(so->w, HnswGetSearchCandidate(w_node, pairingheap_remove_first(so->discarded)));
}
else
{
/*
* Locking ensures when neighbors are read, the elements they
* reference will not be deleted (and replaced) during the
* iteration.
*
* Elements loaded into memory on previous iterations may have
* been deleted (and replaced), so when reading neighbors, the
* element version must be checked.
*/
LockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
so->w = ResumeScanItems(scan);
UnlockPage(scan->indexRelation, HNSW_SCAN_LOCK, ShareLock);
#if defined(HNSW_MEMORY)
ShowMemoryUsage(so);
#endif
}
if (list_length(so->w) == 0)
break;
}
sc = llast(so->w);
element = HnswPtrAccess(base, sc->element);
/* Move to next element if no valid heap TIDs */
if (element->heaptidsLength == 0)
{
so->w = list_delete_last(so->w);
/* Mark memory as free for next iteration */
if (hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF)
{
pfree(element);
pfree(sc);
}
continue;
}
heaptid = &element->heaptids[--element->heaptidsLength];
if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_STRICT)
{
if (sc->distance < so->previousDistance)
continue;
so->previousDistance = sc->distance;
}
MemoryContextSwitchTo(oldCtx);
scan->xs_heaptid = *heaptid;

File diff suppressed because it is too large Load Diff

View File

@@ -1,30 +1,21 @@
#include "postgres.h"
#include "access/genam.h"
#include <math.h>
#include "access/generic_xlog.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "nodes/pg_list.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/*
* Check if deletion list contains an element
* Check if deleted list contains an index TID
*/
static bool
DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
{
return tidhash_lookup(deleting, *indextid) != NULL;
return tidhash_lookup(deleted, *indextid) != NULL;
}
/*
@@ -37,20 +28,17 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
IndexBulkDeleteResult *stats = vacuumstate->stats;
/* Store separately since HnswElement level is uint8 */
/* Store separately since highestPoint.level is uint8 */
int highestLevel = -1;
int fallbackLevel = -1;
/* Initialize highest point and fallback point */
/* Initialize highest point */
highestPoint->blkno = InvalidBlockNumber;
highestPoint->offno = InvalidOffsetNumber;
fallbackPoint->blkno = InvalidBlockNumber;
fallbackPoint->offno = InvalidOffsetNumber;
while (BlockNumberIsValid(blkno))
{
@@ -80,14 +68,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/*
* Skip deleted tuples. It is important they are not added to the
* deletion list to avoid false positives in NeedsUpdated and
* ConfirmRepaired.
*/
if (etup->deleted)
continue;
if (ItemPointerIsValid(&etup->heaptids[0]))
{
for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -121,40 +101,23 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!ItemPointerIsValid(&etup->heaptids[0]))
{
ItemPointerData indextid;
ItemPointerData ip;
bool found;
/* Add to deletion list */
ItemPointerSet(&indextid, blkno, offno);
/* Add to deleted list */
ItemPointerSet(&ip, blkno, offno);
tidhash_insert(vacuumstate->deleting, indextid, &found);
tidhash_insert(vacuumstate->deleted, ip, &found);
Assert(!found);
}
else if (etup->level > highestLevel)
else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
{
if (BlockNumberIsValid(highestPoint->blkno))
{
/* Current highest point becomes fallback */
fallbackPoint->blkno = highestPoint->blkno;
fallbackPoint->offno = highestPoint->offno;
fallbackPoint->level = highestPoint->level;
fallbackLevel = highestLevel;
}
/* Keep track of highest point */
/* Keep track of highest non-entry point */
highestPoint->blkno = blkno;
highestPoint->offno = offno;
highestPoint->level = etup->level;
highestLevel = etup->level;
}
else if (etup->level > fallbackLevel)
{
/* Keep track of second highest point */
fallbackPoint->blkno = blkno;
fallbackPoint->offno = offno;
fallbackPoint->level = etup->level;
fallbackLevel = etup->level;
}
}
blkno = HnswPageGetOpaque(page)->nextblkno;
@@ -166,10 +129,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
UnlockReleaseBuffer(buf);
}
#ifdef HNSW_MEMORY
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
#endif
}
/*
@@ -200,8 +159,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
/* Check if in deleted list */
if (DeletedContains(vacuumstate->deleted, indextid))
{
needsUpdated = true;
break;
@@ -210,8 +169,7 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */
/* There should always be more than zero indextids, but check for safety */
if (!needsUpdated && ntup->count > 0)
if (!needsUpdated)
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
UnlockReleaseBuffer(buf);
@@ -226,12 +184,13 @@ static void
RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswElement entryPoint)
{
Relation index = vacuumstate->index;
HnswSupport *support = &vacuumstate->support;
Buffer buf;
Page page;
GenericXLogState *state;
int m = vacuumstate->m;
int efConstruction = vacuumstate->efConstruction;
FmgrInfo *procinfo = vacuumstate->procinfo;
Oid collation = vacuumstate->collation;
BufferAccessStrategy bas = vacuumstate->bas;
HnswNeighborTuple ntup = vacuumstate->ntup;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
@@ -246,7 +205,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
element->heaptidsLength = 0;
/* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, true);
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
/* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
@@ -270,7 +229,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
UnlockReleaseBuffer(buf);
/* Update neighbors */
HnswUpdateNeighborsOnDisk(index, support, element, m, true, false);
HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, true, false);
}
/*
@@ -280,7 +239,6 @@ static void
RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
{
Relation index = vacuumstate->index;
HnswSupport *support = &vacuumstate->support;
HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement entryPoint;
MemoryContext oldCtx = MemoryContextSwitchTo(vacuumstate->tmpCtx);
@@ -297,27 +255,12 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
/* Get a shared lock */
LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Get latest entry point */
entryPoint = HnswGetEntryPoint(index);
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
/* Use fallback point if highest point is entry point */
if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno)
{
highestPoint = &vacuumstate->fallbackPoint;
if (!BlockNumberIsValid(highestPoint->blkno))
highestPoint = NULL;
}
if (highestPoint != NULL)
{
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, entryPoint);
}
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -335,7 +278,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
if (DeletingElement(vacuumstate->deleting, &epData))
if (DeletedContains(vacuumstate->deleted, &epData))
{
/*
* Replace the entry point with the highest point. If highest
@@ -351,7 +294,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
* is outdated, this can remove connections at higher levels in
* the graph until they are repaired, but this should be fine.
*/
HnswLoadElement(entryPoint, NULL, NULL, index, support, true, NULL);
HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
if (NeedsUpdated(vacuumstate, entryPoint))
{
@@ -421,10 +364,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip updating neighbors if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
@@ -488,103 +427,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(vacuumstate->tmpCtx);
#ifdef HNSW_VACUUM_PROGRESS
if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
{
BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
}
#endif
}
}
/*
* Confirm graph was repaired
*/
static void
ConfirmRepaired(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
while (BlockNumberIsValid(blkno))
{
Buffer buf;
Page page;
OffsetNumber offno;
OffsetNumber maxoffno;
vacuum_delay_point();
buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
OffsetNumber neighborOffno;
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
if (neighborPage == blkno)
{
nbuf = buf;
npage = page;
}
else
{
nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
LockBuffer(nbuf, BUFFER_LOCK_SHARE);
npage = BufferGetPage(nbuf);
}
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Check neighbors */
for (int i = 0; i < ntup->count; i++)
{
ItemPointer indextid = &ntup->indextids[i];
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
elog(ERROR, "hnsw graph not repaired");
}
if (nbuf != buf)
UnlockReleaseBuffer(nbuf);
}
blkno = HnswPageGetOpaque(page)->nextblkno;
UnlockReleaseBuffer(buf);
}
}
@@ -600,15 +442,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BufferAccessStrategy bas = vacuumstate->bas;
/*
* Wait for inserts and index scans to complete. Inserts and scans before
* this point may visit tuples about to be deleted. Inserts and scans
* after this point will not, since the graph has been repaired.
* Wait for index scans to complete. Scans before this point may contain
* tuples about to be deleted. Scans after this point will not, since the
* graph has been repaired.
*/
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
ConfirmRepaired(vacuumstate);
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
@@ -683,22 +520,13 @@ MarkDeleted(HnswVacuumState * vacuumstate)
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Overwrite element */
/* Use memset instead of MemSet to keep clang-tidy happy */
etup->deleted = 1;
memset(&etup->data, 0, VARSIZE_ANY(&etup->data));
MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
/* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++)
ItemPointerSetInvalid(&ntup->indextids[i]);
/* Increment version */
/* This is used to avoid incorrect reads for iterative scans */
/* Reserve some bits for future use */
etup->version++;
if (etup->version > 15)
etup->version = 1;
ntup->version = etup->version;
/*
* We modified the tuples in place, no need to call
* PageIndexTupleOverwrite
@@ -745,18 +573,18 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
vacuumstate->callback_state = callback_state;
vacuumstate->efConstruction = HnswGetEfConstruction(index);
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
vacuumstate->collation = index->rd_indcollation[0];
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw vacuum temporary context",
ALLOCSET_DEFAULT_SIZES);
HnswInitSupport(&vacuumstate->support, index);
/* Get m from metapage */
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
/* Create hash table */
vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
}
/*
@@ -765,7 +593,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
static void
FreeVacuumState(HnswVacuumState * vacuumstate)
{
tidhash_destroy(vacuumstate->deleting);
tidhash_destroy(vacuumstate->deleted);
FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx);
@@ -783,13 +611,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
/* Pass 1: Remove heap TIDs */
HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate));
RemoveHeapTids(&vacuumstate);
/* Pass 2: Repair graph */
HnswBench("RepairGraph", RepairGraph(&vacuumstate));
RepairGraph(&vacuumstate);
/* Passes 3 and 4: Confirm repaired and mark as deleted */
HnswBench("MarkDeleted", MarkDeleted(&vacuumstate));
/* Pass 3: Mark as deleted */
MarkDeleted(&vacuumstate);
FreeVacuumState(&vacuumstate);

608
src/intvec.c Normal file
View File

@@ -0,0 +1,608 @@
#include "postgres.h"
#include <limits.h>
#include <math.h>
#include "catalog/pg_type.h"
#include "fmgr.h"
#include "intvec.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
/*
* Ensure same dimensions
*/
static inline void
CheckDims(IntVector * a, IntVector * b)
{
if (a->dim != b->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("different intvec dimensions %d and %d", a->dim, b->dim)));
}
/*
* Ensure expected dimensions
*/
static inline void
CheckExpectedDim(int32 typmod, int dim)
{
if (typmod != -1 && typmod != dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected %d dimensions, not %d", typmod, dim)));
}
/*
* Ensure valid dimensions
*/
static inline void
CheckDim(int dim)
{
if (dim < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("intvec must have at least 1 dimension")));
if (dim > INTVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("intvec cannot have more than %d dimensions", INTVEC_MAX_DIM)));
}
/*
* Ensure element in range
*/
static inline void
CheckElement(long value)
{
if (value < SCHAR_MIN || value > SCHAR_MAX)
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value \"%ld\" is out of range for type intvec", value)));
}
/*
* Allocate and initialize a new int vector
*/
IntVector *
InitIntVector(int dim)
{
IntVector *result;
int size;
size = INTVEC_SIZE(dim);
result = (IntVector *) palloc0(size);
SET_VARSIZE(result, size);
result->dim = dim;
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
intvec_isspace(char ch)
{
if (ch == ' ' ||
ch == '\t' ||
ch == '\n' ||
ch == '\r' ||
ch == '\v' ||
ch == '\f')
return true;
return false;
}
/*
* Convert textual representation to internal representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_in);
Datum
intvec_in(PG_FUNCTION_ARGS)
{
char *lit = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2);
int8 x[INTVEC_MAX_DIM];
int dim = 0;
char *pt;
char *stringEnd;
IntVector *result;
char *litcopy = pstrdup(lit);
char *str = litcopy;
while (intvec_isspace(*str))
str++;
if (*str != '[')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed intvec literal: \"%s\"", lit),
errdetail("Vector contents must start with \"[\".")));
str++;
pt = strtok(str, ",");
stringEnd = pt;
while (pt != NULL && *stringEnd != ']')
{
long l;
if (dim == INTVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("intvec cannot have more than %d dimensions", INTVEC_MAX_DIM)));
while (intvec_isspace(*pt))
pt++;
/* Check for empty string like float4in */
if (*pt == '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type intvec: \"%s\"", lit)));
/* Use similar logic as int2vectorin */
errno = 0;
l = strtol(pt, &stringEnd, 10);
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type intvec: \"%s\"", lit)));
if (errno == ERANGE || l < SCHAR_MIN || l > SCHAR_MAX)
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value \"%s\" is out of range for type intvec", pt)));
x[dim++] = l;
while (intvec_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0' && *stringEnd != ']')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type intvec: \"%s\"", lit)));
pt = strtok(NULL, ",");
}
if (stringEnd == NULL || *stringEnd != ']')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed intvec literal: \"%s\"", lit),
errdetail("Unexpected end of input.")));
stringEnd++;
/* Only whitespace is allowed after the closing brace */
while (intvec_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed intvec literal: \"%s\"", lit),
errdetail("Junk after closing right brace.")));
/* Ensure no consecutive delimiters since strtok skips */
for (pt = lit + 1; *pt != '\0'; pt++)
{
if (pt[-1] == ',' && *pt == ',')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed intvec literal: \"%s\"", lit)));
}
if (dim < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("intvec must have at least 1 dimension")));
pfree(litcopy);
CheckExpectedDim(typmod, dim);
result = InitIntVector(dim);
for (int i = 0; i < dim; i++)
result->x[i] = x[i];
PG_RETURN_POINTER(result);
}
/*
* Convert internal representation to textual representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_out);
Datum
intvec_out(PG_FUNCTION_ARGS)
{
IntVector *vector = PG_GETARG_INTVEC_P(0);
int dim = vector->dim;
char *buf;
char *ptr;
/*
* Need:
*
* dim * 4 bytes for elements (-128 to 127)
*
* dim - 1 bytes for separator
*
* 3 bytes for [, ], and \0
*/
buf = (char *) palloc(5 * dim + 2);
ptr = buf;
*ptr = '[';
ptr++;
for (int i = 0; i < dim; i++)
{
if (i > 0)
{
*ptr = ',';
ptr++;
}
#if PG_VERSION_NUM >= 140000
ptr += pg_ltoa(vector->x[i], ptr);
#else
pg_ltoa(vector->x[i], ptr);
while (*ptr != '\0')
ptr++;
#endif
}
*ptr = ']';
ptr++;
*ptr = '\0';
PG_FREE_IF_COPY(vector, 0);
PG_RETURN_CSTRING(buf);
}
/*
* Convert type modifier
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_typmod_in);
Datum
intvec_typmod_in(PG_FUNCTION_ARGS)
{
ArrayType *ta = PG_GETARG_ARRAYTYPE_P(0);
int32 *tl;
int n;
tl = ArrayGetIntegerTypmods(ta, &n);
if (n != 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid type modifier")));
if (*tl < 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type intvec must be at least 1")));
if (*tl > INTVEC_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions for type intvec cannot exceed %d", INTVEC_MAX_DIM)));
PG_RETURN_INT32(*tl);
}
/*
* Convert external binary representation to internal representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_recv);
Datum
intvec_recv(PG_FUNCTION_ARGS)
{
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
int32 typmod = PG_GETARG_INT32(2);
IntVector *result;
int16 dim;
int16 unused;
dim = pq_getmsgint(buf, sizeof(int16));
unused = pq_getmsgint(buf, sizeof(int16));
CheckDim(dim);
CheckExpectedDim(typmod, dim);
if (unused != 0)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("expected unused to be 0, not %d", unused)));
result = InitIntVector(dim);
for (int i = 0; i < dim; i++)
result->x[i] = pq_getmsgint(buf, sizeof(int8));
PG_RETURN_POINTER(result);
}
/*
* Convert internal representation to the external binary representation
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_send);
Datum
intvec_send(PG_FUNCTION_ARGS)
{
IntVector *vec = PG_GETARG_INTVEC_P(0);
StringInfoData buf;
pq_begintypsend(&buf);
pq_sendint(&buf, vec->dim, sizeof(int16));
pq_sendint(&buf, vec->unused, sizeof(int16));
for (int i = 0; i < vec->dim; i++)
pq_sendint8(&buf, vec->x[i]);
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
}
/*
* Convert int vector to int vector
* This is needed to check the type modifier
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec);
Datum
intvec(PG_FUNCTION_ARGS)
{
IntVector *vec = PG_GETARG_INTVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
CheckExpectedDim(typmod, vec->dim);
PG_RETURN_POINTER(vec);
}
/*
* Convert array to intvec vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_intvec);
Datum
array_to_intvec(PG_FUNCTION_ARGS)
{
ArrayType *array = PG_GETARG_ARRAYTYPE_P(0);
int32 typmod = PG_GETARG_INT32(1);
Vector *result;
int16 typlen;
bool typbyval;
char typalign;
Datum *elemsp;
int nelemsp;
if (ARR_NDIM(array) > 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("array must be 1-D")));
if (ARR_HASNULL(array) && array_contains_nulls(array))
ereport(ERROR,
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
errmsg("array must not contain nulls")));
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, NULL, &nelemsp);
CheckDim(nelemsp);
CheckExpectedDim(typmod, nelemsp);
result = InitVector(nelemsp);
if (ARR_ELEMTYPE(array) == INT4OID)
{
for (int i = 0; i < nelemsp; i++)
{
long l = DatumGetInt32(elemsp[i]);
CheckElement(l);
result->x[i] = l;
}
}
else
{
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("unsupported array type")));
}
/*
* Free allocation from deconstruct_array. Do not free individual elements
* when pass-by-reference since they point to original array.
*/
pfree(elemsp);
PG_RETURN_POINTER(result);
}
/*
* Get the L2 distance between int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_l2_distance);
Datum
intvec_l2_distance(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
{
int diff = ax[i] - bx[i];
distance += diff * diff;
}
PG_RETURN_FLOAT8(sqrt((double) distance));
}
/*
* Get the L2 squared distance between int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_l2_squared_distance);
Datum
intvec_l2_squared_distance(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
{
int diff = ax[i] - bx[i];
distance += diff * diff;
}
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the inner product of two int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_inner_product);
Datum
intvec_inner_product(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i];
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the negative inner product of two int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_negative_inner_product);
Datum
intvec_negative_inner_product(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i];
PG_RETURN_FLOAT8((double) -distance);
}
/*
* Get the cosine distance between two int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_cosine_distance);
Datum
intvec_cosine_distance(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
int norma = 0;
int normb = 0;
double similarity;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
{
int8 axi = ax[i];
int8 bxi = bx[i];
distance += axi * bxi;
norma += axi * axi;
normb += bxi * bxi;
}
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
similarity = (double) distance / sqrt((double) norma * (double) normb);
#ifdef _MSC_VER
/* /fp:fast may not propagate NaN */
if (isnan(similarity))
PG_RETURN_FLOAT8(NAN);
#endif
/* Keep in range */
if (similarity > 1)
similarity = 1;
else if (similarity < -1)
similarity = -1;
PG_RETURN_FLOAT8(1 - similarity);
}
/*
* Get the L1 distance between two int vectors
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_l1_distance);
Datum
intvec_l1_distance(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
IntVector *b = PG_GETARG_INTVEC_P(1);
int8 *ax = a->x;
int8 *bx = b->x;
int distance = 0;
CheckDims(a, b);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += abs(ax[i] - bx[i]);
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the L2 norm of an int vector
*/
PGDLLEXPORT PG_FUNCTION_INFO_V1(intvec_l2_norm);
Datum
intvec_l2_norm(PG_FUNCTION_ARGS)
{
IntVector *a = PG_GETARG_INTVEC_P(0);
int8 *ax = a->x;
int norm = 0;
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
norm += ax[i] * ax[i];
PG_RETURN_FLOAT8(sqrt((double) norm));
}

23
src/intvec.h Normal file
View File

@@ -0,0 +1,23 @@
#ifndef INTVEC_H
#define INTVEC_H
#include "vector.h"
#define INTVEC_MAX_DIM VECTOR_MAX_DIM
#define INTVEC_SIZE(_dim) (offsetof(IntVector, x) + sizeof(int8)*(_dim))
#define DatumGetIntVector(x) ((IntVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_INTVEC_P(x) DatumGetIntVector(PG_GETARG_DATUM(x))
#define PG_RETURN_INTVEC_P(x) PG_RETURN_POINTER(x)
typedef struct IntVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused;
int8 x[FLEXIBLE_ARRAY_MEMBER];
} IntVector;
IntVector *InitIntVector(int dim);
#endif

View File

@@ -2,37 +2,20 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/parallel.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/sampling.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -40,6 +23,12 @@
#include "pgstat.h"
#endif
#if PG_VERSION_NUM >= 130000
#define CALLBACK_ITEM_POINTER ItemPointer tid
#else
#define CALLBACK_ITEM_POINTER HeapTuple hup
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_status.h"
#include "utils/wait_event.h"
@@ -63,24 +52,24 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/*
* Check with KMEANS_NORM_PROC that the value can be normalized since
* spherical distance function expects unit vectors
* Normalize with KMEANS_NORM_PROC since spherical distance function
* expects unit vectors
*/
if (buildstate->kmeansnormprocinfo != NULL)
{
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value))
return;
}
if (samples->length < targsamples)
{
VectorArraySet(samples, samples->length, DatumGetPointer(value));
VectorArraySet(samples, samples->length, DatumGetVector(value));
samples->length++;
}
else
{
if (buildstate->rowstoskip < 0)
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples);
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
if (buildstate->rowstoskip <= 0)
{
@@ -91,21 +80,18 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
#endif
Assert(k >= 0 && k < targsamples);
VectorArraySet(samples, k, DatumGetPointer(value));
VectorArraySet(samples, k, DatumGetVector(value));
}
buildstate->rowstoskip -= 1;
}
/* Increment after reservoir_get_next_S */
buildstate->samplerows += 1;
}
/*
* Callback for sampling
*/
static void
SampleCallback(Relation index, ItemPointer tid, Datum *values,
SampleCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
@@ -135,7 +121,6 @@ SampleRows(IvfflatBuildState * buildstate)
int targsamples = buildstate->samples->maxlen;
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
buildstate->samplerows = 0;
buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
@@ -145,21 +130,16 @@ SampleRows(IvfflatBuildState * buildstate)
{
BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
/* Set anyvisible to false like table_index_build_scan */
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
}
/* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL)
IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx);
}
/*
* Add tuple to sort
*/
static void
AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
{
double distance;
double minDistance = DBL_MAX;
@@ -173,10 +153,8 @@ AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
/* Normalize if needed */
if (buildstate->normprocinfo != NULL)
{
if (!IvfflatCheckNorm(buildstate->normprocinfo, buildstate->collation, value))
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value))
return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
}
/* Find the list that minimizes the distance */
@@ -222,12 +200,16 @@ AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
* Callback for table_index_build_scan
*/
static void
BuildCallback(Relation index, ItemPointer tid, Datum *values,
BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
bool *isnull, bool tupleIsAlive, void *state)
{
IvfflatBuildState *buildstate = (IvfflatBuildState *) state;
MemoryContext oldCtx;
#if PG_VERSION_NUM < 130000
ItemPointer tid = &hup->t_self;
#endif
/* Skip nulls */
if (isnull[0])
return;
@@ -236,7 +218,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add tuple to sort */
AddTupleToSort(tid, values, buildstate);
AddTupleToSort(index, tid, values, buildstate);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
@@ -249,11 +231,11 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
static inline void
GetNextTuple(Tuplesortstate *sortstate, TupleDesc tupdesc, TupleTableSlot *slot, IndexTuple *itup, int *list)
{
Datum value;
bool isnull;
if (tuplesort_gettupleslot(sortstate, true, false, slot, NULL))
{
Datum value;
bool isnull;
*list = DatumGetInt32(slot_getattr(slot, 1, &isnull));
value = slot_getattr(slot, 3, &isnull);
@@ -275,8 +257,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
IndexTuple itup = NULL; /* silence compiler warning */
int64 inserted = 0;
TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsMinimalTuple);
TupleDesc tupdesc = buildstate->tupdesc;
TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsMinimalTuple);
TupleDesc tupdesc = RelationGetDescr(index);
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
@@ -339,28 +321,16 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->heap = heap;
buildstate->index = index;
buildstate->indexInfo = indexInfo;
buildstate->typeInfo = IvfflatGetTypeInfo(index);
buildstate->tupdesc = RelationGetDescr(index);
buildstate->lists = IvfflatGetLists(index);
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Disallow varbit since require fixed dimensions */
if (TupleDescAttr(index->rd_att, 0)->atttypid == VARBITOID)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("type not supported for ivfflat index")));
/* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("column does not have dimensions")));
elog(ERROR, "column does not have dimensions");
if (buildstate->dimensions > buildstate->typeInfo->maxDimensions)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("column cannot have more than %d dimensions for ivfflat index", buildstate->typeInfo->maxDimensions)));
if (buildstate->dimensions > IVFFLAT_MAX_DIM)
elog(ERROR, "column cannot have more than %d dimensions for ivfflat index", IVFFLAT_MAX_DIM);
buildstate->reltuples = 0;
buildstate->indtuples = 0;
@@ -373,29 +343,17 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
/* Require more than one dimension for spherical k-means */
if (buildstate->kmeansnormprocinfo != NULL && buildstate->dimensions == 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("dimensions must be greater than one for this opclass")));
elog(ERROR, "dimensions must be greater than one for this opclass");
/* Create tuple description for sorting */
buildstate->sortdesc = CreateTemplateTupleDesc(3);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(buildstate->sortdesc);
#endif
buildstate->tupdesc = CreateTemplateTupleDesc(3);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->tupdesc, (AttrNumber) 3, "vector", RelationGetDescr(index)->attrs[0].atttypid, -1, 0);
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
buildstate->slot = MakeSingleTupleTableSlot(buildstate->tupdesc, &TTSOpsVirtual);
buildstate->memoryUsed = 0;
buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
/* TODO Move allocation to page creation */
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions);
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -449,12 +407,11 @@ ComputeCenters(IvfflatBuildState * buildstate)
numSamples = 1;
/* Sample rows */
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
/* TODO Ensure within maintenance_work_mem */
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions);
if (buildstate->heap != NULL)
{
IvfflatBench("sample rows", SampleRows(buildstate));
SampleRows(buildstate);
if (buildstate->samples->length < buildstate->lists)
{
@@ -466,7 +423,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
}
/* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed));
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
/* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples);
@@ -502,8 +459,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
* Create list pages
*/
static void
CreateListPages(Relation index, VectorArray centers, int lists,
ForkNumber forkNum, ListInfo * *listInfo)
CreateListPages(Relation index, VectorArray centers, int dimensions,
int lists, ForkNumber forkNum, ListInfo * *listInfo)
{
Buffer buf;
Page page;
@@ -511,7 +468,7 @@ CreateListPages(Relation index, VectorArray centers, int lists,
Size listSize;
IvfflatList list;
listSize = MAXALIGN(IVFFLAT_LIST_SIZE(centers->itemsize));
listSize = MAXALIGN(IVFFLAT_LIST_SIZE(dimensions));
list = palloc0(listSize);
buf = IvfflatNewBuffer(index, forkNum);
@@ -521,13 +478,10 @@ CreateListPages(Relation index, VectorArray centers, int lists,
{
OffsetNumber offno;
/* Zero memory for each list */
MemSet(list, 0, listSize);
/* Load list */
list->startPage = InvalidBlockNumber;
list->insertPage = InvalidBlockNumber;
memcpy(&list->center, VectorArrayGet(centers, i), VARSIZE_ANY(VectorArrayGet(centers, i)));
memcpy(&list->center, VectorArrayGet(centers, i), VECTOR_SIZE(dimensions));
/* Ensure free space */
if (PageGetFreeSpace(page) < listSize)
@@ -593,20 +547,6 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
}
#endif
/*
* Initialize build sort state
*/
static Tuplesortstate *
InitBuildSortState(TupleDesc tupdesc, int memory, SortCoordinate coordinate)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, memory, coordinate, false);
}
/*
* Within leader, wait for end of heap scan
*/
@@ -646,7 +586,7 @@ ParallelHeapScan(IvfflatBuildState * buildstate)
* Perform a worker's portion of a parallel sort
*/
static void
IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, Sharedsort *sharedsort, char *ivfcenters, int sortmem, bool progress)
IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, Sharedsort *sharedsort, Vector * ivfcenters, int sortmem, bool progress)
{
SortCoordinate coordinate;
IvfflatBuildState buildstate;
@@ -654,6 +594,12 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
double reltuples;
IndexInfo *indexInfo;
/* Sort options, which must match AssignTuples */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
/* Initialize local tuplesort coordination state */
coordinate = palloc0(sizeof(SortCoordinateData));
coordinate->isWorker = true;
@@ -664,16 +610,12 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
indexInfo = BuildIndexInfo(ivfspool->index);
indexInfo->ii_Concurrent = ivfshared->isconcurrent;
InitBuildState(&buildstate, ivfspool->heap, ivfspool->index, indexInfo);
memcpy(buildstate.centers->items, ivfcenters, buildstate.centers->itemsize * buildstate.centers->maxlen);
memcpy(buildstate.centers->items, ivfcenters, VECTOR_SIZE(buildstate.centers->dim) * buildstate.centers->maxlen);
buildstate.centers->length = buildstate.centers->maxlen;
ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
ivfspool->sortstate = tuplesort_begin_heap(buildstate.tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, sortmem, coordinate, false);
buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
ParallelTableScanFromIvfflatShared(ivfshared));
reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -716,7 +658,7 @@ IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc)
IvfflatSpool *ivfspool;
IvfflatShared *ivfshared;
Sharedsort *sharedsort;
char *ivfcenters;
Vector *ivfcenters;
Relation heapRel;
Relation indexRel;
LOCKMODE heapLockmode;
@@ -830,7 +772,7 @@ IvfflatBeginParallel(IvfflatBuildState * buildstate, bool isconcurrent, int requ
Size estcenters;
IvfflatShared *ivfshared;
Sharedsort *sharedsort;
char *ivfcenters;
Vector *ivfcenters;
IvfflatLeader *ivfleader = (IvfflatLeader *) palloc0(sizeof(IvfflatLeader));
bool leaderparticipates = true;
int querylen;
@@ -857,7 +799,7 @@ IvfflatBeginParallel(IvfflatBuildState * buildstate, bool isconcurrent, int requ
shm_toc_estimate_chunk(&pcxt->estimator, estivfshared);
estsort = tuplesort_estimate_shared(scantuplesortstates);
shm_toc_estimate_chunk(&pcxt->estimator, estsort);
estcenters = buildstate->centers->itemsize * buildstate->centers->maxlen;
estcenters = VECTOR_SIZE(buildstate->dimensions) * buildstate->lists;
shm_toc_estimate_chunk(&pcxt->estimator, estcenters);
shm_toc_estimate_keys(&pcxt->estimator, 3);
@@ -909,7 +851,7 @@ IvfflatBeginParallel(IvfflatBuildState * buildstate, bool isconcurrent, int requ
tuplesort_initialize_shared(sharedsort, scantuplesortstates,
pcxt->seg);
ivfcenters = shm_toc_allocate(pcxt->toc, estcenters);
ivfcenters = (Vector *) shm_toc_allocate(pcxt->toc, estcenters);
memcpy(ivfcenters, buildstate->centers->items, estcenters);
shm_toc_insert(pcxt->toc, PARALLEL_KEY_IVFFLAT_SHARED, ivfshared);
@@ -967,6 +909,12 @@ AssignTuples(IvfflatBuildState * buildstate)
int parallel_workers = 0;
SortCoordinate coordinate = NULL;
/* Sort options, which must match IvfflatParallelScanAndSort */
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_ASSIGN);
/* Calculate parallel workers */
@@ -987,7 +935,7 @@ AssignTuples(IvfflatBuildState * buildstate)
}
/* Begin serial/leader tuplesort */
buildstate->sortstate = InitBuildSortState(buildstate->sortdesc, maintenance_work_mem, coordinate);
buildstate->sortstate = tuplesort_begin_heap(buildstate->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, maintenance_work_mem, coordinate, false);
/* Add tuples to sort */
if (buildstate->heap != NULL)
@@ -1040,13 +988,9 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
/* Create pages */
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
if (forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);
FreeBuildState(buildstate);
}
@@ -1059,10 +1003,6 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
IndexBuildResult *result;
IvfflatBuildState buildstate;
#ifdef IVFFLAT_BENCH
SeedRandom(42);
#endif
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

View File

@@ -3,35 +3,21 @@
#include <float.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/pg_list.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
#endif
int ivfflat_probes;
int ivfflat_iterative_scan;
int ivfflat_max_probes;
static relopt_kind ivfflat_relopt_kind;
static const struct config_enum_entry ivfflat_iterative_scan_options[] = {
{"off", IVFFLAT_ITERATIVE_SCAN_OFF, false},
{"relaxed_order", IVFFLAT_ITERATIVE_SCAN_RELAXED, false},
{NULL, 0, false}
};
/*
* Initialize index options and variables
*/
@@ -40,21 +26,16 @@ IvfflatInit(void)
{
ivfflat_relopt_kind = add_reloption_kind();
add_int_reloption(ivfflat_relopt_kind, "lists", "Number of inverted lists",
IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, AccessExclusiveLock);
IVFFLAT_DEFAULT_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif
);
DefineCustomIntVariable("ivfflat.probes", "Sets the number of probes",
"Valid range is 1..lists.", &ivfflat_probes,
IVFFLAT_DEFAULT_PROBES, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
DefineCustomEnumVariable("ivfflat.iterative_scan", "Sets the mode for iterative scans",
NULL, &ivfflat_iterative_scan,
IVFFLAT_ITERATIVE_SCAN_OFF, ivfflat_iterative_scan_options, PGC_USERSET, 0, NULL, NULL, NULL);
/* If this is less than probes, probes is used */
DefineCustomIntVariable("ivfflat.max_probes", "Sets the max number of probes for iterative scans",
NULL, &ivfflat_max_probes,
IVFFLAT_MAX_LISTS, IVFFLAT_MIN_LISTS, IVFFLAT_MAX_LISTS, PGC_USERSET, 0, NULL, NULL, NULL);
MarkGUCPrefixReserved("ivfflat");
}
@@ -91,30 +72,22 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
GenericCosts costs;
int lists;
double ratio;
double sequentialRatio = 0.5;
double startupPages;
double spc_seq_page_cost;
Relation index;
/* Never use index without order */
if (path->indexorderbys == NIL)
if (path->indexorderbys == NULL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
*indexStartupCost = DBL_MAX;
*indexTotalCost = DBL_MAX;
*indexSelectivity = 0;
*indexCorrelation = 0;
*indexPages = 0;
#if PG_VERSION_NUM >= 180000
/* See "On disable_cost" thread on pgsql-hackers */
path->path.disabled_nodes = 2;
#endif
return;
}
MemSet(&costs, 0, sizeof(costs));
genericcostestimate(root, path, loop_count, &costs);
index = index_open(path->indexinfo->indexoid, NoLock);
IvfflatGetMetaPageInfo(index, &lists, NULL);
index_close(index, NoLock);
@@ -124,26 +97,41 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
if (ratio > 1.0)
ratio = 1.0;
/*
* This gives us the subset of tuples to visit. This value is passed into
* the generic cost estimator to determine the number of pages to visit
* during the index scan.
*/
costs.numIndexTuples = path->indexinfo->tuples * ratio;
genericcostestimate(root, path, loop_count, &costs);
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
/* Change some page cost from random to sequential */
costs.indexTotalCost -= sequentialRatio * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Startup cost is cost before returning the first row */
costs.indexStartupCost = costs.indexTotalCost * ratio;
/* Adjust cost if needed since TOAST not included in seq scan cost */
startupPages = costs.numIndexPages * ratio;
if (startupPages > path->indexinfo->rel->pages && ratio < 0.5)
if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
{
/* Change rest of page cost from random to sequential */
costs.indexStartupCost -= (1 - sequentialRatio) * startupPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Change all page cost from random to sequential */
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */
costs.indexStartupCost -= (startupPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
else
{
/* Change some page cost from random to sequential */
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
}
*indexStartupCost = costs.indexStartupCost;
/*
* If the list selectivity is lower than what is returned from the generic
* cost estimator, use that.
*/
if (ratio < costs.indexSelectivity)
costs.indexSelectivity = ratio;
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost;
*indexSelectivity = costs.indexSelectivity;
*indexCorrelation = costs.indexCorrelation;
@@ -160,10 +148,23 @@ ivfflatoptions(Datum reloptions, bool validate)
{"lists", RELOPT_TYPE_INT, offsetof(IvfflatOptions, lists)},
};
#if PG_VERSION_NUM >= 130000
return (bytea *) build_reloptions(reloptions, validate,
ivfflat_relopt_kind,
sizeof(IvfflatOptions),
tab, lengthof(tab));
#else
relopt_value *options;
int numoptions;
IvfflatOptions *rdopts;
options = parseRelOptions(reloptions, validate, ivfflat_relopt_kind, &numoptions);
rdopts = allocateReloptStruct(sizeof(IvfflatOptions), options, numoptions);
fillRelOptions((void *) rdopts, sizeof(IvfflatOptions), options, numoptions,
validate, tab, lengthof(tab));
return (bytea *) rdopts;
#endif
}
/*
@@ -180,80 +181,19 @@ ivfflatvalidate(Oid opclassoid)
*
* See https://www.postgresql.org/docs/current/index-api.html
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 5,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = ivfflatbuild,
.ambuildempty = ivfflatbuildempty,
.aminsert = ivfflatinsert,
.aminsertcleanup = NULL,
.ambulkdelete = ivfflatbulkdelete,
.amvacuumcleanup = ivfflatvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = ivfflatcostestimate,
.amgettreeheight = NULL,
.amoptions = ivfflatoptions,
.amproperty = NULL,
.ambuildphasename = ivfflatbuildphasename,
.amvalidate = ivfflatvalidate,
.amadjustmembers = NULL,
.ambeginscan = ivfflatbeginscan,
.amrescan = ivfflatrescan,
.amgettuple = ivfflatgettuple,
.amgetbitmap = NULL,
.amendscan = ivfflatendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 5;
amroutine->amsupport = 4;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -264,31 +204,21 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amclusterable = false;
amroutine->ampredlocks = false;
amroutine->amcanparallel = false;
#if PG_VERSION_NUM >= 170000
amroutine->amcanbuildparallel = true;
#endif
amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
#if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false;
#endif
amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL;
#endif
amroutine->amkeytype = InvalidOid;
/* Interface functions */
amroutine->ambuild = ivfflatbuild;
amroutine->ambuildempty = ivfflatbuildempty;
amroutine->aminsert = ivfflatinsert;
#if PG_VERSION_NUM >= 170000
amroutine->aminsertcleanup = NULL;
#endif
amroutine->ambulkdelete = ivfflatbulkdelete;
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
amroutine->amcostestimate = ivfflatcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = ivfflatoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -309,11 +239,5 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -9,15 +9,10 @@
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/condition_variable.h"
#include "utils/sampling.h"
#include "utils/tuplesort.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
@@ -26,10 +21,6 @@
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define IVFFLAT_MAX_DIM 2000
/* Support functions */
@@ -37,7 +28,6 @@ typedef Pointer Item;
#define IVFFLAT_NORM_PROC 2
#define IVFFLAT_KMEANS_DISTANCE_PROC 3
#define IVFFLAT_KMEANS_NORM_PROC 4
#define IVFFLAT_TYPE_INFO_PROC 5
#define IVFFLAT_VERSION 1
#define IVFFLAT_MAGIC_NUMBER 0x14FF1A7
@@ -59,7 +49,7 @@ typedef Pointer Item;
#define PROGRESS_IVFFLAT_PHASE_ASSIGN 3
#define PROGRESS_IVFFLAT_PHASE_LOAD 4
#define IVFFLAT_LIST_SIZE(size) (offsetof(IvfflatListData, center) + size)
#define IVFFLAT_LIST_SIZE(_dim) (offsetof(IvfflatListData, center) + VECTOR_SIZE(_dim))
#define IvfflatPageGetOpaque(page) ((IvfflatPageOpaque) PageGetSpecialPointer(page))
#define IvfflatPageGetMeta(page) ((IvfflatMetaPageData *) PageGetContents(page))
@@ -82,31 +72,20 @@ typedef Pointer Item;
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#define SeedRandom(seed) srandom(seed)
#endif
/* Variables */
extern int ivfflat_probes;
extern int ivfflat_iterative_scan;
extern int ivfflat_max_probes;
typedef enum IvfflatIterativeScanMode
{
IVFFLAT_ITERATIVE_SCAN_OFF,
IVFFLAT_ITERATIVE_SCAN_RELAXED
} IvfflatIterativeScanMode;
typedef struct VectorArrayData
{
int length;
int maxlen;
int dim;
Size itemsize;
char *items;
Vector *items;
} VectorArrayData;
typedef VectorArrayData * VectorArray;
@@ -165,26 +144,15 @@ typedef struct IvfflatLeader
IvfflatShared *ivfshared;
Sharedsort *sharedsort;
Snapshot snapshot;
char *ivfcenters;
Vector *ivfcenters;
} IvfflatLeader;
typedef struct IvfflatTypeInfo
{
int maxDimensions;
Datum (*normalize) (PG_FUNCTION_ARGS);
Size (*itemSize) (int dimensions);
void (*updateCenter) (Pointer v, int dimensions, float *x);
void (*sumCenter) (Pointer v, float *x);
} IvfflatTypeInfo;
typedef struct IvfflatBuildState
{
/* Info */
Relation heap;
Relation index;
IndexInfo *indexInfo;
const IvfflatTypeInfo *typeInfo;
TupleDesc tupdesc;
/* Settings */
int dimensions;
@@ -204,7 +172,6 @@ typedef struct IvfflatBuildState
VectorArray samples;
VectorArray centers;
ListInfo *listInfo;
Size itemsize;
#ifdef IVFFLAT_KMEANS_DEBUG
double inertia;
@@ -215,16 +182,14 @@ typedef struct IvfflatBuildState
/* Sampling */
BlockSamplerData bs;
ReservoirStateData rstate;
double samplerows;
double rowstoskip;
int rowstoskip;
/* Sorting */
Tuplesortstate *sortstate;
TupleDesc sortdesc;
TupleDesc tupdesc;
TupleTableSlot *slot;
/* Memory */
Size memoryUsed;
MemoryContext tmpCtx;
/* Parallel builds */
@@ -268,69 +233,40 @@ typedef struct IvfflatScanList
typedef struct IvfflatScanOpaqueData
{
const IvfflatTypeInfo *typeInfo;
int probes;
int maxProbes;
int dimensions;
bool first;
Datum value;
MemoryContext tmpCtx;
/* Sorting */
Tuplesortstate *sortstate;
TupleDesc tupdesc;
TupleTableSlot *vslot;
TupleTableSlot *mslot;
BufferAccessStrategy bas;
TupleTableSlot *slot;
bool isnull;
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
Datum (*distfunc) (FmgrInfo *flinfo, Oid collation, Datum arg1, Datum arg2);
/* Lists */
pairingheap *listQueue;
BlockNumber *listPages;
int listIndex;
IvfflatScanList *lists;
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
} IvfflatScanOpaqueData;
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
#define VECTOR_ARRAY_SIZE(_length, _size) (sizeof(VectorArrayData) + (_length) * MAXALIGN(_size))
/* Use functions instead of macros to avoid double evaluation */
static inline Pointer
VectorArrayGet(VectorArray arr, int offset)
{
if (offset >= arr->maxlen)
elog(ERROR, "safety check failed");
return ((char *) arr->items) + (offset * arr->itemsize);
}
static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val)
{
Size size = VARSIZE_ANY(val);
if (size > arr->itemsize)
elog(ERROR, "safety check failed");
memcpy(VectorArrayGet(arr, offset), val, size);
}
#define VECTOR_ARRAY_SIZE(_length, _dim) (sizeof(VectorArrayData) + (_length) * VECTOR_SIZE(_dim))
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * VECTOR_SIZE((_arr)->dim))
#define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset))
#define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE((_arr)->dim))
/* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
VectorArray VectorArrayInit(int maxlen, int dimensions);
void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed);
void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
void IvfflatCheckMemoryUsage(Size totalSize);
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value);
int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
@@ -340,7 +276,6 @@ Buffer IvfflatNewBuffer(Relation index, ForkNumber forkNum);
void IvfflatInitPage(Buffer buf, Page page);
void IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
void IvfflatInit(void);
const IvfflatTypeInfo *IvfflatGetTypeInfo(Relation index);
PGDLLEXPORT void IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -2,16 +2,11 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
/*
* Find the list that minimizes the distance function
@@ -70,9 +65,8 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
* Insert a tuple into the index
*/
static void
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
{
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
IndexTuple itup;
Datum value;
FmgrInfo *normprocinfo;
@@ -91,19 +85,12 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
if (normprocinfo != NULL)
{
Oid collation = index->rd_indcollation[0];
if (!IvfflatCheckNorm(normprocinfo, collation, value))
if (!IvfflatNormValue(normprocinfo, index->rd_indcollation[0], &value))
return;
value = IvfflatNormValue(typeInfo, collation, value);
}
/* Ensure index is valid */
IvfflatGetMetaPageInfo(index, NULL, NULL);
/* Find the insert page - sets the page and list info */
FindInsertPage(index, &value, &insertPage, &listInfo);
FindInsertPage(index, values, &insertPage, &listInfo);
Assert(BlockNumberIsValid(insertPage));
originalInsertPage = insertPage;
@@ -209,7 +196,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
oldCtx = MemoryContextSwitchTo(insertCtx);
/* Insert tuple */
InsertTuple(index, values, isnull, heap_tid);
InsertTuple(index, values, isnull, heap_tid, heap);
/* Delete memory context */
MemoryContextSwitchTo(oldCtx);

View File

@@ -1,18 +1,13 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/genam.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#ifdef IVFFLAT_MEMORY
#include "utils/memutils.h"
#endif
/*
@@ -51,12 +46,12 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
for (j = 0; j < numSamples; j++)
{
Datum vec = PointerGetDatum(VectorArrayGet(samples, j));
Vector *vec = VectorArrayGet(samples, j);
double distance;
/* Only need to compute distance for new center */
/* TODO Use triangle inequality to reduce distance calculations */
distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, vec, PointerGetDatum(VectorArrayGet(centers, i))));
distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, i))));
/* Set lower bound */
lowerBound[j * numCenters + i] = distance;
@@ -91,45 +86,73 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
}
/*
* Norm centers
* Apply norm to vector
*/
static void
NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers)
static inline void
ApplyNorm(FmgrInfo *normprocinfo, Oid collation, Vector * vec)
{
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context",
ALLOCSET_DEFAULT_SIZES);
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(vec)));
IvfflatNormVectors(typeInfo, collation, centers, normCtx);
MemoryContextDelete(normCtx);
/* TODO Handle zero norm */
if (norm > 0)
{
for (int i = 0; i < vec->dim; i++)
vec->x[i] /= norm;
}
}
/*
* Quick approach if we have no data
* Compare vectors
*/
static int
CompareVectors(const void *a, const void *b)
{
return vector_cmp_internal((Vector *) a, (Vector *) b);
}
/*
* Quick approach if we have little data
*/
static void
RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
QuickCenters(Relation index, VectorArray samples, VectorArray centers)
{
int dimensions = centers->dim;
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
Oid collation = index->rd_indcollation[0];
float *x = (float *) palloc(sizeof(float) * dimensions);
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
/* Fill with random data */
/* Copy existing vectors while avoiding duplicates */
if (samples->length > 0)
{
qsort(samples->items, samples->length, VECTOR_SIZE(samples->dim), CompareVectors);
for (int i = 0; i < samples->length; i++)
{
Vector *vec = VectorArrayGet(samples, i);
if (i == 0 || CompareVectors(vec, VectorArrayGet(samples, i - 1)) != 0)
{
VectorArraySet(centers, centers->length, vec);
centers->length++;
}
}
}
/* Fill remaining with random data */
while (centers->length < centers->maxlen)
{
Pointer center = VectorArrayGet(centers, centers->length);
Vector *vec = VectorArrayGet(centers, centers->length);
for (int i = 0; i < dimensions; i++)
x[i] = (float) RandomDouble();
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
typeInfo->updateCenter(center, dimensions, x);
for (int j = 0; j < dimensions; j++)
vec->x[j] = RandomDouble();
/* Normalize if needed (only needed for random centers) */
if (normprocinfo != NULL)
ApplyNorm(normprocinfo, collation, vec);
centers->length++;
}
if (normprocinfo != NULL)
NormCenters(typeInfo, collation, centers);
}
#ifdef IVFFLAT_MEMORY
@@ -137,104 +160,18 @@ RandomCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeI
* Show memory usage
*/
static void
ShowMemoryUsage(MemoryContext context, Size estimatedSize)
ShowMemoryUsage(Size estimatedSize)
{
#if PG_VERSION_NUM >= 130000
elog(INFO, "total memory: %zu MB",
MemoryContextMemAllocated(context, true) / (1024 * 1024));
MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
#else
MemoryContextStats(CurrentMemoryContext);
#endif
elog(INFO, "estimated memory: %zu MB", estimatedSize / (1024 * 1024));
}
#endif
/*
* Sum centers
*/
static void
SumCenters(VectorArray samples, float *agg, int *closestCenters, const IvfflatTypeInfo * typeInfo)
{
for (int j = 0; j < samples->length; j++)
{
float *x = agg + ((int64) closestCenters[j] * samples->dim);
typeInfo->sumCenter(VectorArrayGet(samples, j), x);
}
}
/*
* Update centers
*/
static void
UpdateCenters(float *agg, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{
for (int j = 0; j < centers->length; j++)
{
float *x = agg + ((int64) j * centers->dim);
typeInfo->updateCenter(VectorArrayGet(centers, j), centers->dim, x);
}
}
/*
* Compute new centers
*/
static void
ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *centerCounts, int *closestCenters, FmgrInfo *normprocinfo, Oid collation, const IvfflatTypeInfo * typeInfo)
{
int dimensions = newCenters->dim;
int numCenters = newCenters->length;
int numSamples = samples->length;
/* Reset sum and count */
for (int j = 0; j < numCenters; j++)
{
float *x = agg + ((int64) j * dimensions);
for (int k = 0; k < dimensions; k++)
x[k] = 0.0;
centerCounts[j] = 0;
}
/* Increment sum of closest center */
SumCenters(samples, agg, closestCenters, typeInfo);
/* Increment count of closest center */
for (int j = 0; j < numSamples; j++)
centerCounts[closestCenters[j]] += 1;
/* Divide sum by count */
for (int j = 0; j < numCenters; j++)
{
float *x = agg + ((int64) j * dimensions);
if (centerCounts[j] > 0)
{
/* Double avoids overflow, but requires more memory */
/* TODO Update bounds */
for (int k = 0; k < dimensions; k++)
{
if (isinf(x[k]))
x[k] = x[k] > 0 ? FLT_MAX : -FLT_MAX;
}
for (int k = 0; k < dimensions; k++)
x[k] /= centerCounts[j];
}
else
{
/* TODO Handle empty centers properly */
for (int k = 0; k < dimensions; k++)
x[k] = RandomDouble();
}
}
/* Set new centers */
UpdateCenters(agg, newCenters, typeInfo);
/* Normalize if needed */
if (normprocinfo != NULL)
NormCenters(typeInfo, collation, newCenters);
}
/*
* Use Elkan for performance. This requires distance function to satisfy triangle inequality.
*
@@ -244,16 +181,19 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/
static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
{
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
Vector *vec;
Vector *newCenter;
int64 j;
int64 k;
int dimensions = centers->dim;
int numCenters = centers->maxlen;
int numSamples = samples->length;
VectorArray newCenters;
float *agg;
int *centerCounts;
int *closestCenters;
float *lowerBound;
@@ -263,8 +203,9 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *newcdist;
/* Calculate allocation sizes */
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->dim);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->dim);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, dimensions);
Size centerCountsSize = sizeof(int) * numCenters;
Size closestCentersSize = sizeof(int) * numSamples;
Size lowerBoundSize = sizeof(float) * numSamples * numCenters;
@@ -274,13 +215,18 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */
Size totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
Size totalSize = samplesSize + centersSize + newCentersSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */
IvfflatCheckMemoryUsage(totalSize);
/* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
/* Ensure indexing does not overflow */
if (numCenters > INT_MAX / numCenters)
if (numCenters * numCenters > INT_MAX)
elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */
@@ -290,7 +236,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
/* Allocate space */
/* Use float instead of double to save memory */
agg = palloc(aggSize);
centerCounts = palloc(centerCountsSize);
closestCenters = palloc(closestCentersSize);
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
@@ -299,25 +244,29 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
newcdist = palloc(newcdistSize);
/* Initialize new centers */
newCenters = VectorArrayInit(numCenters, dimensions, centers->itemsize);
newCenters->length = numCenters;
newCenters = VectorArrayInit(numCenters, dimensions);
for (j = 0; j < numCenters; j++)
{
vec = VectorArrayGet(newCenters, j);
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
}
#ifdef IVFFLAT_MEMORY
ShowMemoryUsage(MemoryContextGetParent(CurrentMemoryContext), totalSize);
ShowMemoryUsage(totalSize);
#endif
/* Pick initial centers */
InitCenters(index, samples, centers, lowerBound);
/* Assign each x to its closest initial center c(x) = argmin d(x,c) */
for (int64 j = 0; j < numSamples; j++)
for (j = 0; j < numSamples; j++)
{
float minDistance = FLT_MAX;
int closestCenter = 0;
/* Find closest center */
for (int64 k = 0; k < numCenters; k++)
for (k = 0; k < numCenters; k++)
{
/* TODO Use Lemma 1 in k-means++ initialization */
float distance = lowerBound[j * numCenters + k];
@@ -343,13 +292,13 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
CHECK_FOR_INTERRUPTS();
/* Step 1: For all centers, compute distance */
for (int64 j = 0; j < numCenters; j++)
for (j = 0; j < numCenters; j++)
{
Datum vec = PointerGetDatum(VectorArrayGet(centers, j));
vec = VectorArrayGet(centers, j);
for (int64 k = j + 1; k < numCenters; k++)
for (k = j + 1; k < numCenters; k++)
{
float distance = 0.5 * DatumGetFloat8(FunctionCall2Coll(procinfo, collation, vec, PointerGetDatum(VectorArrayGet(centers, k))));
float distance = 0.5 * DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
halfcdist[j * numCenters + k] = distance;
halfcdist[k * numCenters + j] = distance;
@@ -357,11 +306,11 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
}
/* For all centers c, compute s(c) */
for (int64 j = 0; j < numCenters; j++)
for (j = 0; j < numCenters; j++)
{
float minDistance = FLT_MAX;
for (int64 k = 0; k < numCenters; k++)
for (k = 0; k < numCenters; k++)
{
float distance;
@@ -378,7 +327,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
rjreset = iteration != 0;
for (int64 j = 0; j < numSamples; j++)
for (j = 0; j < numSamples; j++)
{
bool rj;
@@ -388,9 +337,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
rj = rjreset;
for (int64 k = 0; k < numCenters; k++)
for (k = 0; k < numCenters; k++)
{
Datum vec;
float dxcx;
/* Step 3: For all remaining points x and centers c */
@@ -403,12 +351,12 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
if (upperBound[j] <= halfcdist[closestCenters[j] * numCenters + k])
continue;
vec = PointerGetDatum(VectorArrayGet(samples, j));
vec = VectorArrayGet(samples, j);
/* Step 3a */
if (rj)
{
dxcx = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, vec, PointerGetDatum(VectorArrayGet(centers, closestCenters[j]))));
dxcx = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, closestCenters[j]))));
/* d(x,c(x)) computed, which is a form of d(x,c) */
lowerBound[j * numCenters + closestCenters[j]] = dxcx;
@@ -422,7 +370,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
/* Step 3b */
if (dxcx > lowerBound[j * numCenters + k] || dxcx > halfcdist[closestCenters[j] * numCenters + k])
{
float dxc = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, vec, PointerGetDatum(VectorArrayGet(centers, k))));
float dxc = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(vec), PointerGetDatum(VectorArrayGet(centers, k))));
/* d(x,c) calculated */
lowerBound[j * numCenters + k] = dxc;
@@ -441,15 +389,66 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
}
/* Step 4: For each center c, let m(c) be mean of all points assigned */
ComputeNewCenters(samples, agg, newCenters, centerCounts, closestCenters, normprocinfo, collation, typeInfo);
for (j = 0; j < numCenters; j++)
{
vec = VectorArrayGet(newCenters, j);
for (k = 0; k < dimensions; k++)
vec->x[k] = 0.0;
centerCounts[j] = 0;
}
for (j = 0; j < numSamples; j++)
{
int closestCenter;
vec = VectorArrayGet(samples, j);
closestCenter = closestCenters[j];
/* Increment sum and count of closest center */
newCenter = VectorArrayGet(newCenters, closestCenter);
for (k = 0; k < dimensions; k++)
newCenter->x[k] += vec->x[k];
centerCounts[closestCenter] += 1;
}
for (j = 0; j < numCenters; j++)
{
vec = VectorArrayGet(newCenters, j);
if (centerCounts[j] > 0)
{
/* Double avoids overflow, but requires more memory */
/* TODO Update bounds */
for (k = 0; k < dimensions; k++)
{
if (isinf(vec->x[k]))
vec->x[k] = vec->x[k] > 0 ? FLT_MAX : -FLT_MAX;
}
for (k = 0; k < dimensions; k++)
vec->x[k] /= centerCounts[j];
}
else
{
/* TODO Handle empty centers properly */
for (k = 0; k < dimensions; k++)
vec->x[k] = RandomDouble();
}
/* Normalize if needed */
if (normprocinfo != NULL)
ApplyNorm(normprocinfo, collation, vec);
}
/* Step 5 */
for (int j = 0; j < numCenters; j++)
for (j = 0; j < numCenters; j++)
newcdist[j] = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(VectorArrayGet(centers, j)), PointerGetDatum(VectorArrayGet(newCenters, j))));
for (int64 j = 0; j < numSamples; j++)
for (j = 0; j < numSamples; j++)
{
for (int64 k = 0; k < numCenters; k++)
for (k = 0; k < numCenters; k++)
{
float distance = lowerBound[j * numCenters + k] - newcdist[k];
@@ -462,78 +461,77 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
/* Step 6 */
/* We reset r(x) before Step 3 in the next iteration */
for (int j = 0; j < numSamples; j++)
for (j = 0; j < numSamples; j++)
upperBound[j] += newcdist[closestCenters[j]];
/* Step 7 */
for (int j = 0; j < numCenters; j++)
for (j = 0; j < numCenters; j++)
VectorArraySet(centers, j, VectorArrayGet(newCenters, j));
if (changes == 0 && iteration != 0)
break;
}
}
/*
* Ensure no NaN or infinite values
*/
static void
CheckElements(VectorArray centers, const IvfflatTypeInfo * typeInfo)
{
float *scratch = palloc(sizeof(float) * centers->dim);
for (int i = 0; i < centers->length; i++)
{
for (int j = 0; j < centers->dim; j++)
scratch[j] = 0;
/* /fp:fast may not propagate NaN with MSVC, but that's alright */
typeInfo->sumCenter(VectorArrayGet(centers, i), scratch);
for (int j = 0; j < centers->dim; j++)
{
if (isnan(scratch[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (isinf(scratch[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
}
/*
* Ensure no zero vectors for cosine distance
*/
static void
CheckNorms(VectorArray centers, Relation index)
{
/* Check NORM_PROC instead of KMEANS_NORM_PROC */
FmgrInfo *normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
Oid collation = index->rd_indcollation[0];
if (normprocinfo == NULL)
return;
for (int i = 0; i < centers->length; i++)
{
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
if (norm == 0)
elog(ERROR, "Zero norm detected. Please report a bug.");
}
VectorArrayFree(newCenters);
pfree(centerCounts);
pfree(closestCenters);
pfree(lowerBound);
pfree(upperBound);
pfree(s);
pfree(halfcdist);
pfree(newcdist);
}
/*
* Detect issues with centers
*/
static void
CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeInfo)
CheckCenters(Relation index, VectorArray centers)
{
FmgrInfo *normprocinfo;
if (centers->length != centers->maxlen)
elog(ERROR, "Not enough centers. Please report a bug.");
CheckElements(centers, typeInfo);
CheckNorms(centers, index);
/* Ensure no NaN or infinite values */
for (int i = 0; i < centers->length; i++)
{
Vector *vec = VectorArrayGet(centers, i);
for (int j = 0; j < vec->dim; j++)
{
if (isnan(vec->x[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (isinf(vec->x[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
/* Ensure no duplicate centers */
/* Fine to sort in-place */
qsort(centers->items, centers->length, VECTOR_SIZE(centers->dim), CompareVectors);
for (int i = 1; i < centers->length; i++)
{
if (CompareVectors(VectorArrayGet(centers, i), VectorArrayGet(centers, i - 1)) == 0)
elog(ERROR, "Duplicate centers detected. Please report a bug.");
}
/* Ensure no zero vectors for cosine distance */
/* Check NORM_PROC instead of KMEANS_NORM_PROC */
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
if (normprocinfo != NULL)
{
Oid collation = index->rd_indcollation[0];
for (int i = 0; i < centers->length; i++)
{
double norm = DatumGetFloat8(FunctionCall1Coll(normprocinfo, collation, PointerGetDatum(VectorArrayGet(centers, i))));
if (norm == 0)
elog(ERROR, "Zero norm detected. Please report a bug.");
}
}
}
/*
@@ -541,20 +539,12 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine
*/
void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers)
{
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context",
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(kmeansCtx);
if (samples->length == 0)
RandomCenters(index, centers, typeInfo);
if (samples->length <= centers->maxlen)
QuickCenters(index, samples, centers);
else
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
ElkanKmeans(index, samples, centers);
CheckCenters(index, centers, typeInfo);
MemoryContextSwitchTo(oldCtx);
MemoryContextDelete(kmeansCtx);
CheckCenters(index, centers);
}

View File

@@ -2,29 +2,14 @@
#include <float.h>
#include "access/genam.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/tupdesc.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "fmgr.h"
#include "lib/pairingheap.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
/*
* Compare list distances
@@ -32,10 +17,10 @@
static int
CompareLists(const pairingheap_node *a, const pairingheap_node *b, void *arg)
{
if (GetScanListConst(a)->distance > GetScanListConst(b)->distance)
if (((const IvfflatScanList *) a)->distance > ((const IvfflatScanList *) b)->distance)
return 1;
if (GetScanListConst(a)->distance < GetScanListConst(b)->distance)
if (((const IvfflatScanList *) a)->distance < ((const IvfflatScanList *) b)->distance)
return -1;
return 0;
@@ -71,9 +56,9 @@ GetScanLists(IndexScanDesc scan, Datum value)
double distance;
/* Use procinfo from the index instead of scan key for performance */
distance = DatumGetFloat8(so->distfunc(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
if (listCount < so->maxProbes)
if (listCount < so->probes)
{
IvfflatScanList *scanlist;
@@ -86,15 +71,15 @@ GetScanLists(IndexScanDesc scan, Datum value)
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Calculate max distance */
if (listCount == so->maxProbes)
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
if (listCount == so->probes)
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
else if (distance < maxDistance)
{
IvfflatScanList *scanlist;
/* Remove */
scanlist = GetScanList(pairingheap_remove_first(so->listQueue));
scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
/* Reuse */
scanlist->startPage = list->startPage;
@@ -102,7 +87,7 @@ GetScanLists(IndexScanDesc scan, Datum value)
pairingheap_add(so->listQueue, &scanlist->ph_node);
/* Update max distance */
maxDistance = GetScanList(pairingheap_first(so->listQueue))->distance;
maxDistance = ((IvfflatScanList *) pairingheap_first(so->listQueue))->distance;
}
}
@@ -110,11 +95,6 @@ GetScanLists(IndexScanDesc scan, Datum value)
UnlockReleaseBuffer(cbuf);
}
for (int i = listCount - 1; i >= 0; i--)
so->listPages[i] = GetScanList(pairingheap_remove_first(so->listQueue))->startPage;
Assert(pairingheap_is_empty(so->listQueue));
}
/*
@@ -125,15 +105,20 @@ GetScanItems(IndexScanDesc scan, Datum value)
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
TupleDesc tupdesc = RelationGetDescr(scan->indexRelation);
TupleTableSlot *slot = so->vslot;
int batchProbes = 0;
double tuples = 0;
TupleTableSlot *slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
tuplesort_reset(so->sortstate);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
/* Search closest probes lists */
while (so->listIndex < so->maxProbes && (++batchProbes) <= so->probes)
while (!pairingheap_is_empty(so->listQueue))
{
BlockNumber searchPage = so->listPages[so->listIndex++];
BlockNumber searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
/* Search all entry pages for list */
while (BlockNumberIsValid(searchPage))
@@ -142,7 +127,7 @@ GetScanItems(IndexScanDesc scan, Datum value)
Page page;
OffsetNumber maxoffno;
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, so->bas);
buf = ReadBufferExtended(scan->indexRelation, MAIN_FORKNUM, searchPage, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
@@ -164,13 +149,15 @@ GetScanItems(IndexScanDesc scan, Datum value)
* performance
*/
ExecClearTuple(slot);
slot->tts_values[0] = so->distfunc(so->procinfo, so->collation, datum, value);
slot->tts_values[0] = FunctionCall2Coll(so->procinfo, so->collation, datum, value);
slot->tts_isnull[0] = false;
slot->tts_values[1] = PointerGetDatum(&itup->t_tid);
slot->tts_isnull[1] = false;
ExecStoreVirtualTuple(slot);
tuplesort_puttupleslot(so->sortstate, slot);
tuples++;
}
searchPage = IvfflatPageGetOpaque(page)->nextblkno;
@@ -179,71 +166,15 @@ GetScanItems(IndexScanDesc scan, Datum value)
}
}
FreeAccessStrategy(bas);
if (tuples < 100)
ereport(DEBUG1,
(errmsg("index scan found few tuples"),
errdetail("Index may have been created with little data."),
errhint("Recreate the index and possibly decrease lists.")));
tuplesort_performsort(so->sortstate);
#if defined(IVFFLAT_MEMORY)
elog(INFO, "memory: %zu MB", MemoryContextMemAllocated(CurrentMemoryContext, true) / (1024 * 1024));
#endif
}
/*
* Zero distance
*/
static Datum
ZeroDistance(FmgrInfo *flinfo, Oid collation, Datum arg1, Datum arg2)
{
return Float8GetDatum(0.0);
}
/*
* Get scan value
*/
static Datum
GetScanValue(IndexScanDesc scan)
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
Datum value;
if (scan->orderByData->sk_flags & SK_ISNULL)
{
value = PointerGetDatum(NULL);
so->distfunc = ZeroDistance;
}
else
{
value = scan->orderByData->sk_argument;
so->distfunc = FunctionCall2Coll;
/* Value should not be compressed or toasted */
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Normalize if needed */
if (so->normprocinfo != NULL)
{
MemoryContext oldCtx = MemoryContextSwitchTo(so->tmpCtx);
value = IvfflatNormValue(so->typeInfo, so->collation, value);
MemoryContextSwitchTo(oldCtx);
}
}
return value;
}
/*
* Initialize scan sort state
*/
static Tuplesortstate *
InitScanSortState(TupleDesc tupdesc)
{
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
return tuplesort_begin_heap(tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
}
/*
@@ -256,31 +187,23 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
IvfflatScanOpaque so;
int lists;
int dimensions;
AttrNumber attNums[] = {1};
Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false};
int probes = ivfflat_probes;
int maxProbes;
MemoryContext oldCtx;
scan = RelationGetIndexScan(index, nkeys, norderbys);
/* Get lists and dimensions from metapage */
IvfflatGetMetaPageInfo(index, &lists, &dimensions);
if (ivfflat_iterative_scan != IVFFLAT_ITERATIVE_SCAN_OFF)
maxProbes = Max(ivfflat_max_probes, probes);
else
maxProbes = probes;
if (probes > lists)
probes = lists;
if (maxProbes > lists)
maxProbes = lists;
so = (IvfflatScanOpaque) palloc(sizeof(IvfflatScanOpaqueData));
so->typeInfo = IvfflatGetTypeInfo(index);
so = (IvfflatScanOpaque) palloc(offsetof(IvfflatScanOpaqueData, lists) + probes * sizeof(IvfflatScanList));
so->first = true;
so->probes = probes;
so->maxProbes = maxProbes;
so->dimensions = dimensions;
/* Set support functions */
@@ -288,40 +211,17 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
so->collation = index->rd_indcollation[0];
so->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat scan temporary context",
ALLOCSET_DEFAULT_SIZES);
oldCtx = MemoryContextSwitchTo(so->tmpCtx);
/* Create tuple description for sorting */
so->tupdesc = CreateTemplateTupleDesc(2);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(so->tupdesc);
#endif
/* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc);
so->sortstate = tuplesort_begin_heap(so->tupdesc, 1, attNums, sortOperators, sortCollations, nullsFirstFlags, work_mem, NULL, false);
/* Need separate slots for puttuple and gettuple */
so->vslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsVirtual);
so->mslot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
/*
* Reuse same set of shared buffers for scan
*
* See postgres/src/backend/storage/buffer/README for description
*/
so->bas = GetAccessStrategy(BAS_BULKREAD);
so->slot = MakeSingleTupleTableSlot(so->tupdesc, &TTSOpsMinimalTuple);
so->listQueue = pairingheap_allocate(CompareLists, scan);
so->listPages = palloc(maxProbes * sizeof(BlockNumber));
so->listIndex = 0;
so->lists = palloc(maxProbes * sizeof(IvfflatScanList));
MemoryContextSwitchTo(oldCtx);
scan->opaque = so;
@@ -336,9 +236,13 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
#if PG_VERSION_NUM >= 130000
if (!so->first)
tuplesort_reset(so->sortstate);
#endif
so->first = true;
pairingheap_reset(so->listQueue);
so->listIndex = 0;
if (keys && scan->numberOfKeys > 0)
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
@@ -354,8 +258,6 @@ bool
ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
ItemPointer heaptid;
bool isnull;
/*
* Index can be used to scan backward, but Postgres doesn't support
@@ -369,10 +271,6 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)
@@ -383,27 +281,41 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
if (!IsMVCCSnapshot(scan->xs_snapshot))
elog(ERROR, "non-MVCC snapshots are not supported with ivfflat");
value = GetScanValue(scan);
if (scan->orderByData->sk_flags & SK_ISNULL)
value = PointerGetDatum(InitVector(so->dimensions));
else
{
value = scan->orderByData->sk_argument;
/* Value should not be compressed or toasted */
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Fine if normalization fails */
if (so->normprocinfo != NULL)
IvfflatNormValue(so->normprocinfo, so->collation, &value);
}
IvfflatBench("GetScanLists", GetScanLists(scan, value));
IvfflatBench("GetScanItems", GetScanItems(scan, value));
so->first = false;
so->value = value;
/* Clean up if we allocated a new value */
if (value != scan->orderByData->sk_argument)
pfree(DatumGetPointer(value));
}
while (!tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
if (tuplesort_gettupleslot(so->sortstate, true, false, so->slot, NULL))
{
if (so->listIndex == so->maxProbes)
return false;
ItemPointer heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->slot, 2, &so->isnull));
IvfflatBench("GetScanItems", GetScanItems(scan, so->value));
scan->xs_heaptid = *heaptid;
scan->xs_recheck = false;
scan->xs_recheckorderby = false;
return true;
}
heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
scan->xs_heaptid = *heaptid;
scan->xs_recheck = false;
scan->xs_recheckorderby = false;
return true;
return false;
}
/*
@@ -414,11 +326,9 @@ ivfflatendscan(IndexScanDesc scan)
{
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
/* Free any temporary files */
pairingheap_free(so->listQueue);
tuplesort_end(so->sortstate);
MemoryContextDelete(so->tmpCtx);
pfree(so);
scan->opaque = NULL;
}

View File

@@ -1,38 +1,22 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate a vector array
*/
VectorArray
VectorArrayInit(int maxlen, int dimensions, Size itemsize)
VectorArrayInit(int maxlen, int dimensions)
{
VectorArray res = palloc(sizeof(VectorArrayData));
/* Ensure items are aligned to prevent UB */
itemsize = MAXALIGN(itemsize);
res->length = 0;
res->maxlen = maxlen;
res->dim = dimensions;
res->itemsize = itemsize;
res->items = palloc_extended(maxlen * itemsize, MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
res->items = palloc_extended(maxlen * VECTOR_SIZE(dimensions), MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
return res;
}
@@ -46,6 +30,16 @@ VectorArrayFree(VectorArray arr)
pfree(arr);
}
/*
* Print vector array - useful for debugging
*/
void
PrintVectorArray(char *msg, VectorArray arr)
{
for (int i = 0; i < arr->length; i++)
PrintVector(msg, VectorArrayGet(arr, i));
}
/*
* Get the number of lists in the index
*/
@@ -73,55 +67,32 @@ IvfflatOptionalProcInfo(Relation index, uint16 procnum)
}
/*
* Normalize value
*/
Datum
IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value)
{
return DirectFunctionCall1Coll(typeInfo->normalize, collation, value);
}
/*
* Check if non-zero norm
* Divide by the norm
*
* Returns false if value should not be indexed
*
* The caller needs to free the pointer stored in value
* if it's different than the original value
*/
bool
IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value)
{
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
}
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
/*
* Normalize vectors
*/
void
IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
{
MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
for (int i = 0; i < arr->length; i++)
if (norm > 0)
{
Datum value = PointerGetDatum(VectorArrayGet(arr, i));
Datum newValue = IvfflatNormValue(typeInfo, collation, value);
Vector *v = DatumGetVector(*value);
Vector *result = InitVector(v->dim);
VectorArraySet(arr, i, DatumGetPointer(newValue));
MemoryContextReset(tmpCtx);
for (int i = 0; i < v->dim; i++)
result->x[i] = v->x[i] / norm;
*value = PointerGetDatum(result);
return true;
}
MemoryContextSwitchTo(oldCtx);
}
/*
* Check memory usage
*/
void
IvfflatCheckMemoryUsage(Size totalSize)
{
/* Add one to error message to ceil */
if (totalSize > maintenance_work_mem * (Size) 1024)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
return false;
}
/*
@@ -212,11 +183,7 @@ IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions)
page = BufferGetPage(buf);
metap = IvfflatPageGetMeta(page);
if (unlikely(metap->magicNumber != IVFFLAT_MAGIC_NUMBER))
elog(ERROR, "ivfflat index is not valid");
if (lists != NULL)
*lists = metap->lists;
*lists = metap->lists;
if (dimensions != NULL)
*dimensions = metap->dimensions;
@@ -270,150 +237,3 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
UnlockReleaseBuffer(buf);
}
}
PGDLLEXPORT Datum l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum halfvec_l2_normalize(PG_FUNCTION_ARGS);
PGDLLEXPORT Datum sparsevec_l2_normalize(PG_FUNCTION_ARGS);
static Size
VectorItemSize(int dimensions)
{
return VECTOR_SIZE(dimensions);
}
static Size
HalfvecItemSize(int dimensions)
{
return HALFVEC_SIZE(dimensions);
}
static Size
BitItemSize(int dimensions)
{
return VARBITTOTALLEN(dimensions);
}
static void
VectorUpdateCenter(Pointer v, int dimensions, float *x)
{
Vector *vec = (Vector *) v;
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int i = 0; i < dimensions; i++)
vec->x[i] = x[i];
}
static void
HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
{
HalfVector *vec = (HalfVector *) v;
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int i = 0; i < dimensions; i++)
vec->x[i] = Float4ToHalfUnchecked(x[i]);
}
static void
BitUpdateCenter(Pointer v, int dimensions, float *x)
{
VarBit *vec = (VarBit *) v;
unsigned char *nx = VARBITS(vec);
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (uint32 i = 0; i < VARBITBYTES(vec); i++)
nx[i] = 0;
for (int i = 0; i < dimensions; i++)
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
}
static void
VectorSumCenter(Pointer v, float *x)
{
Vector *vec = (Vector *) v;
int dim = vec->dim;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
x[i] += vec->x[i];
}
static void
HalfvecSumCenter(Pointer v, float *x)
{
HalfVector *vec = (HalfVector *) v;
int dim = vec->dim;
/* Auto-vectorized on aarch64 */
for (int i = 0; i < dim; i++)
x[i] += HalfToFloat4(vec->x[i]);
}
static void
BitSumCenter(Pointer v, float *x)
{
VarBit *vec = (VarBit *) v;
for (int i = 0; i < VARBITLEN(vec); i++)
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
}
/*
* Get type info
*/
const IvfflatTypeInfo *
IvfflatGetTypeInfo(Relation index)
{
FmgrInfo *procinfo = IvfflatOptionalProcInfo(index, IVFFLAT_TYPE_INFO_PROC);
if (procinfo == NULL)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM,
.normalize = l2_normalize,
.itemSize = VectorItemSize,
.updateCenter = VectorUpdateCenter,
.sumCenter = VectorSumCenter
};
return (&typeInfo);
}
else
return (const IvfflatTypeInfo *) DatumGetPointer(FunctionCall0Coll(procinfo, InvalidOid));
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_halfvec_support);
Datum
ivfflat_halfvec_support(PG_FUNCTION_ARGS)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM * 2,
.normalize = halfvec_l2_normalize,
.itemSize = HalfvecItemSize,
.updateCenter = HalfvecUpdateCenter,
.sumCenter = HalfvecSumCenter
};
PG_RETURN_POINTER(&typeInfo);
}
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflat_bit_support);
Datum
ivfflat_bit_support(PG_FUNCTION_ARGS)
{
static const IvfflatTypeInfo typeInfo = {
.maxDimensions = IVFFLAT_MAX_DIM * 32,
.normalize = NULL,
.itemSize = BitItemSize,
.updateCenter = BitUpdateCenter,
.sumCenter = BitSumCenter
};
PG_RETURN_POINTER(&typeInfo);
}

View File

@@ -1,16 +1,9 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "commands/vacuum.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/*
* Bulk delete tuples from the index
@@ -33,7 +26,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Page cpage;
OffsetNumber coffno;
OffsetNumber cmaxoffno;
BlockNumber listPages[MaxOffsetNumber];
BlockNumber startPages[MaxOffsetNumber];
ListInfo listInfo;
cbuf = ReadBuffer(index, blkno);
@@ -47,7 +40,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
{
IvfflatList list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, coffno));
listPages[coffno - FirstOffsetNumber] = list->startPage;
startPages[coffno - FirstOffsetNumber] = list->startPage;
}
listInfo.blkno = blkno;
@@ -57,7 +50,7 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
for (coffno = FirstOffsetNumber; coffno <= cmaxoffno; coffno = OffsetNumberNext(coffno))
{
BlockNumber searchPage = listPages[coffno - FirstOffsetNumber];
BlockNumber searchPage = startPages[coffno - FirstOffsetNumber];
BlockNumber insertPage = InvalidBlockNumber;
/* Iterate over entry pages */

File diff suppressed because it is too large Load Diff

View File

@@ -1,40 +0,0 @@
#ifndef SPARSEVEC_H
#define SPARSEVEC_H
#define SPARSEVEC_MAX_DIM 1000000000
#define SPARSEVEC_MAX_NNZ 16000
#define DatumGetSparseVector(x) ((SparseVector *) PG_DETOAST_DATUM(x))
#define PG_GETARG_SPARSEVEC_P(x) DatumGetSparseVector(PG_GETARG_DATUM(x))
#define PG_RETURN_SPARSEVEC_P(x) PG_RETURN_POINTER(x)
/*
* Indices use 0-based numbering for the on-disk (and binary) format (consistent with C)
* and are always sorted. Values come after indices.
*/
typedef struct SparseVector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int32 dim; /* number of dimensions */
int32 nnz; /* number of non-zero elements */
int32 unused; /* reserved for future use, always zero */
int32 indices[FLEXIBLE_ARRAY_MEMBER];
} SparseVector;
/* Use functions instead of macros to avoid double evaluation */
static inline Size
SPARSEVEC_SIZE(int nnz)
{
return offsetof(SparseVector, indices) + (nnz * sizeof(int32)) + (nnz * sizeof(float));
}
static inline float *
SPARSEVEC_VALUES(SparseVector * x)
{
return (float *) (((char *) x) + offsetof(SparseVector, indices) + (x->nnz * sizeof(int32)));
}
SparseVector *InitSparseVector(int dim, int nnz);
#endif

View File

@@ -2,48 +2,34 @@
#include <math.h>
#include "bitutils.h"
#include "bitvec.h"
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "hnsw.h"
#include "ivfflat.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/varbit.h"
#include "utils/numeric.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#if PG_VERSION_NUM < 130000
#define TYPALIGN_DOUBLE 'd'
#define TYPALIGN_INT 'i'
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
#if defined(USE_TARGET_CLONES) && !defined(__FMA__)
#define VECTOR_TARGET_CLONES __attribute__((target_clones("default", "fma")))
#else
#define VECTOR_TARGET_CLONES
#endif
#if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.4");
#else
PG_MODULE_MAGIC;
#endif
/*
* Initialize index options and variables
@@ -52,8 +38,6 @@ PGDLLEXPORT void _PG_init(void);
void
_PG_init(void)
{
BitvecInit();
HalfvecInit();
HnswInit();
IvfflatInit();
}
@@ -133,9 +117,9 @@ InitVector(int dim)
return result;
}
#if PG_VERSION_NUM >= 170000
#define vector_isspace(ch) scanner_isspace(ch)
#else
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
vector_isspace(char ch)
{
@@ -148,7 +132,6 @@ vector_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -164,10 +147,28 @@ CheckStateArray(ArrayType *statearray, const char *caller)
return (float8 *) ARR_DATA_PTR(statearray);
}
#if PG_VERSION_NUM < 120003
static pg_noinline void
float_overflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: overflow")));
}
static pg_noinline void
float_underflow_error(void)
{
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("value out of range: underflow")));
}
#endif
/*
* Convert textual representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
Datum
vector_in(PG_FUNCTION_ARGS)
{
@@ -175,33 +176,27 @@ vector_in(PG_FUNCTION_ARGS)
int32 typmod = PG_GETARG_INT32(2);
float x[VECTOR_MAX_DIM];
int dim = 0;
char *pt = lit;
char *pt;
char *stringEnd;
Vector *result;
char *litcopy = pstrdup(lit);
char *str = litcopy;
while (vector_isspace(*pt))
pt++;
while (vector_isspace(*str))
str++;
if (*pt != '[')
if (*str != '[')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit),
errmsg("malformed vector literal: \"%s\"", lit),
errdetail("Vector contents must start with \"[\".")));
pt++;
str++;
pt = strtok(str, ",");
stringEnd = pt;
while (vector_isspace(*pt))
pt++;
if (*pt == ']')
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
for (;;)
while (pt != NULL && *stringEnd != ']')
{
float val;
char *stringEnd;
if (dim == VECTOR_MAX_DIM)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
@@ -216,55 +211,61 @@ vector_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
errno = 0;
/* Use strtof like float4in to avoid a double-rounding problem */
/* Postgres sets LC_NUMERIC to C on startup */
val = strtof(pt, &stringEnd);
x[dim] = strtof(pt, &stringEnd);
CheckElement(x[dim]);
dim++;
if (stringEnd == pt)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Check for range error like float4in */
if (errno == ERANGE && isinf(val))
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
errmsg("\"%s\" is out of range for type vector", pnstrdup(pt, stringEnd - pt))));
while (vector_isspace(*stringEnd))
stringEnd++;
CheckElement(val);
x[dim++] = val;
pt = stringEnd;
while (vector_isspace(*pt))
pt++;
if (*pt == ',')
pt++;
else if (*pt == ']')
{
pt++;
break;
}
else
if (*stringEnd != '\0' && *stringEnd != ']')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
pt = strtok(NULL, ",");
}
/* Only whitespace is allowed after the closing brace */
while (vector_isspace(*pt))
pt++;
if (*pt != '\0')
if (stringEnd == NULL || *stringEnd != ']')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit),
errmsg("malformed vector literal: \"%s\"", lit),
errdetail("Unexpected end of input.")));
stringEnd++;
/* Only whitespace is allowed after the closing brace */
while (vector_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit),
errdetail("Junk after closing right brace.")));
CheckDim(dim);
/* Ensure no consecutive delimiters since strtok skips */
for (pt = lit + 1; *pt != '\0'; pt++)
{
if (pt[-1] == ',' && *pt == ',')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit)));
}
if (dim < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
pfree(litcopy);
CheckExpectedDim(typmod, dim);
result = InitVector(dim);
@@ -274,13 +275,10 @@ vector_in(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
#define AppendChar(ptr, c) (*(ptr)++ = (c))
#define AppendFloat(ptr, f) ((ptr) += float_to_shortest_decimal_bufn((f), (ptr)))
/*
* Convert internal representation to textual representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_out);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
Datum
vector_out(PG_FUNCTION_ARGS)
{
@@ -288,6 +286,7 @@ vector_out(PG_FUNCTION_ARGS)
int dim = vector->dim;
char *buf;
char *ptr;
int n;
/*
* Need:
@@ -302,17 +301,21 @@ vector_out(PG_FUNCTION_ARGS)
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
ptr = buf;
AppendChar(ptr, '[');
*ptr = '[';
ptr++;
for (int i = 0; i < dim; i++)
{
if (i > 0)
AppendChar(ptr, ',');
{
*ptr = ',';
ptr++;
}
AppendFloat(ptr, vector->x[i]);
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
ptr += n;
}
AppendChar(ptr, ']');
*ptr = ']';
ptr++;
*ptr = '\0';
PG_FREE_IF_COPY(vector, 0);
@@ -334,7 +337,7 @@ PrintVector(char *msg, Vector * vector)
/*
* Convert type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_typmod_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
Datum
vector_typmod_in(PG_FUNCTION_ARGS)
{
@@ -365,7 +368,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
/*
* Convert external binary representation to internal representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_recv);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
Datum
vector_recv(PG_FUNCTION_ARGS)
{
@@ -399,7 +402,7 @@ vector_recv(PG_FUNCTION_ARGS)
/*
* Convert internal representation to the external binary representation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_send);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
Datum
vector_send(PG_FUNCTION_ARGS)
{
@@ -419,7 +422,7 @@ vector_send(PG_FUNCTION_ARGS)
* Convert vector to vector
* This is needed to check the type modifier
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
Datum
vector(PG_FUNCTION_ARGS)
{
@@ -434,7 +437,7 @@ vector(PG_FUNCTION_ARGS)
/*
* Convert array to vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(array_to_vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
Datum
array_to_vector(PG_FUNCTION_ARGS)
{
@@ -508,7 +511,7 @@ array_to_vector(PG_FUNCTION_ARGS)
/*
* Convert vector to float4[]
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_to_float4);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
Datum
vector_to_float4(PG_FUNCTION_ARGS)
{
@@ -529,150 +532,131 @@ vector_to_float4(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Convert half vector to vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(halfvec_to_vector);
Datum
halfvec_to_vector(PG_FUNCTION_ARGS)
{
HalfVector *vec = PG_GETARG_HALFVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
Vector *result;
CheckDim(vec->dim);
CheckExpectedDim(typmod, vec->dim);
result = InitVector(vec->dim);
for (int i = 0; i < vec->dim; i++)
result->x[i] = HalfToFloat4(vec->x[i]);
PG_RETURN_POINTER(result);
}
VECTOR_TARGET_CLONES static float
VectorL2SquaredDistance(int dim, float *ax, float *bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
{
float diff = ax[i] - bx[i];
distance += diff * diff;
}
return distance;
}
/*
* Get the L2 distance between vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
Datum
l2_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
float diff;
CheckDims(a, b);
PG_RETURN_FLOAT8(sqrt((double) VectorL2SquaredDistance(a->dim, a->x, b->x)));
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
{
diff = ax[i] - bx[i];
distance += diff * diff;
}
PG_RETURN_FLOAT8(sqrt((double) distance));
}
/*
* Get the L2 squared distance between vectors
* This saves a sqrt calculation
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
Datum
vector_l2_squared_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
float diff;
CheckDims(a, b);
PG_RETURN_FLOAT8((double) VectorL2SquaredDistance(a->dim, a->x, b->x));
}
VECTOR_TARGET_CLONES static float
VectorInnerProduct(int dim, float *ax, float *bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
distance += ax[i] * bx[i];
for (int i = 0; i < a->dim; i++)
{
diff = ax[i] - bx[i];
distance += diff * diff;
}
return distance;
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the inner product of two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
Datum
inner_product(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
CheckDims(a, b);
PG_RETURN_FLOAT8((double) VectorInnerProduct(a->dim, a->x, b->x));
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i];
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the negative inner product of two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_negative_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
Datum
vector_negative_inner_product(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
CheckDims(a, b);
PG_RETURN_FLOAT8((double) -VectorInnerProduct(a->dim, a->x, b->x));
}
VECTOR_TARGET_CLONES static double
VectorCosineSimilarity(int dim, float *ax, float *bx)
{
float similarity = 0.0;
float norma = 0.0;
float normb = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
{
similarity += ax[i] * bx[i];
norma += ax[i] * ax[i];
normb += bx[i] * bx[i];
}
for (int i = 0; i < a->dim; i++)
distance += ax[i] * bx[i];
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
return (double) similarity / sqrt((double) norma * (double) normb);
PG_RETURN_FLOAT8((double) distance * -1);
}
/*
* Get the cosine distance between two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(cosine_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
Datum
cosine_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
float norma = 0.0;
float normb = 0.0;
double similarity;
CheckDims(a, b);
similarity = VectorCosineSimilarity(a->dim, a->x, b->x);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
{
distance += ax[i] * bx[i];
norma += ax[i] * ax[i];
normb += bx[i] * bx[i];
}
/* Use sqrt(a * b) over sqrt(a) * sqrt(b) */
similarity = (double) distance / sqrt((double) norma * (double) normb);
#ifdef _MSC_VER
/* /fp:fast may not propagate NaN */
@@ -694,17 +678,24 @@ cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm)
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_spherical_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
Datum
vector_spherical_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float dp = 0.0;
double distance;
CheckDims(a, b);
distance = (double) VectorInnerProduct(a->dim, a->x, b->x);
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
dp += ax[i] * bx[i];
distance = (double) dp;
/* Prevent NaN with acos with loss of precision */
if (distance > 1)
@@ -715,38 +706,32 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
PG_RETURN_FLOAT8(acos(distance) / M_PI);
}
/* Does not require FMA, but keep logic simple */
VECTOR_TARGET_CLONES static float
VectorL1Distance(int dim, float *ax, float *bx)
{
float distance = 0.0;
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
distance += fabsf(ax[i] - bx[i]);
return distance;
}
/*
* Get the L1 distance between two vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l1_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l1_distance);
Datum
l1_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
float distance = 0.0;
CheckDims(a, b);
PG_RETURN_FLOAT8((double) VectorL1Distance(a->dim, a->x, b->x));
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
distance += fabsf(ax[i] - bx[i]);
PG_RETURN_FLOAT8((double) distance);
}
/*
* Get the dimensions of a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_dims);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
Datum
vector_dims(PG_FUNCTION_ARGS)
{
@@ -758,7 +743,7 @@ vector_dims(PG_FUNCTION_ARGS)
/*
* Get the L2 norm of a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_norm);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
Datum
vector_norm(PG_FUNCTION_ARGS)
{
@@ -773,49 +758,10 @@ vector_norm(PG_FUNCTION_ARGS)
PG_RETURN_FLOAT8(sqrt(norm));
}
/*
* Normalize a vector with the L2 norm
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(l2_normalize);
Datum
l2_normalize(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
double norm = 0;
Vector *result;
float *rx;
result = InitVector(a->dim);
rx = result->x;
/* Auto-vectorized */
for (int i = 0; i < a->dim; i++)
norm += (double) ax[i] * (double) ax[i];
norm = sqrt(norm);
/* Return zero vector for zero norm */
if (norm > 0)
{
for (int i = 0; i < a->dim; i++)
rx[i] = ax[i] / norm;
/* Check for overflow */
for (int i = 0; i < a->dim; i++)
{
if (isinf(rx[i]))
float_overflow_error();
}
}
PG_RETURN_POINTER(result);
}
/*
* Add vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_add);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
Datum
vector_add(PG_FUNCTION_ARGS)
{
@@ -848,7 +794,7 @@ vector_add(PG_FUNCTION_ARGS)
/*
* Subtract vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_sub);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
Datum
vector_sub(PG_FUNCTION_ARGS)
{
@@ -881,7 +827,7 @@ vector_sub(PG_FUNCTION_ARGS)
/*
* Multiply vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_mul);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_mul);
Datum
vector_mul(PG_FUNCTION_ARGS)
{
@@ -914,110 +860,6 @@ vector_mul(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Concatenate vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_concat);
Datum
vector_concat(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
Vector *result;
int dim = a->dim + b->dim;
CheckDim(dim);
result = InitVector(dim);
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++)
result->x[i] = a->x[i];
/* Auto-vectorized */
for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
result->x[i + start] = b->x[i];
PG_RETURN_POINTER(result);
}
/*
* Quantize a vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(binary_quantize);
Datum
binary_quantize(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
/* Auto-vectorized */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (ax[i + j] > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);
}
/*
* Get a subvector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(subvector);
Datum
subvector(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
int32 start = PG_GETARG_INT32(1);
int32 count = PG_GETARG_INT32(2);
int32 end;
float *ax = a->x;
Vector *result;
int dim;
if (count < 1)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
/*
* Check if (start + count > a->dim), avoiding integer overflow. a->dim
* and count are both positive, so a->dim - count won't overflow.
*/
if (start > a->dim - count)
end = a->dim + 1;
else
end = start + count;
/* Indexing starts at 1, like substring */
if (start < 1)
start = 1;
else if (start > a->dim)
ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension")));
dim = end - start;
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < dim; i++)
result->x[i] = ax[start - 1 + i];
PG_RETURN_POINTER(result);
}
/*
* Internal helper to compare vectors
*/
@@ -1048,85 +890,103 @@ vector_cmp_internal(Vector * a, Vector * b)
/*
* Less than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_lt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
Datum
vector_lt(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) < 0);
}
/*
* Less than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_le);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
Datum
vector_le(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) <= 0);
}
/*
* Equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_eq);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
Datum
vector_eq(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) == 0);
}
/*
* Not equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ne);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
Datum
vector_ne(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) != 0);
}
/*
* Greater than or equal
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_ge);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
Datum
vector_ge(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) >= 0);
}
/*
* Greater than
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_gt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
Datum
vector_gt(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
/* TODO Remove in 0.7.0 */
CheckDims(a, b);
PG_RETURN_BOOL(vector_cmp_internal(a, b) > 0);
}
/*
* Compare vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_cmp);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
Datum
vector_cmp(PG_FUNCTION_ARGS)
{
@@ -1139,7 +999,7 @@ vector_cmp(PG_FUNCTION_ARGS)
/*
* Accumulate vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_accum);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_accum);
Datum
vector_accum(PG_FUNCTION_ARGS)
{
@@ -1198,13 +1058,12 @@ vector_accum(PG_FUNCTION_ARGS)
}
/*
* Combine vectors or half vectors (also used for halfvec_combine)
* Combine vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_combine);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_combine);
Datum
vector_combine(PG_FUNCTION_ARGS)
{
/* Must also update parameters of halfvec_combine if modifying */
ArrayType *statearray1 = PG_GETARG_ARRAYTYPE_P(0);
ArrayType *statearray2 = PG_GETARG_ARRAYTYPE_P(1);
float8 *statevalues1;
@@ -1271,7 +1130,7 @@ vector_combine(PG_FUNCTION_ARGS)
/*
* Average vectors
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(vector_avg);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_avg);
Datum
vector_avg(PG_FUNCTION_ARGS)
{
@@ -1301,26 +1160,3 @@ vector_avg(PG_FUNCTION_ARGS)
PG_RETURN_POINTER(result);
}
/*
* Convert sparse vector to dense vector
*/
FUNCTION_PREFIX PG_FUNCTION_INFO_V1(sparsevec_to_vector);
Datum
sparsevec_to_vector(PG_FUNCTION_ARGS)
{
SparseVector *svec = PG_GETARG_SPARSEVEC_P(0);
int32 typmod = PG_GETARG_INT32(1);
Vector *result;
int dim = svec->dim;
float *values = SPARSEVEC_VALUES(svec);
CheckDim(dim);
CheckExpectedDim(typmod, dim);
result = InitVector(dim);
for (int i = 0; i < svec->nnz; i++)
result->x[svec->indices[i]] = values[i];
PG_RETURN_POINTER(result);
}

View File

@@ -12,7 +12,7 @@ typedef struct Vector
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int16 dim; /* number of dimensions */
int16 unused; /* reserved for future use, always zero */
int16 unused;
float x[FLEXIBLE_ARRAY_MEMBER];
} Vector;
@@ -20,11 +20,4 @@ Vector *InitVector(int dim);
void PrintVector(char *msg, Vector * vector);
int vector_cmp_internal(Vector * a, Vector * b);
/* TODO Move to better place */
#if PG_VERSION_NUM >= 160000
#define FUNCTION_PREFIX
#else
#define FUNCTION_PREFIX PGDLLEXPORT
#endif
#endif

View File

@@ -1,140 +0,0 @@
SELECT hamming_distance('111', '111');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('111', '110');
hamming_distance
------------------
1
(1 row)
SELECT hamming_distance('111', '100');
hamming_distance
------------------
2
(1 row)
SELECT hamming_distance('111', '000');
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance('10101010101010101010', '01010101010101010101');
hamming_distance
------------------
20
(1 row)
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
hamming_distance
------------------
513
(1 row)
SELECT hamming_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
hamming_distance
------------------
2
(1 row)
SELECT hamming_distance('', '');
hamming_distance
------------------
0
(1 row)
SELECT hamming_distance('111', '00');
ERROR: different bit lengths 3 and 2
SELECT hamming_distance('111', '000'::varbit(4));
hamming_distance
------------------
3
(1 row)
SELECT hamming_distance('111', '0000'::varbit(4));
ERROR: different bit lengths 3 and 4
SELECT jaccard_distance('1111', '1111');
jaccard_distance
------------------
0
(1 row)
SELECT jaccard_distance('1111', '1110');
jaccard_distance
------------------
0.25
(1 row)
SELECT jaccard_distance('1111', '1100');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance('1111', '1000');
jaccard_distance
------------------
0.75
(1 row)
SELECT jaccard_distance('1111', '0000');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1100', '1000');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
jaccard_distance
------------------
0
(1 row)
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
jaccard_distance
------------------
0.5
(1 row)
SELECT jaccard_distance('', '');
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '000');
ERROR: different bit lengths 4 and 3
SELECT jaccard_distance('1111', '0000'::varbit(5));
jaccard_distance
------------------
1
(1 row)
SELECT jaccard_distance('1111', '00000'::varbit(5));
ERROR: different bit lengths 4 and 5

View File

@@ -1,5 +1,4 @@
SET enable_seqscan = off;
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
@@ -9,53 +8,10 @@ SELECT * FROM t WHERE val = '[1,2,3]';
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
SELECT * FROM t ORDER BY val LIMIT 1;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
val
---------
[1,2,3]
(1 row)
SELECT * FROM t ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
(1 row)
SELECT * FROM t ORDER BY val;
val
-----------------
{}/3
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
(4 rows)
DROP TABLE t;

View File

@@ -28,26 +28,6 @@ SELECT ARRAY[1,2,3]::numeric[]::vector;
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector::real[];
float4
---------
{1,2,3}
(1 row)
SELECT '{1,2,3}'::real[]::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '{1,2,3}'::real[]::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '{1,2,3}'::real[]::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '{NULL}'::real[]::vector;
ERROR: array must not contain nulls
SELECT '{NaN}'::real[]::vector;
@@ -60,210 +40,12 @@ SELECT '{}'::real[]::vector;
ERROR: vector must have at least 1 dimension
SELECT '{{1}}'::real[]::vector;
ERROR: array must be 1-D
SELECT '{1,2,3}'::double precision[]::vector;
vector
SELECT '[1,2,3]'::vector::real[];
float4
---------
[1,2,3]
{1,2,3}
(1 row)
SELECT '{1,2,3}'::double precision[]::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '{1,2,3}'::double precision[]::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '{4e38,-4e38}'::double precision[]::vector;
ERROR: infinite value not allowed in vector
SELECT '{1e-46,-1e-46}'::double precision[]::vector;
vector
--------
[0,-0]
(1 row)
SELECT '[1,2,3]'::vector::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[65520]'::vector::halfvec;
ERROR: "65520" is out of range for type halfvec
SELECT '[1e-8]'::vector::halfvec;
halfvec
---------
[0]
(1 row)
SELECT '[1,2,3]'::halfvec::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '{1,2,3}'::real[]::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '{1,2,3}'::real[]::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '{1,2,3}'::real[]::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '{65520,-65520}'::real[]::halfvec;
ERROR: "65520" is out of range for type halfvec
SELECT '{1e-8,-1e-8}'::real[]::halfvec;
halfvec
---------
[0,-0]
(1 row)
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
sparsevec
-----------------
{2:1.5,4:3.5}/5
(1 row)
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(5);
sparsevec
-----------------
{2:1.5,4:3.5}/5
(1 row)
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(4);
ERROR: expected 4 dimensions, not 5
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector;
vector
-----------------
[0,1.5,0,3.5,0]
(1 row)
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(5);
vector
-----------------
[0,1.5,0,3.5,0]
(1 row)
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(4);
ERROR: expected 4 dimensions, not 5
SELECT '{}/16001'::sparsevec::vector;
ERROR: vector cannot have more than 16000 dimensions
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec;
sparsevec
-----------------
{2:1.5,4:3.5}/5
(1 row)
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(5);
sparsevec
-----------------
{2:1.5,4:3.5}/5
(1 row)
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(4);
ERROR: expected 4 dimensions, not 5
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec;
halfvec
-----------------
[0,1.5,0,3.5,0]
(1 row)
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(5);
halfvec
-----------------
[0,1.5,0,3.5,0]
(1 row)
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(4);
ERROR: expected 4 dimensions, not 5
SELECT '{}/16001'::sparsevec::halfvec;
ERROR: halfvec cannot have more than 16000 dimensions
SELECT '{1:65520}/1'::sparsevec::halfvec;
ERROR: "65520" is out of range for type halfvec
SELECT '{1:1e-8}/1'::sparsevec::halfvec;
halfvec
---------
[0]
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
array
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
sparsevec
-----------------
{1:1,3:2,5:3}/6
(1 row)
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
ERROR: expected 5 dimensions, not 6
SELECT '{NULL}'::real[]::sparsevec;
ERROR: array must not contain nulls
SELECT '{NaN}'::real[]::sparsevec;
ERROR: NaN not allowed in sparsevec
SELECT '{Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{-Infinity}'::real[]::sparsevec;
ERROR: infinite value not allowed in sparsevec
SELECT '{}'::real[]::sparsevec;
ERROR: sparsevec must have at least 1 dimension
SELECT '{{1}}'::real[]::sparsevec;
ERROR: array must be 1-D
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,50 +1,15 @@
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/vector.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/vector.bin' WITH (FORMAT binary)
CREATE TABLE t (val vector(3), val2 intvec(3));
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
CREATE TABLE t2 (val vector(3), val2 intvec(3));
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
DROP TABLE t2;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val halfvec(3));
\copy t TO 'results/halfvec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/halfvec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
(4 rows)
DROP TABLE t;
DROP TABLE t2;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE TABLE t2 (val sparsevec(3));
\copy t TO 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/sparsevec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
-----------------
{}/3
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
val | val2
---------+---------
[0,0,0] | [0,0,0]
[1,1,1] | [1,1,1]
[1,2,3] | [1,2,3]
|
(4 rows)
DROP TABLE t;

260
test/expected/functions.out Normal file
View File

@@ -0,0 +1,260 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
?column?
----------
[5,7,9]
(1 row)
SELECT '[3e38]'::vector + '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2,3]'::vector - '[4,5,6]';
?column?
------------
[-3,-3,-3]
(1 row)
SELECT '[-3e38]'::vector - '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2,3]'::vector * '[4,5,6]';
?column?
-----------
[4,10,18]
(1 row)
SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow
SELECT '[1,2,3]'::vector = '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector = '[1,2]';
ERROR: different vector dimensions 3 and 2
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp
------------
0
(1 row)
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[1,2]', '[2,3,4]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[2,3]', '[1,2,3]');
vector_cmp
------------
1
(1 row)
SELECT vector_dims('[1,2,3]');
vector_dims
-------------
3
(1 row)
SELECT round(vector_norm('[1,1]')::numeric, 5);
round
---------
1.41421
(1 row)
SELECT vector_norm('[3,4]');
vector_norm
-------------
5
(1 row)
SELECT vector_norm('[0,1]');
vector_norm
-------------
1
(1 row)
SELECT vector_norm('[3e37,4e37]')::real;
vector_norm
-------------
5e+37
(1 row)
SELECT l2_distance('[0,0]'::vector, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::vector, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
l2_distance
-------------
Infinity
(1 row)
SELECT inner_product('[1,2]'::vector, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT inner_product('[3e38]'::vector, '[3e38]');
inner_product
---------------
Infinity
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
cosine_distance
-----------------
NaN
(1 row)
SELECT l1_distance('[0,0]'::vector, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::vector, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
l1_distance
-------------
Infinity
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
avg
-----
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: expected 2 dimensions, not 1
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
avg
---------
[3e+38]
(1 row)
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
sum
-----
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: different vector dimensions 2 and 1
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
ERROR: value out of range: overflow

View File

@@ -1,642 +0,0 @@
SELECT '[1,2,3]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::halfvec;
halfvec
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::halfvec;
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::halfvec;
halfvec
------------
[1.234375]
(1 row)
SELECT '[hello,1]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[hello,1]"
LINE 1: SELECT '[hello,1]'::halfvec;
^
SELECT '[NaN,1]'::halfvec;
ERROR: NaN not allowed in halfvec
LINE 1: SELECT '[NaN,1]'::halfvec;
^
SELECT '[Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[Infinity,1]'::halfvec;
^
SELECT '[-Infinity,1]'::halfvec;
ERROR: infinite value not allowed in halfvec
LINE 1: SELECT '[-Infinity,1]'::halfvec;
^
SELECT '[65519,-65519]'::halfvec;
halfvec
----------------
[65504,-65504]
(1 row)
SELECT '[65520,-65520]'::halfvec;
ERROR: "65520" is out of range for type halfvec
LINE 1: SELECT '[65520,-65520]'::halfvec;
^
SELECT '[1e-8,-1e-8]'::halfvec;
halfvec
---------
[0,-0]
(1 row)
SELECT '[4e38,1]'::halfvec;
ERROR: "4e38" is out of range for type halfvec
LINE 1: SELECT '[4e38,1]'::halfvec;
^
SELECT '[1e-46,1]'::halfvec;
halfvec
---------
[0,1]
(1 row)
SELECT '[1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3"
LINE 1: SELECT '[1,2,3'::halfvec;
^
SELECT '[1,2,3]9'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::halfvec;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::halfvec;
ERROR: invalid input syntax for type halfvec: "1,2,3"
LINE 1: SELECT '1,2,3'::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT ''::halfvec;
ERROR: invalid input syntax for type halfvec: ""
LINE 1: SELECT ''::halfvec;
^
DETAIL: Vector contents must start with "[".
SELECT '['::halfvec;
ERROR: invalid input syntax for type halfvec: "["
LINE 1: SELECT '['::halfvec;
^
SELECT '[ '::halfvec;
ERROR: invalid input syntax for type halfvec: "[ "
LINE 1: SELECT '[ '::halfvec;
^
SELECT '[,'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,"
LINE 1: SELECT '[,'::halfvec;
^
SELECT '[]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[]'::halfvec;
^
SELECT '[ ]'::halfvec;
ERROR: halfvec must have at least 1 dimension
LINE 1: SELECT '[ ]'::halfvec;
^
SELECT '[,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[,]"
LINE 1: SELECT '[,]'::halfvec;
^
SELECT '[1,]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,]"
LINE 1: SELECT '[1,]'::halfvec;
^
SELECT '[1a]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1a]"
LINE 1: SELECT '[1a]'::halfvec;
^
SELECT '[1,,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1,,3]"
LINE 1: SELECT '[1,,3]'::halfvec;
^
SELECT '[1, ,3]'::halfvec;
ERROR: invalid input syntax for type halfvec: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::halfvec;
^
SELECT '[1,2,3]'::halfvec(3);
halfvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::halfvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::halfvec(3, 2);
^
SELECT '[1,2,3]'::halfvec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::halfvec('a');
^
SELECT '[1,2,3]'::halfvec(0);
ERROR: dimensions for type halfvec must be at least 1
LINE 1: SELECT '[1,2,3]'::halfvec(0);
^
SELECT '[1,2,3]'::halfvec(16001);
ERROR: dimensions for type halfvec cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::halfvec(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::halfvec(2)[];
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::halfvec + '[4,5,6]';
?column?
----------
[5,7,9]
(1 row)
SELECT '[65519]'::halfvec + '[65519]';
ERROR: value out of range: overflow
SELECT '[1,2]'::halfvec + '[3]';
ERROR: different halfvec dimensions 2 and 1
SELECT '[1,2,3]'::halfvec - '[4,5,6]';
?column?
------------
[-3,-3,-3]
(1 row)
SELECT '[-65519]'::halfvec - '[65519]';
ERROR: value out of range: overflow
SELECT '[1,2]'::halfvec - '[3]';
ERROR: different halfvec dimensions 2 and 1
SELECT '[1,2,3]'::halfvec * '[4,5,6]';
?column?
-----------
[4,10,18]
(1 row)
SELECT '[65519]'::halfvec * '[65519]';
ERROR: value out of range: overflow
SELECT '[1e-7]'::halfvec * '[1e-7]';
ERROR: value out of range: underflow
SELECT '[1,2]'::halfvec * '[3]';
ERROR: different halfvec dimensions 2 and 1
SELECT '[1,2,3]'::halfvec || '[4,5]';
?column?
-------------
[1,2,3,4,5]
(1 row)
SELECT array_fill(0, ARRAY[16000])::halfvec || '[1]';
ERROR: halfvec cannot have more than 16000 dimensions
SELECT '[1,2,3]'::halfvec < '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec < '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec <= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::halfvec <= '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec = '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::halfvec = '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec != '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec != '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::halfvec >= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::halfvec >= '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::halfvec > '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::halfvec > '[1,2]';
?column?
----------
t
(1 row)
SELECT halfvec_cmp('[1,2,3]', '[1,2,3]');
halfvec_cmp
-------------
0
(1 row)
SELECT halfvec_cmp('[1,2,3]', '[0,0,0]');
halfvec_cmp
-------------
1
(1 row)
SELECT halfvec_cmp('[0,0,0]', '[1,2,3]');
halfvec_cmp
-------------
-1
(1 row)
SELECT halfvec_cmp('[1,2]', '[1,2,3]');
halfvec_cmp
-------------
-1
(1 row)
SELECT halfvec_cmp('[1,2,3]', '[1,2]');
halfvec_cmp
-------------
1
(1 row)
SELECT halfvec_cmp('[1,2]', '[2,3,4]');
halfvec_cmp
-------------
-1
(1 row)
SELECT halfvec_cmp('[2,3]', '[1,2,3]');
halfvec_cmp
-------------
1
(1 row)
SELECT vector_dims('[1,2,3]'::halfvec);
vector_dims
-------------
3
(1 row)
SELECT round(l2_norm('[1,1]'::halfvec)::numeric, 5);
round
---------
1.41421
(1 row)
SELECT l2_norm('[3,4]'::halfvec);
l2_norm
---------
5
(1 row)
SELECT l2_norm('[0,1]'::halfvec);
l2_norm
---------
1
(1 row)
SELECT l2_norm('[0,0]'::halfvec);
l2_norm
---------
0
(1 row)
SELECT l2_norm('[2]'::halfvec);
l2_norm
---------
2
(1 row)
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::halfvec, '[1,1,1,1,1,1,1,4,5]');
l2_distance
-------------
5
(1 row)
SELECT '[0,0]'::halfvec <-> '[3,4]';
?column?
----------
5
(1 row)
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT inner_product('[65504]'::halfvec, '[65504]');
inner_product
---------------
4290774016
(1 row)
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
inner_product
---------------
45
(1 row)
SELECT '[1,2]'::halfvec <#> '[3,4]';
?column?
----------
-11
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
cosine_distance
-----------------
2
(1 row)
SELECT '[1,2]'::halfvec <=> '[2,4]';
?column?
----------
0
(1 row)
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::halfvec, '[3]');
ERROR: different halfvec dimensions 2 and 1
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
l1_distance
-------------
0
(1 row)
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[0,3,2,5,4,7,6,9,8]');
l1_distance
-------------
9
(1 row)
SELECT '[0,0]'::halfvec <+> '[3,4]';
?column?
----------
7
(1 row)
SELECT l2_normalize('[3,4]'::halfvec);
l2_normalize
------------------------
[0.60009766,0.7998047]
(1 row)
SELECT l2_normalize('[3,0]'::halfvec);
l2_normalize
--------------
[1,0]
(1 row)
SELECT l2_normalize('[0,0.1]'::halfvec);
l2_normalize
--------------
[0,1]
(1 row)
SELECT l2_normalize('[0,0]'::halfvec);
l2_normalize
--------------
[0,0]
(1 row)
SELECT l2_normalize('[65504]'::halfvec);
l2_normalize
--------------
[1]
(1 row)
SELECT binary_quantize('[1,0,-1]'::halfvec);
binary_quantize
-----------------
100
(1 row)
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
binary_quantize
-----------------
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 2147483647, 10);
ERROR: halfvec must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2147483647);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, -2147483644, 2147483647);
subvector
-----------
[1,2]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]', NULL]) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY[]::halfvec[]) v;
avg
-----
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::halfvec, '[3]']) v;
ERROR: expected 2 dimensions, not 1
SELECT avg(v) FROM unnest(ARRAY['[65504]'::halfvec, '[65504]']) v;
avg
---------
[65504]
(1 row)
SELECT halfvec_avg(array_agg(n)) FROM generate_series(1, 16002) n;
ERROR: halfvec cannot have more than 16000 dimensions
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]', NULL]) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY[]::halfvec[]) v;
sum
-----
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::halfvec, '[3]']) v;
ERROR: different halfvec dimensions 2 and 1
SELECT sum(v) FROM unnest(ARRAY['[65504]'::halfvec, '[65504]']) v;
ERROR: value out of range: overflow

View File

@@ -1,51 +0,0 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
val
-----
111
110
100
000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- jaccard
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
val
------
1111
1110
1100
0000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- varbit
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: type not supported for hnsw index
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -1,102 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::intvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::intvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,33 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::intvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

21
test/expected/hnsw_ip.out Normal file
View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;

33
test/expected/hnsw_l2.out Normal file
View File

@@ -0,0 +1,33 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 1);
ERROR: value 1 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
ERROR: value 101 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
ERROR: value 3 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m
SHOW hnsw.ef_search;
hnsw.ef_search
----------------
40
(1 row)
SET hnsw.ef_search = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
SET hnsw.ef_search = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
DROP TABLE t;

View File

@@ -1,112 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:4}/3
{1:1,2:2,3:3}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:1,3:1}/3
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
val
-----------------
{1:1,2:2,3:3}/3
{1:1,2:2,3:4}/3
{1:1,2:1,3:1}/3
{}/3
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- non-zero elements
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
ERROR: sparsevec cannot have more than 1000 non-zero elements for hnsw index
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;

View File

@@ -1,201 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- L1
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- iterative
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
SET hnsw.iterative_scan = strict_order;
SET hnsw.ef_search = 1;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
SET hnsw.iterative_scan = relaxed_order;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET hnsw.iterative_scan;
RESET hnsw.ef_search;
DROP TABLE t;
-- unlogged
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;
-- options
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 1);
ERROR: value 1 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
ERROR: value 101 out of bounds for option "m"
DETAIL: Valid values are between "2" and "100".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
ERROR: value 3 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m
SHOW hnsw.ef_search;
hnsw.ef_search
----------------
40
(1 row)
SET hnsw.ef_search = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
SET hnsw.ef_search = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.ef_search" (1 .. 1000)
SHOW hnsw.iterative_scan;
hnsw.iterative_scan
---------------------
off
(1 row)
SET hnsw.iterative_scan = on;
ERROR: invalid value for parameter "hnsw.iterative_scan": "on"
HINT: Available values: off, relaxed_order, strict_order.
SHOW hnsw.max_scan_tuples;
hnsw.max_scan_tuples
----------------------
20000
(1 row)
SET hnsw.max_scan_tuples = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.max_scan_tuples" (1 .. 2147483647)
SHOW hnsw.scan_mem_multiplier;
hnsw.scan_mem_multiplier
--------------------------
1
(1 row)
SET hnsw.scan_mem_multiplier = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
SET hnsw.scan_mem_multiplier = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
DROP TABLE t;

151
test/expected/input.out Normal file
View File

@@ -0,0 +1,151 @@
SELECT '[1,2,3]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::vector;
vector
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]"
LINE 1: SELECT '[hello,1]'::vector;
^
SELECT '[NaN,1]'::vector;
ERROR: NaN not allowed in vector
LINE 1: SELECT '[NaN,1]'::vector;
^
SELECT '[Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[Infinity,1]'::vector;
^
SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector;
^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[1,2,3'::vector;
ERROR: malformed vector literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
^
DETAIL: Unexpected end of input.
SELECT '[1,2,3]9'::vector;
ERROR: malformed vector literal: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::vector;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::vector;
ERROR: malformed vector literal: "1,2,3"
LINE 1: SELECT '1,2,3'::vector;
^
DETAIL: Vector contents must start with "[".
SELECT ''::vector;
ERROR: malformed vector literal: ""
LINE 1: SELECT ''::vector;
^
DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: malformed vector literal: "["
LINE 1: SELECT '['::vector;
^
DETAIL: Unexpected end of input.
SELECT '[,'::vector;
ERROR: malformed vector literal: "[,"
LINE 1: SELECT '[,'::vector;
^
DETAIL: Unexpected end of input.
SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector;
^
SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]"
LINE 1: SELECT '[1,]'::vector;
^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: malformed vector literal: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::vector(2)[];
ERROR: expected 2 dimensions, not 3

View File

@@ -0,0 +1,92 @@
SELECT l2_distance('[0,0]'::intvec, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::intvec, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::intvec, '[3]');
ERROR: different intvec dimensions 2 and 1
SELECT '[0,0]'::intvec <-> '[3,4]';
?column?
----------
5
(1 row)
SELECT inner_product('[1,2]'::intvec, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::intvec, '[3]');
ERROR: different intvec dimensions 2 and 1
SELECT inner_product('[127]'::intvec, '[127]');
inner_product
---------------
16129
(1 row)
SELECT '[1,2]'::intvec <#> '[3,4]';
?column?
----------
-11
(1 row)
SELECT cosine_distance('[1,2]'::intvec, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::intvec, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::intvec, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::intvec, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::intvec, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::intvec, '[3]');
ERROR: different intvec dimensions 2 and 1
SELECT '[1,2]'::intvec <=> '[2,4]';
?column?
----------
0
(1 row)
SELECT l1_distance('[0,0]'::intvec, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::intvec, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::intvec, '[3]');
ERROR: different intvec dimensions 2 and 1

View File

@@ -0,0 +1,119 @@
SELECT '[1,2,3]'::intvec;
intvec
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::intvec;
intvec
------------
[-1,-2,-3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::intvec;
intvec
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::intvec;
ERROR: invalid input syntax for type intvec: "[1.23456]"
LINE 1: SELECT '[1.23456]'::intvec;
^
SELECT '[hello,1]'::intvec;
ERROR: invalid input syntax for type intvec: "[hello,1]"
LINE 1: SELECT '[hello,1]'::intvec;
^
SELECT '[127,-128]'::intvec;
intvec
------------
[127,-128]
(1 row)
SELECT '[128,-129]'::intvec;
ERROR: value "128" is out of range for type intvec
LINE 1: SELECT '[128,-129]'::intvec;
^
SELECT '[1,2,3'::intvec;
ERROR: malformed intvec literal: "[1,2,3"
LINE 1: SELECT '[1,2,3'::intvec;
^
DETAIL: Unexpected end of input.
SELECT '[1,2,3]9'::intvec;
ERROR: malformed intvec literal: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::intvec;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::intvec;
ERROR: malformed intvec literal: "1,2,3"
LINE 1: SELECT '1,2,3'::intvec;
^
DETAIL: Vector contents must start with "[".
SELECT ''::intvec;
ERROR: malformed intvec literal: ""
LINE 1: SELECT ''::intvec;
^
DETAIL: Vector contents must start with "[".
SELECT '['::intvec;
ERROR: malformed intvec literal: "["
LINE 1: SELECT '['::intvec;
^
DETAIL: Unexpected end of input.
SELECT '[,'::intvec;
ERROR: malformed intvec literal: "[,"
LINE 1: SELECT '[,'::intvec;
^
DETAIL: Unexpected end of input.
SELECT '[]'::intvec;
ERROR: intvec must have at least 1 dimension
LINE 1: SELECT '[]'::intvec;
^
SELECT '[1,]'::intvec;
ERROR: invalid input syntax for type intvec: "[1,]"
LINE 1: SELECT '[1,]'::intvec;
^
SELECT '[1a]'::intvec;
ERROR: invalid input syntax for type intvec: "[1a]"
LINE 1: SELECT '[1a]'::intvec;
^
SELECT '[1,,3]'::intvec;
ERROR: malformed intvec literal: "[1,,3]"
LINE 1: SELECT '[1,,3]'::intvec;
^
SELECT '[1, ,3]'::intvec;
ERROR: invalid input syntax for type intvec: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::intvec;
^
SELECT '[1,2,3]'::intvec(3);
intvec
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::intvec(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::intvec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::intvec(3, 2);
^
SELECT '[1,2,3]'::intvec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::intvec('a');
^
SELECT '[1,2,3]'::intvec(0);
ERROR: dimensions for type intvec must be at least 1
LINE 1: SELECT '[1,2,3]'::intvec(0);
^
SELECT '[1,2,3]'::intvec(16001);
ERROR: dimensions for type intvec cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::intvec(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::intvec[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::intvec(2)[];
ERROR: expected 2 dimensions, not 3

View File

@@ -1,37 +0,0 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
val
-----
111
110
100
000
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- varbit
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops) WITH (lists = 1);
ERROR: type not supported for ivfflat index
CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
ERROR: column cannot have more than 64000 dimensions for ivfflat index
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;

View File

@@ -0,0 +1,26 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -1,84 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val halfvec_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
count
-------
3
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,21 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,39 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector);
val
---------
[0,0,0]
[1,1,1]
[1,2,3]
[1,2,4]
(4 rows)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;

View File

@@ -0,0 +1,14 @@
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
ERROR: value 0 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
SHOW ivfflat.probes;
ivfflat.probes
----------------
1
(1 row)
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;

View File

@@ -1,175 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,2,4]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
SELECT COUNT(*) FROM t;
count
-------
5
(1 row)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
DROP TABLE t;
-- inner product
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_ip_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
val
---------
[1,2,4]
[1,2,3]
[1,1,1]
[0,0,0]
(4 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
count
-------
4
(1 row)
DROP TABLE t;
-- cosine
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_cosine_ops) WITH (lists = 1);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
val
---------
[1,1,1]
[1,2,3]
[1,2,4]
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
3
(1 row)
DROP TABLE t;
-- iterative
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 3);
SET ivfflat.iterative_scan = relaxed_order;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
SET ivfflat.max_probes = 1;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
(1 row)
SET ivfflat.max_probes = 2;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
(2 rows)
TRUNCATE t;
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
-----
(0 rows)
RESET ivfflat.iterative_scan;
RESET ivfflat.max_probes;
DROP TABLE t;
-- unlogged
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 1);
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
val
---------
[1,2,3]
[1,1,1]
[0,0,0]
(3 rows)
DROP TABLE t;
-- options
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
ERROR: value 0 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
SHOW ivfflat.probes;
ivfflat.probes
----------------
1
(1 row)
SET ivfflat.probes = 0;
ERROR: 0 is outside the valid range for parameter "ivfflat.probes" (1 .. 32768)
SET ivfflat.probes = 32769;
ERROR: 32769 is outside the valid range for parameter "ivfflat.probes" (1 .. 32768)
SHOW ivfflat.iterative_scan;
ivfflat.iterative_scan
------------------------
off
(1 row)
SET ivfflat.iterative_scan = on;
ERROR: invalid value for parameter "ivfflat.iterative_scan": "on"
HINT: Available values: off, relaxed_order.
SHOW ivfflat.max_probes;
ivfflat.max_probes
--------------------
32768
(1 row)
SET ivfflat.max_probes = 0;
ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
SET ivfflat.max_probes = 32769;
ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
DROP TABLE t;

View File

@@ -1,653 +0,0 @@
SELECT '{1:1.5,3:3.5}/5'::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:-2,3:-4}/5'::sparsevec;
sparsevec
---------------
{1:-2,3:-4}/5
(1 row)
SELECT '{1:2.,3:4.}/5'::sparsevec;
sparsevec
-------------
{1:2,3:4}/5
(1 row)
SELECT ' { 1 : 1.5 , 3 : 3.5 } / 5 '::sparsevec;
sparsevec
-----------------
{1:1.5,3:3.5}/5
(1 row)
SELECT '{1:1.23456}/1'::sparsevec;
sparsevec
---------------
{1:1.23456}/1
(1 row)
SELECT '{1:hello,2:1}/2'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:hello,2:1}/2"
LINE 1: SELECT '{1:hello,2:1}/2'::sparsevec;
^
SELECT '{1:NaN,2:1}/2'::sparsevec;
ERROR: NaN not allowed in sparsevec
LINE 1: SELECT '{1:NaN,2:1}/2'::sparsevec;
^
SELECT '{1:Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:-Infinity,2:1}/2'::sparsevec;
ERROR: infinite value not allowed in sparsevec
LINE 1: SELECT '{1:-Infinity,2:1}/2'::sparsevec;
^
SELECT '{1:1.5e38,2:-1.5e38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e+38,2:-1.5e+38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e+38,2:-1.5e+38}/2
(1 row)
SELECT '{1:1.5e-38,2:-1.5e-38}/2'::sparsevec;
sparsevec
--------------------------
{1:1.5e-38,2:-1.5e-38}/2
(1 row)
SELECT '{1:4e38,2:1}/2'::sparsevec;
ERROR: "4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:4e38,2:1}/2'::sparsevec;
^
SELECT '{1:-4e38,2:1}/2'::sparsevec;
ERROR: "-4e38" is out of range for type sparsevec
LINE 1: SELECT '{1:-4e38,2:1}/2'::sparsevec;
^
SELECT '{1:1e-46,2:1}/2'::sparsevec;
ERROR: "1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:1e-46,2:1}/2'::sparsevec;
^
SELECT '{1:-1e-46,2:1}/2'::sparsevec;
ERROR: "-1e-46" is out of range for type sparsevec
LINE 1: SELECT '{1:-1e-46,2:1}/2'::sparsevec;
^
SELECT ''::sparsevec;
ERROR: invalid input syntax for type sparsevec: ""
LINE 1: SELECT ''::sparsevec;
^
DETAIL: Vector contents must start with "{".
SELECT '{'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{"
LINE 1: SELECT '{'::sparsevec;
^
SELECT '{ '::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{ "
LINE 1: SELECT '{ '::sparsevec;
^
SELECT '{:'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:"
LINE 1: SELECT '{:'::sparsevec;
^
SELECT '{,'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,"
LINE 1: SELECT '{,'::sparsevec;
^
SELECT '{}'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}"
LINE 1: SELECT '{}'::sparsevec;
^
DETAIL: Unexpected end of input.
SELECT '{}/'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/"
LINE 1: SELECT '{}/'::sparsevec;
^
SELECT '{}/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{}/1a'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{}/1a"
LINE 1: SELECT '{}/1a'::sparsevec;
^
DETAIL: Junk after closing.
SELECT '{ }/1'::sparsevec;
sparsevec
-----------
{}/1
(1 row)
SELECT '{:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:}/1"
LINE 1: SELECT '{:}/1'::sparsevec;
^
SELECT '{,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{,}/1"
LINE 1: SELECT '{,}/1'::sparsevec;
^
SELECT '{1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1,}/1"
LINE 1: SELECT '{1,}/1'::sparsevec;
^
SELECT '{:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{:1}/1"
LINE 1: SELECT '{:1}/1'::sparsevec;
^
SELECT '{1:}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:}/1"
LINE 1: SELECT '{1:}/1'::sparsevec;
^
SELECT '{1a:1}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1a:1}/1"
LINE 1: SELECT '{1a:1}/1'::sparsevec;
^
SELECT '{1:1a}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1a}/1"
LINE 1: SELECT '{1:1a}/1'::sparsevec;
^
SELECT '{1:1,}/1'::sparsevec;
ERROR: invalid input syntax for type sparsevec: "{1:1,}/1"
LINE 1: SELECT '{1:1,}/1'::sparsevec;
^
SELECT '{1:0,2:1,3:0}/3'::sparsevec;
sparsevec
-----------
{2:1}/3
(1 row)
SELECT '{2:1,1:1}/2'::sparsevec;
sparsevec
-------------
{1:1,2:1}/2
(1 row)
SELECT '{1:1,1:1}/2'::sparsevec;
ERROR: sparsevec indices must not contain duplicates
LINE 1: SELECT '{1:1,1:1}/2'::sparsevec;
^
SELECT '{1:1,2:1,1:1}/2'::sparsevec;
ERROR: sparsevec indices must not contain duplicates
LINE 1: SELECT '{1:1,2:1,1:1}/2'::sparsevec;
^
SELECT '{}/5'::sparsevec;
sparsevec
-----------
{}/5
(1 row)
SELECT '{}/-1'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-1'::sparsevec;
^
SELECT '{}/1000000001'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/1000000001'::sparsevec;
^
SELECT '{}/2147483648'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/2147483648'::sparsevec;
^
SELECT '{}/-2147483649'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-2147483649'::sparsevec;
^
SELECT '{}/9223372036854775808'::sparsevec;
ERROR: sparsevec cannot have more than 1000000000 dimensions
LINE 1: SELECT '{}/9223372036854775808'::sparsevec;
^
SELECT '{}/-9223372036854775809'::sparsevec;
ERROR: sparsevec must have at least 1 dimension
LINE 1: SELECT '{}/-9223372036854775809'::sparsevec;
^
SELECT '{2147483647:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2147483647:1}/1'::sparsevec;
^
SELECT '{2147483648:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2147483648:1}/1'::sparsevec;
^
SELECT '{-2147483648:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{-2147483648:1}/1'::sparsevec;
^
SELECT '{-2147483649:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{-2147483649:1}/1'::sparsevec;
^
SELECT '{0:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{0:1}/1'::sparsevec;
^
SELECT '{2:1}/1'::sparsevec;
ERROR: sparsevec index out of bounds
LINE 1: SELECT '{2:1}/1'::sparsevec;
^
SELECT '{}/3'::sparsevec(3);
sparsevec
-----------
{}/3
(1 row)
SELECT '{}/3'::sparsevec(2);
ERROR: expected 2 dimensions, not 3
SELECT '{}/3'::sparsevec(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '{}/3'::sparsevec(3, 2);
^
SELECT '{}/3'::sparsevec('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '{}/3'::sparsevec('a');
^
SELECT '{}/3'::sparsevec(0);
ERROR: dimensions for type sparsevec must be at least 1
LINE 1: SELECT '{}/3'::sparsevec(0);
^
SELECT '{}/3'::sparsevec(1000000001);
ERROR: dimensions for type sparsevec cannot exceed 1000000000
LINE 1: SELECT '{}/3'::sparsevec(1000000001);
^
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec < '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec <= '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec = '{1:1,2:2}/2';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec != '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2,3:3}/3';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec >= '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2,3:3}/3';
?column?
----------
f
(1 row)
SELECT '{1:1,2:2,3:3}/3'::sparsevec > '{1:1,2:2}/2';
?column?
----------
t
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
0
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{}/3', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2,3:3}/3', '{1:1,2:2}/2');
sparsevec_cmp
---------------
1
(1 row)
SELECT sparsevec_cmp('{1:1,2:2}/2', '{1:2,2:3,3:4}/3');
sparsevec_cmp
---------------
-1
(1 row)
SELECT sparsevec_cmp('{1:2,2:3}/2', '{1:1,2:2,3:3}/3');
sparsevec_cmp
---------------
1
(1 row)
SELECT round(l2_norm('{1:1,2:1}/2'::sparsevec)::numeric, 5);
round
---------
1.41421
(1 row)
SELECT l2_norm('{1:3,2:4}/2'::sparsevec);
l2_norm
---------
5
(1 row)
SELECT l2_norm('{2:1}/2'::sparsevec);
l2_norm
---------
1
(1 row)
SELECT l2_norm('{1:3e37,2:4e37}/2'::sparsevec)::real;
l2_norm
---------
5e+37
(1 row)
SELECT l2_norm('{}/2'::sparsevec);
l2_norm
---------
0
(1 row)
SELECT l2_norm('{1:2}/1'::sparsevec);
l2_norm
---------
2
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3}/2'::sparsevec, '{2:4}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{2:4}/2'::sparsevec, '{1:3}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{1:3,2:4}/2'::sparsevec, '{}/2');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
l2_distance
-------------
1
(1 row)
SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
?column?
----------
5
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
inner_product
---------------
10
(1 row)
SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT inner_product('{1:1,3:3}/4'::sparsevec, '{2:2,4:4}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{2:2,4:4}/4'::sparsevec, '{1:1,3:3}/4');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT inner_product('{1:1}/2'::sparsevec, '{}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{}/2'::sparsevec, '{1:1}/2');
inner_product
---------------
0
(1 row)
SELECT inner_product('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
inner_product
---------------
Infinity
(1 row)
SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
inner_product
---------------
18
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <#> '{1:3,2:4}/2';
?column?
----------
-11
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{}/2');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1,2:1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{2:2}/2'::sparsevec, '{1:2}/2');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1.1,2:1.1}/2');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1.1,2:-1.1}/2');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('{}/1'::sparsevec, '{}/1');
cosine_distance
-----------------
NaN
(1 row)
SELECT '{1:1,2:2}/2'::sparsevec <=> '{1:2,2:4}/2';
?column?
----------
0
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('{}/2'::sparsevec, '{2:1}/2');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
ERROR: different sparsevec dimensions 2 and 1
SELECT l1_distance('{1:3e38}/1'::sparsevec, '{1:-3e38}/1');
l1_distance
-------------
Infinity
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7}/8'::sparsevec, '{2:2,4:4,6:6,8:8}/8');
l1_distance
-------------
36
(1 row)
SELECT l1_distance('{1:1,3:3,5:5,7:7,9:9}/9'::sparsevec, '{2:2,4:4,6:6,8:8}/9');
l1_distance
-------------
45
(1 row)
SELECT '{}/2'::sparsevec <+> '{1:3,2:4}/2';
?column?
----------
7
(1 row)
SELECT l2_normalize('{1:3,2:4}/2'::sparsevec);
l2_normalize
-----------------
{1:0.6,2:0.8}/2
(1 row)
SELECT l2_normalize('{1:3}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:0.1}/2'::sparsevec);
l2_normalize
--------------
{2:1}/2
(1 row)
SELECT l2_normalize('{}/2'::sparsevec);
l2_normalize
--------------
{}/2
(1 row)
SELECT l2_normalize('{1:3e38}/1'::sparsevec);
l2_normalize
--------------
{1:1}/1
(1 row)
SELECT l2_normalize('{1:3e38,2:1e-37}/2'::sparsevec);
l2_normalize
--------------
{1:1}/2
(1 row)
SELECT l2_normalize('{2:3e37,4:3e-37,6:4e37,8:4e-37}/9'::sparsevec);
l2_normalize
-----------------
{2:0.6,6:0.8}/9
(1 row)

View File

@@ -1,678 +0,0 @@
SELECT '[1,2,3]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[-1,-2,-3]'::vector;
vector
------------
[-1,-2,-3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]"
LINE 1: SELECT '[hello,1]'::vector;
^
SELECT '[NaN,1]'::vector;
ERROR: NaN not allowed in vector
LINE 1: SELECT '[NaN,1]'::vector;
^
SELECT '[Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[Infinity,1]'::vector;
^
SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector;
^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: "4e38" is out of range for type vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[-4e38,1]'::vector;
ERROR: "-4e38" is out of range for type vector
LINE 1: SELECT '[-4e38,1]'::vector;
^
SELECT '[1e-46,1]'::vector;
vector
--------
[0,1]
(1 row)
SELECT '[-1e-46,1]'::vector;
vector
--------
[-0,1]
(1 row)
SELECT '[1,2,3'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3"
LINE 1: SELECT '[1,2,3'::vector;
^
SELECT '[1,2,3]9'::vector;
ERROR: invalid input syntax for type vector: "[1,2,3]9"
LINE 1: SELECT '[1,2,3]9'::vector;
^
DETAIL: Junk after closing right brace.
SELECT '1,2,3'::vector;
ERROR: invalid input syntax for type vector: "1,2,3"
LINE 1: SELECT '1,2,3'::vector;
^
DETAIL: Vector contents must start with "[".
SELECT ''::vector;
ERROR: invalid input syntax for type vector: ""
LINE 1: SELECT ''::vector;
^
DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: invalid input syntax for type vector: "["
LINE 1: SELECT '['::vector;
^
SELECT '[ '::vector;
ERROR: invalid input syntax for type vector: "[ "
LINE 1: SELECT '[ '::vector;
^
SELECT '[,'::vector;
ERROR: invalid input syntax for type vector: "[,"
LINE 1: SELECT '[,'::vector;
^
SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector;
^
SELECT '[ ]'::vector;
ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[ ]'::vector;
^
SELECT '[,]'::vector;
ERROR: invalid input syntax for type vector: "[,]"
LINE 1: SELECT '[,]'::vector;
^
SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]"
LINE 1: SELECT '[1,]'::vector;
^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: invalid input syntax for type vector: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(3);
vector
---------
[1,2,3]
(1 row)
SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector(3, 2);
ERROR: invalid type modifier
LINE 1: SELECT '[1,2,3]'::vector(3, 2);
^
SELECT '[1,2,3]'::vector('a');
ERROR: invalid input syntax for type integer: "a"
LINE 1: SELECT '[1,2,3]'::vector('a');
^
SELECT '[1,2,3]'::vector(0);
ERROR: dimensions for type vector must be at least 1
LINE 1: SELECT '[1,2,3]'::vector(0);
^
SELECT '[1,2,3]'::vector(16001);
ERROR: dimensions for type vector cannot exceed 16000
LINE 1: SELECT '[1,2,3]'::vector(16001);
^
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
unnest
---------
[1,2,3]
[4,5,6]
(2 rows)
SELECT '{"[1,2,3]"}'::vector(2)[];
ERROR: expected 2 dimensions, not 3
SELECT '[1,2,3]'::vector + '[4,5,6]';
?column?
----------
[5,7,9]
(1 row)
SELECT '[3e38]'::vector + '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2]'::vector + '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector - '[4,5,6]';
?column?
------------
[-3,-3,-3]
(1 row)
SELECT '[-3e38]'::vector - '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2]'::vector - '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector * '[4,5,6]';
?column?
-----------
[4,10,18]
(1 row)
SELECT '[1e37]'::vector * '[1e37]';
ERROR: value out of range: overflow
SELECT '[1e-37]'::vector * '[1e-37]';
ERROR: value out of range: underflow
SELECT '[1,2]'::vector * '[3]';
ERROR: different vector dimensions 2 and 1
SELECT '[1,2,3]'::vector || '[4,5]';
?column?
-------------
[1,2,3,4,5]
(1 row)
SELECT array_fill(0, ARRAY[16000])::vector || '[1]';
ERROR: vector cannot have more than 16000 dimensions
SELECT '[1,2,3]'::vector < '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector < '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector <= '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector = '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector = '[1,2]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector != '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2,3]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector >= '[1,2]';
?column?
----------
t
(1 row)
SELECT '[1,2,3]'::vector > '[1,2,3]';
?column?
----------
f
(1 row)
SELECT '[1,2,3]'::vector > '[1,2]';
?column?
----------
t
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
vector_cmp
------------
0
(1 row)
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2]', '[1,2,3]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[1,2,3]', '[1,2]');
vector_cmp
------------
1
(1 row)
SELECT vector_cmp('[1,2]', '[2,3,4]');
vector_cmp
------------
-1
(1 row)
SELECT vector_cmp('[2,3]', '[1,2,3]');
vector_cmp
------------
1
(1 row)
SELECT vector_dims('[1,2,3]'::vector);
vector_dims
-------------
3
(1 row)
SELECT round(vector_norm('[1,1]')::numeric, 5);
round
---------
1.41421
(1 row)
SELECT vector_norm('[3,4]');
vector_norm
-------------
5
(1 row)
SELECT vector_norm('[0,1]');
vector_norm
-------------
1
(1 row)
SELECT vector_norm('[3e37,4e37]')::real;
vector_norm
-------------
5e+37
(1 row)
SELECT vector_norm('[0,0]');
vector_norm
-------------
0
(1 row)
SELECT vector_norm('[2]');
vector_norm
-------------
2
(1 row)
SELECT l2_distance('[0,0]'::vector, '[3,4]');
l2_distance
-------------
5
(1 row)
SELECT l2_distance('[0,0]'::vector, '[0,1]');
l2_distance
-------------
1
(1 row)
SELECT l2_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
l2_distance
-------------
Infinity
(1 row)
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::vector, '[1,1,1,1,1,1,1,4,5]');
l2_distance
-------------
5
(1 row)
SELECT '[0,0]'::vector <-> '[3,4]';
?column?
----------
5
(1 row)
SELECT inner_product('[1,2]'::vector, '[3,4]');
inner_product
---------------
11
(1 row)
SELECT inner_product('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT inner_product('[3e38]'::vector, '[3e38]');
inner_product
---------------
Infinity
(1 row)
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::vector, '[1,2,3,4,5,6,7,8,9]');
inner_product
---------------
45
(1 row)
SELECT '[1,2]'::vector <#> '[3,4]';
?column?
----------
-11
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
cosine_distance
-----------------
1
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
cosine_distance
-----------------
2
(1 row)
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
cosine_distance
-----------------
NaN
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
cosine_distance
-----------------
0
(1 row)
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
cosine_distance
-----------------
2
(1 row)
SELECT '[1,2]'::vector <=> '[2,4]';
?column?
----------
0
(1 row)
SELECT l1_distance('[0,0]'::vector, '[3,4]');
l1_distance
-------------
7
(1 row)
SELECT l1_distance('[0,0]'::vector, '[0,1]');
l1_distance
-------------
1
(1 row)
SELECT l1_distance('[1,2]'::vector, '[3]');
ERROR: different vector dimensions 2 and 1
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
l1_distance
-------------
Infinity
(1 row)
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[1,2,3,4,5,6,7,8,9]');
l1_distance
-------------
0
(1 row)
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::vector, '[0,3,2,5,4,7,6,9,8]');
l1_distance
-------------
9
(1 row)
SELECT '[0,0]'::vector <+> '[3,4]';
?column?
----------
7
(1 row)
SELECT l2_normalize('[3,4]'::vector);
l2_normalize
--------------
[0.6,0.8]
(1 row)
SELECT l2_normalize('[3,0]'::vector);
l2_normalize
--------------
[1,0]
(1 row)
SELECT l2_normalize('[0,0.1]'::vector);
l2_normalize
--------------
[0,1]
(1 row)
SELECT l2_normalize('[0,0]'::vector);
l2_normalize
--------------
[0,0]
(1 row)
SELECT l2_normalize('[3e38]'::vector);
l2_normalize
--------------
[1]
(1 row)
SELECT binary_quantize('[1,0,-1]'::vector);
binary_quantize
-----------------
100
(1 row)
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
binary_quantize
-----------------
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------
[1,2,3]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);
subvector
-----------
[3,4]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -1, 3);
subvector
-----------
[1]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 3, 9);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 0);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, -1);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, -1, 2);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 2147483647, 10);
ERROR: vector must have at least 1 dimension
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2147483647);
subvector
-----------
[3,4,5]
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, -2147483644, 2147483647);
subvector
-----------
[1,2]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
avg
-----------
[2,3.5,5]
(1 row)
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
avg
-----
(1 row)
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: expected 2 dimensions, not 1
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
avg
---------
[3e+38]
(1 row)
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
ERROR: vector cannot have more than 16000 dimensions
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
sum
----------
[4,7,10]
(1 row)
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
sum
-----
(1 row)
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: different vector dimensions 2 and 1
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
ERROR: value out of range: overflow

View File

@@ -1,11 +0,0 @@
package PostgreSQL::Test::Cluster;
use PostgresNode;
sub new
{
my ($class, $name) = @_;
return get_new_node($name);
}
1;

View File

@@ -1,5 +0,0 @@
package PostgreSQL::Test::Utils;
use TestLib;
1;

View File

@@ -0,0 +1,8 @@
use PostgreSQL::Test::Cluster;
sub get_new_node
{
return PostgreSQL::Test::Cluster->new(@_);
}
1;

3
test/perl/TestLib.pm Normal file
View File

@@ -0,0 +1,3 @@
use PostgreSQL::Test::Utils;
1;

View File

@@ -1,27 +0,0 @@
SELECT hamming_distance('111', '111');
SELECT hamming_distance('111', '110');
SELECT hamming_distance('111', '100');
SELECT hamming_distance('111', '000');
SELECT hamming_distance('10101010101010101010', '01010101010101010101');
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
SELECT hamming_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
SELECT hamming_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
SELECT hamming_distance('', '');
SELECT hamming_distance('111', '00');
SELECT hamming_distance('111', '000'::varbit(4));
SELECT hamming_distance('111', '0000'::varbit(4));
SELECT jaccard_distance('1111', '1111');
SELECT jaccard_distance('1111', '1110');
SELECT jaccard_distance('1111', '1100');
SELECT jaccard_distance('1111', '1000');
SELECT jaccard_distance('1111', '0000');
SELECT jaccard_distance('1100', '1000');
SELECT jaccard_distance('10101010101010101010', '01010101010101010101');
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101');
SELECT jaccard_distance('101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101', '010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010');
SELECT jaccard_distance('110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011', '100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001');
SELECT jaccard_distance('', '');
SELECT jaccard_distance('1111', '000');
SELECT jaccard_distance('1111', '0000'::varbit(5));
SELECT jaccard_distance('1111', '00000'::varbit(5));

View File

@@ -1,34 +1,10 @@
SET enable_seqscan = off;
-- vector
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '[1,2,3]';
SELECT * FROM t ORDER BY val;
DROP TABLE t;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t (val);
SELECT * FROM t WHERE val = '{1:1,2:2,3:3}/3';
SELECT * FROM t ORDER BY val;
SELECT * FROM t ORDER BY val LIMIT 1;
DROP TABLE t;

View File

@@ -3,77 +3,13 @@ SELECT ARRAY[1.0,2.0,3.0]::vector;
SELECT ARRAY[1,2,3]::float4[]::vector;
SELECT ARRAY[1,2,3]::float8[]::vector;
SELECT ARRAY[1,2,3]::numeric[]::vector;
SELECT '[1,2,3]'::vector::real[];
SELECT '{1,2,3}'::real[]::vector;
SELECT '{1,2,3}'::real[]::vector(3);
SELECT '{1,2,3}'::real[]::vector(2);
SELECT '{NULL}'::real[]::vector;
SELECT '{NaN}'::real[]::vector;
SELECT '{Infinity}'::real[]::vector;
SELECT '{-Infinity}'::real[]::vector;
SELECT '{}'::real[]::vector;
SELECT '{{1}}'::real[]::vector;
SELECT '{1,2,3}'::double precision[]::vector;
SELECT '{1,2,3}'::double precision[]::vector(3);
SELECT '{1,2,3}'::double precision[]::vector(2);
SELECT '{4e38,-4e38}'::double precision[]::vector;
SELECT '{1e-46,-1e-46}'::double precision[]::vector;
SELECT '[1,2,3]'::vector::halfvec;
SELECT '[1,2,3]'::vector::halfvec(3);
SELECT '[1,2,3]'::vector::halfvec(2);
SELECT '[65520]'::vector::halfvec;
SELECT '[1e-8]'::vector::halfvec;
SELECT '[1,2,3]'::halfvec::vector;
SELECT '[1,2,3]'::halfvec::vector(3);
SELECT '[1,2,3]'::halfvec::vector(2);
SELECT '{1,2,3}'::real[]::halfvec;
SELECT '{1,2,3}'::real[]::halfvec(3);
SELECT '{1,2,3}'::real[]::halfvec(2);
SELECT '{65520,-65520}'::real[]::halfvec;
SELECT '{1e-8,-1e-8}'::real[]::halfvec;
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec;
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(5);
SELECT '[0,1.5,0,3.5,0]'::vector::sparsevec(4);
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector;
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(5);
SELECT '{2:1.5,4:3.5}/5'::sparsevec::vector(4);
SELECT '{}/16001'::sparsevec::vector;
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec;
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(5);
SELECT '[0,1.5,0,3.5,0]'::halfvec::sparsevec(4);
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec;
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(5);
SELECT '{2:1.5,4:3.5}/5'::sparsevec::halfvec(4);
SELECT '{}/16001'::sparsevec::halfvec;
SELECT '{1:65520}/1'::sparsevec::halfvec;
SELECT '{1:1e-8}/1'::sparsevec::halfvec;
SELECT ARRAY[1,0,2,0,3,0]::sparsevec;
SELECT ARRAY[1.0,0.0,2.0,0.0,3.0,0.0]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float4[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::float8[]::sparsevec;
SELECT ARRAY[1,0,2,0,3,0]::numeric[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec;
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(6);
SELECT '{1,0,2,0,3,0}'::real[]::sparsevec(5);
SELECT '{NULL}'::real[]::sparsevec;
SELECT '{NaN}'::real[]::sparsevec;
SELECT '{Infinity}'::real[]::sparsevec;
SELECT '{-Infinity}'::real[]::sparsevec;
SELECT '{}'::real[]::sparsevec;
SELECT '{{1}}'::real[]::sparsevec;
SELECT '[1,2,3]'::vector::real[];
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;

View File

@@ -1,42 +1,10 @@
-- vector
CREATE TABLE t (val vector(3), val2 intvec(3));
INSERT INTO t (val, val2) VALUES ('[0,0,0]', '[0,0,0]'), ('[1,2,3]', '[1,2,3]'), ('[1,1,1]', '[1,1,1]'), (NULL, NULL);
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3), val2 intvec(3));
CREATE TABLE t2 (val vector(3));
\copy t TO 'results/vector.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/vector.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
DROP TABLE t;
DROP TABLE t2;
-- halfvec
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val halfvec(3));
\copy t TO 'results/halfvec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/halfvec.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
DROP TABLE t;
DROP TABLE t2;
-- sparsevec
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE TABLE t2 (val sparsevec(3));
\copy t TO 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/sparsevec.bin' WITH (FORMAT binary)
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;

62
test/sql/functions.sql Normal file
View File

@@ -0,0 +1,62 @@
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT '[1,2,3]'::vector * '[4,5,6]';
SELECT '[1e37]'::vector * '[1e37]';
SELECT '[1e-37]'::vector * '[1e-37]';
SELECT '[1,2,3]'::vector = '[1,2,3]';
SELECT '[1,2,3]'::vector = '[1,2]';
SELECT vector_cmp('[1,2,3]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[0,0,0]');
SELECT vector_cmp('[0,0,0]', '[1,2,3]');
SELECT vector_cmp('[1,2]', '[1,2,3]');
SELECT vector_cmp('[1,2,3]', '[1,2]');
SELECT vector_cmp('[1,2]', '[2,3,4]');
SELECT vector_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]');
SELECT round(vector_norm('[1,1]')::numeric, 5);
SELECT vector_norm('[3,4]');
SELECT vector_norm('[0,1]');
SELECT vector_norm('[3e37,4e37]')::real;
SELECT l2_distance('[0,0]'::vector, '[3,4]');
SELECT l2_distance('[0,0]'::vector, '[0,1]');
SELECT l2_distance('[1,2]'::vector, '[3]');
SELECT l2_distance('[3e38]'::vector, '[-3e38]');
SELECT inner_product('[1,2]'::vector, '[3,4]');
SELECT inner_product('[1,2]'::vector, '[3]');
SELECT inner_product('[3e38]'::vector, '[3e38]');
SELECT cosine_distance('[1,2]'::vector, '[2,4]');
SELECT cosine_distance('[1,2]'::vector, '[0,0]');
SELECT cosine_distance('[1,1]'::vector, '[1,1]');
SELECT cosine_distance('[1,0]'::vector, '[0,2]');
SELECT cosine_distance('[1,1]'::vector, '[-1,-1]');
SELECT cosine_distance('[1,2]'::vector, '[3]');
SELECT cosine_distance('[1,1]'::vector, '[1.1,1.1]');
SELECT cosine_distance('[1,1]'::vector, '[-1.1,-1.1]');
SELECT cosine_distance('[3e38]'::vector, '[3e38]');
SELECT l1_distance('[0,0]'::vector, '[3,4]');
SELECT l1_distance('[0,0]'::vector, '[0,1]');
SELECT l1_distance('[1,2]'::vector, '[3]');
SELECT l1_distance('[3e38]'::vector, '[-3e38]');
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[3e38]'::vector, '[3e38]']) v;

View File

@@ -1,148 +0,0 @@
SELECT '[1,2,3]'::halfvec;
SELECT '[-1,-2,-3]'::halfvec;
SELECT '[1.,2.,3.]'::halfvec;
SELECT ' [ 1, 2 , 3 ] '::halfvec;
SELECT '[1.23456]'::halfvec;
SELECT '[hello,1]'::halfvec;
SELECT '[NaN,1]'::halfvec;
SELECT '[Infinity,1]'::halfvec;
SELECT '[-Infinity,1]'::halfvec;
SELECT '[65519,-65519]'::halfvec;
SELECT '[65520,-65520]'::halfvec;
SELECT '[1e-8,-1e-8]'::halfvec;
SELECT '[4e38,1]'::halfvec;
SELECT '[1e-46,1]'::halfvec;
SELECT '[1,2,3'::halfvec;
SELECT '[1,2,3]9'::halfvec;
SELECT '1,2,3'::halfvec;
SELECT ''::halfvec;
SELECT '['::halfvec;
SELECT '[ '::halfvec;
SELECT '[,'::halfvec;
SELECT '[]'::halfvec;
SELECT '[ ]'::halfvec;
SELECT '[,]'::halfvec;
SELECT '[1,]'::halfvec;
SELECT '[1a]'::halfvec;
SELECT '[1,,3]'::halfvec;
SELECT '[1, ,3]'::halfvec;
SELECT '[1,2,3]'::halfvec(3);
SELECT '[1,2,3]'::halfvec(2);
SELECT '[1,2,3]'::halfvec(3, 2);
SELECT '[1,2,3]'::halfvec('a');
SELECT '[1,2,3]'::halfvec(0);
SELECT '[1,2,3]'::halfvec(16001);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::halfvec[]);
SELECT '{"[1,2,3]"}'::halfvec(2)[];
SELECT '[1,2,3]'::halfvec + '[4,5,6]';
SELECT '[65519]'::halfvec + '[65519]';
SELECT '[1,2]'::halfvec + '[3]';
SELECT '[1,2,3]'::halfvec - '[4,5,6]';
SELECT '[-65519]'::halfvec - '[65519]';
SELECT '[1,2]'::halfvec - '[3]';
SELECT '[1,2,3]'::halfvec * '[4,5,6]';
SELECT '[65519]'::halfvec * '[65519]';
SELECT '[1e-7]'::halfvec * '[1e-7]';
SELECT '[1,2]'::halfvec * '[3]';
SELECT '[1,2,3]'::halfvec || '[4,5]';
SELECT array_fill(0, ARRAY[16000])::halfvec || '[1]';
SELECT '[1,2,3]'::halfvec < '[1,2,3]';
SELECT '[1,2,3]'::halfvec < '[1,2]';
SELECT '[1,2,3]'::halfvec <= '[1,2,3]';
SELECT '[1,2,3]'::halfvec <= '[1,2]';
SELECT '[1,2,3]'::halfvec = '[1,2,3]';
SELECT '[1,2,3]'::halfvec = '[1,2]';
SELECT '[1,2,3]'::halfvec != '[1,2,3]';
SELECT '[1,2,3]'::halfvec != '[1,2]';
SELECT '[1,2,3]'::halfvec >= '[1,2,3]';
SELECT '[1,2,3]'::halfvec >= '[1,2]';
SELECT '[1,2,3]'::halfvec > '[1,2,3]';
SELECT '[1,2,3]'::halfvec > '[1,2]';
SELECT halfvec_cmp('[1,2,3]', '[1,2,3]');
SELECT halfvec_cmp('[1,2,3]', '[0,0,0]');
SELECT halfvec_cmp('[0,0,0]', '[1,2,3]');
SELECT halfvec_cmp('[1,2]', '[1,2,3]');
SELECT halfvec_cmp('[1,2,3]', '[1,2]');
SELECT halfvec_cmp('[1,2]', '[2,3,4]');
SELECT halfvec_cmp('[2,3]', '[1,2,3]');
SELECT vector_dims('[1,2,3]'::halfvec);
SELECT round(l2_norm('[1,1]'::halfvec)::numeric, 5);
SELECT l2_norm('[3,4]'::halfvec);
SELECT l2_norm('[0,1]'::halfvec);
SELECT l2_norm('[0,0]'::halfvec);
SELECT l2_norm('[2]'::halfvec);
SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
SELECT l2_distance('[0,0]'::halfvec, '[0,1]');
SELECT l2_distance('[1,2]'::halfvec, '[3]');
SELECT l2_distance('[1,1,1,1,1,1,1,1,1]'::halfvec, '[1,1,1,1,1,1,1,4,5]');
SELECT '[0,0]'::halfvec <-> '[3,4]';
SELECT inner_product('[1,2]'::halfvec, '[3,4]');
SELECT inner_product('[1,2]'::halfvec, '[3]');
SELECT inner_product('[65504]'::halfvec, '[65504]');
SELECT inner_product('[1,1,1,1,1,1,1,1,1]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
SELECT '[1,2]'::halfvec <#> '[3,4]';
SELECT cosine_distance('[1,2]'::halfvec, '[2,4]');
SELECT cosine_distance('[1,2]'::halfvec, '[0,0]');
SELECT cosine_distance('[1,1]'::halfvec, '[1,1]');
SELECT cosine_distance('[1,0]'::halfvec, '[0,2]');
SELECT cosine_distance('[1,1]'::halfvec, '[-1,-1]');
SELECT cosine_distance('[1,2]'::halfvec, '[3]');
SELECT cosine_distance('[1,1]'::halfvec, '[1.1,1.1]');
SELECT cosine_distance('[1,1]'::halfvec, '[-1.1,-1.1]');
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
SELECT cosine_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[-1,-2,-3,-4,-5,-6,-7,-8,-9]');
SELECT '[1,2]'::halfvec <=> '[2,4]';
SELECT l1_distance('[0,0]'::halfvec, '[3,4]');
SELECT l1_distance('[0,0]'::halfvec, '[0,1]');
SELECT l1_distance('[1,2]'::halfvec, '[3]');
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[1,2,3,4,5,6,7,8,9]');
SELECT l1_distance('[1,2,3,4,5,6,7,8,9]'::halfvec, '[0,3,2,5,4,7,6,9,8]');
SELECT '[0,0]'::halfvec <+> '[3,4]';
SELECT l2_normalize('[3,4]'::halfvec);
SELECT l2_normalize('[3,0]'::halfvec);
SELECT l2_normalize('[0,0.1]'::halfvec);
SELECT l2_normalize('[0,0]'::halfvec);
SELECT l2_normalize('[65504]'::halfvec);
SELECT binary_quantize('[1,0,-1]'::halfvec);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 9);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 0);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, -1);
SELECT subvector('[1,2,3,4,5]'::halfvec, -1, 2);
SELECT subvector('[1,2,3,4,5]'::halfvec, 2147483647, 10);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2147483647);
SELECT subvector('[1,2,3,4,5]'::halfvec, -2147483644, 2147483647);
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::halfvec[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::halfvec, '[3]']) v;
SELECT avg(v) FROM unnest(ARRAY['[65504]'::halfvec, '[65504]']) v;
SELECT halfvec_avg(array_agg(n)) FROM generate_series(1, 16002) n;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]']) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2,3]'::halfvec, '[3,5,7]', NULL]) v;
SELECT sum(v) FROM unnest(ARRAY[]::halfvec[]) v;
SELECT sum(v) FROM unnest(ARRAY['[1,2]'::halfvec, '[3]']) v;
SELECT sum(v) FROM unnest(ARRAY['[65504]'::halfvec, '[65504]']) v;

View File

@@ -1,35 +0,0 @@
SET enable_seqscan = off;
-- hamming
CREATE TABLE t (val bit(3));
INSERT INTO t (val) VALUES (B'000'), (B'100'), (B'111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
INSERT INTO t (val) VALUES (B'110');
SELECT * FROM t ORDER BY val <~> B'111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <~> (SELECT NULL::bit)) t2;
DROP TABLE t;
-- jaccard
CREATE TABLE t (val bit(4));
INSERT INTO t (val) VALUES (B'0000'), (B'1100'), (B'1111'), (NULL);
CREATE INDEX ON t USING hnsw (val bit_jaccard_ops);
INSERT INTO t (val) VALUES (B'1110');
SELECT * FROM t ORDER BY val <%> B'1111';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <%> (SELECT NULL::bit)) t2;
DROP TABLE t;
-- varbit
CREATE TABLE t (val varbit(3));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
DROP TABLE t;

13
test/sql/hnsw_cosine.sql Normal file
View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
DROP TABLE t;

View File

@@ -1,58 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::halfvec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
DROP TABLE t;
-- inner product
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- cosine
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- L1
CREATE TABLE t (val halfvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val halfvec_l1_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <+> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
DROP TABLE t;

View File

@@ -0,0 +1,13 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_cosine_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <=> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::intvec)) t2;
DROP TABLE t;

View File

@@ -0,0 +1,12 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::intvec)) t2;
DROP TABLE t;

View File

@@ -0,0 +1,16 @@
SET enable_seqscan = off;
CREATE TABLE t (val intvec(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val intvec_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::intvec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
DROP TABLE t;

12
test/sql/hnsw_ip.sql Normal file
View File

@@ -0,0 +1,12 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_ip_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <#> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::vector)) t2;
DROP TABLE t;

16
test/sql/hnsw_l2.sql Normal file
View File

@@ -0,0 +1,16 @@
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
INSERT INTO t (val) VALUES ('[1,2,4]');
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::vector)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '[3,3,3]';
DROP TABLE t;

13
test/sql/hnsw_options.sql Normal file
View File

@@ -0,0 +1,13 @@
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 1);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
SHOW hnsw.ef_search;
SET hnsw.ef_search = 0;
SET hnsw.ef_search = 1001;
DROP TABLE t;

View File

@@ -1,68 +0,0 @@
SET enable_seqscan = off;
-- L2
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <-> (SELECT NULL::sparsevec)) t2;
SELECT COUNT(*) FROM t;
TRUNCATE t;
SELECT * FROM t ORDER BY val <-> '{1:3,2:3,3:3}/3';
DROP TABLE t;
-- inner product
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_ip_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <#> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <#> (SELECT NULL::sparsevec)) t2;
DROP TABLE t;
-- cosine
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_cosine_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <=> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '{}/3') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::sparsevec)) t2;
DROP TABLE t;
-- L1
CREATE TABLE t (val sparsevec(3));
INSERT INTO t (val) VALUES ('{}/3'), ('{1:1,2:2,3:3}/3'), ('{1:1,2:1,3:1}/3'), (NULL);
CREATE INDEX ON t USING hnsw (val sparsevec_l1_ops);
INSERT INTO t (val) VALUES ('{1:1,2:2,3:4}/3');
SELECT * FROM t ORDER BY val <+> '{1:3,2:3,3:3}/3';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::sparsevec)) t2;
DROP TABLE t;
-- non-zero elements
CREATE TABLE t (val sparsevec(1001));
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
TRUNCATE t;
CREATE INDEX ON t USING hnsw (val sparsevec_l2_ops);
INSERT INTO t (val) VALUES (array_fill(1, ARRAY[1001])::vector::sparsevec);
DROP TABLE t;

Some files were not shown because too many files have changed in this diff Show More