Compare commits

...

89 Commits

Author SHA1 Message Date
Andrew Kane
cab9da72c0 Version bump to 0.8.2 [skip ci] 2026-02-25 10:46:57 -08:00
Andrew Kane
d702ee31f2 Added test case for low memory HNSW index builds 2026-02-25 10:41:17 -08:00
Andrew Kane
9ed39fb94b Hardened shared memory allocator
Co-authored-by: chungkn from OneMount Group <kimngocchung.k2a@gmail.com>
2026-02-25 10:39:01 -08:00
Andrew Kane
b7e680d41a Fixed integer wraparound leading to buffer overflow with parallel HNSW index build
Reported-by: chungkn from OneMount Group <kimngocchung.k2a@gmail.com>
2026-02-25 10:35:14 -08:00
Andrew Kane
f5841f07fd Updated CI 2026-02-23 14:06:35 -08:00
Andrew Kane
35ab919bf5 Switched to statically-allocated IndexAmRoutine for Postgres 19 [skip ci] 2026-01-21 16:40:35 -08:00
Andrew Kane
5c08be0291 Updated checkout action [skip ci] 2026-01-15 20:45:56 -08:00
Jack Huang
544686feb1 Add RISC-V64 architecture support in Makefile (#948) 2026-01-14 06:11:58 -08:00
Andrew Kane
1a8ebb1129 Updated license year [skip ci] 2026-01-05 13:01:58 -08:00
Andrew Kane
aca89aec73 Switched to scanner_isspace for Postgres 17+ - resolves #942 2026-01-05 12:54:04 -08:00
Andrew Kane
c9875f5150 Simplified test case - resolves #943 2026-01-05 12:42:14 -08:00
Andrew Kane
df2c3d301f Added link to pgvector-cobol [skip ci] 2025-12-13 11:33:35 -08:00
Andrew Kane
d557f1c9b1 Added link to pgvector-tcl [skip ci] 2025-12-11 23:22:56 -08:00
Andrew Kane
3d6dd20469 Removed unneeded check 2025-12-11 16:28:45 -08:00
Andrew Kane
c711da411c Improved includes for indexes 2025-12-11 15:35:37 -08:00
Andrew Kane
3975e212cc Improved includes for ivfkmeans [skip ci] 2025-12-11 13:26:36 -08:00
Andrew Kane
2b741ef068 Improved includes for types 2025-12-11 00:35:18 -08:00
Andrew Kane
f85a7d3fd0 Keep clang-tidy happy [skip ci] 2025-12-10 20:03:26 -08:00
Andrew Kane
5c25f81789 Fixed inconsistent parameter names [skip ci] 2025-12-10 18:49:00 -08:00
Andrew Kane
d72ee71f23 Fixed compilation errors with Postgres 19 2025-12-10 14:59:29 -08:00
Andrew Kane
b7c0085afe Added instructions for APK [skip ci] 2025-12-01 22:52:23 -08:00
Andrew Kane
2b742576b4 Added links [skip ci] 2025-11-24 12:41:30 -08:00
Andrew Kane
a126c02184 Added link to pgvector-ada [skip ci] 2025-10-30 22:29:36 -07:00
Andrew Kane
744362305b Updated changelog [skip ci] 2025-10-30 11:48:44 -07:00
Andrew Kane
69e78e36a3 Added link to pgvetor-pascal [skip ci] 2025-10-30 11:40:29 -07:00
Andrew Kane
8dd37d4fb2 Improved install target for Windows - #917 [skip ci] 2025-10-30 11:03:59 -07:00
Andrew Kane
a24125839a Updated readme [skip ci] 2025-10-28 10:25:34 -07:00
Andrew Kane
2ee3113417 Updated readme [skip ci] 2025-10-22 13:10:58 -07:00
Andrew Kane
3ebb9a506e Added varatt headers for Postgres 16+ 2025-10-22 11:56:05 -07:00
Andrew Kane
727d4836d2 Fixed Index Searches in EXPLAIN output for Postgres 18 2025-10-22 11:47:11 -07:00
Andrew Kane
d823c44591 Removed unused parameter [skip ci] 2025-09-27 15:58:18 -07:00
Andrew Kane
8f4aa0751c Removed unused parameters [skip ci] 2025-09-27 15:51:32 -07:00
Andrew Kane
1ff30720c7 Updated readme [skip ci] 2025-09-26 13:02:37 -07:00
Andrew Kane
4cef9213ec Added Docker images for Postgres 18 [skip ci] 2025-09-25 20:00:10 -07:00
Andrew Kane
90add68d6f Updated style to be consistent with Mac [skip ci] 2025-09-25 15:55:30 -07:00
Andrew Kane
bc05dbf312 Updated CI for Postgres 18 [skip ci] 2025-09-25 15:54:37 -07:00
Andrew Kane
02fefc0e3f Updated readme for Postgres 18 [skip ci] 2025-09-25 15:46:28 -07:00
Andrew Kane
e7899b1250 Updated checkout action [skip ci] 2025-09-22 11:55:01 -07:00
Andrew Kane
9e29dff78c Updated CI [skip ci] 2025-09-19 19:16:08 -07:00
Andrew Kane
13c0dbf530 Test with macos-15-intel on CI [skip ci] 2025-09-19 18:34:51 -07:00
Andrew Kane
c205a12107 Updated CI [skip ci] 2025-09-05 17:03:37 -07:00
Andrew Kane
db3755a58f Added windows-2025 to CI [skip ci] 2025-09-05 16:41:22 -07:00
Andrew Kane
778dacf20c Version bump to 0.8.1 [skip ci] 2025-09-04 17:51:09 -07:00
Andrew Kane
3f687687ee Fixed compilation error with Postgres 19 2025-09-04 15:58:23 -07:00
Andrew Kane
815f48e489 Updated changelog [skip ci] 2025-09-04 15:36:18 -07:00
Andrew Kane
67e648b13e Added another test for binary_quantize function [skip ci] 2025-08-29 01:09:18 -07:00
Andrew Kane
c3ff955231 Reordered supported Docker tags [skip ci] 2025-08-23 19:57:14 -07:00
Andrew Kane
6f46a1035d Added new Docker images to readme [skip ci] 2025-08-23 19:46:07 -07:00
Andrew Kane
bbe66e821b Added Docker images for Debian 13 / Trixie [skip ci] 2025-08-23 19:34:11 -07:00
Julien Rouhaud
dd3a1e9137 Use NIL for empty lists (#890)
Postgres standard way to check for list emptiness is to compare a pointer to
NIL rather than NULL.
2025-08-23 03:31:22 -07:00
Andrew Kane
ea4746f6c0 Updated comment [skip ci] 2025-08-19 14:26:36 -07:00
Andrew Kane
6aec80ccdb Improved performance of binary_quantize function for halfvec 2025-08-19 13:47:11 -07:00
Luca Giacchino
0c9070ba82 Add autovectorized implementation of binary quantize 2025-08-19 13:18:50 -07:00
Andrew Kane
30382418da Ran pgindent [skip ci] 2025-08-19 12:44:54 -07:00
Andrew Kane
26b50e536f Updated FreeBSD package name in readme [skip ci] 2025-08-09 12:09:29 -07:00
Andrew Kane
e29fc3aa1a Switched to PG_MODULE_MAGIC_EXT for Postgres 18+ [skip ci] 2025-07-31 19:59:27 -07:00
Andrew Kane
6ef7fccb5c Added Postgres 19 to CI [skip ci] 2025-07-27 19:50:46 -07:00
Andrew Kane
5b8b68ba1d Use consistent style [skip ci] 2025-07-27 18:19:28 -07:00
Andrew Kane
3be8693c13 Added supported Docker tags to readme [skip ci] 2025-07-27 13:44:59 -07:00
Andrew Kane
247c8dc8a5 Updated changelog [skip ci] 2025-07-27 13:25:51 -07:00
Andrew Kane
3600ab93e5 Added Docker images with -bookworm suffix [skip ci] 2025-07-27 13:14:29 -07:00
Andrew Kane
665db75a3c Updated Dockerfile to use release [skip ci] 2025-07-27 12:34:23 -07:00
Andrew Kane
44163d0a97 Revert "Added OS to Dockerfile [skip ci]"
This reverts commit 33ca8a61e2.
2025-07-09 22:01:34 -07:00
Andrew Kane
33ca8a61e2 Added OS to Dockerfile [skip ci] 2025-07-09 16:25:05 -07:00
Andrew Kane
742e2d1d28 Synced .dockerignore with .gitignore [skip ci] 2025-07-09 16:22:43 -07:00
Andrew Kane
a7c49d8283 Updated readme [skip ci] 2025-07-01 11:13:29 -07:00
Andrew Kane
ae9ee81e4d Fixed relaxed results example for Postgres 17 - #862 [skip ci] 2025-07-01 10:11:50 -07:00
Andrew Kane
fa1dee4e3b Updated link [skip ci] 2025-07-01 03:35:26 -07:00
Andrew Kane
e6bad96a03 Ran pgindent [skip ci] 2025-06-18 20:07:46 -07:00
Jon Daniel
3a49d141b3 Vectorizing vector_concat for improved performance (#861)
* Vectorizing vector_concat for improved performance

On an ARM chip this should generate SIMD instructions to copy the two
incoming vectors to the new vector as opposed to doing it all in
software.

* Moving declarations to above CheckDim

* Removing const from dims

* Formatting
2025-06-18 20:06:32 -07:00
Andrew Kane
ce09c9a27a Improved variable names [skip ci] 2025-06-18 17:32:37 -07:00
Andrew Kane
870ca6724d Fixed CI [skip ci] 2025-06-18 16:26:30 -07:00
Andrew Kane
8ad680f009 Ran pgindent [skip ci] 2025-06-18 16:11:15 -07:00
Jon Daniel
fe697e8788 vectorize: optimize VectorSumCenter and HalfvecSumCenter (#860)
* vectorize: optimize VectorSumCenter and HalfvecSumCenter

The functions VectorSumCenter and HalfvecSumCenter were not being
vectorized by the compiler. A few slight changes will allow these
optimizations to take place and get a performance boost by utilizing
SIMD instructions.

This optimization helps improve performance of vector operations in IVF
index building and updating.

* Removing const, commenting that it is only vectoirzed on ARM
2025-06-18 16:09:43 -07:00
Andrew Kane
bf28ed8176 Set random seed for IVFFLAT_BENCH [skip ci] 2025-06-18 10:57:18 -07:00
Andrew Kane
799cfebf70 Updated readme [skip ci] 2025-06-04 14:29:53 -07:00
Andrew Kane
3cd1f09f66 Updated Windows installation instructions - #593 [skip ci] 2025-06-04 14:18:53 -07:00
Andrew Kane
e2efe62fe5 Updated readme [skip ci] 2025-05-12 13:11:43 -07:00
Andrew Kane
7b58352336 Updated readme [skip ci] 2025-05-06 21:39:51 -07:00
Andrew Kane
83d410eae9 Updated readme [skip ci] 2025-04-13 13:07:31 -07:00
Andrew Kane
ebbfe8dba0 Fixed CI for Postgres 18 [skip ci] 2025-04-05 13:05:27 -07:00
Andrew Kane
e575866297 Revert "Fixed warnings with Postgres 18 [skip ci]"
This reverts commit 32e95a8598.
2025-04-05 12:56:00 -07:00
Andrew Kane
35f4f7fc80 Improved warning check [skip ci] 2025-04-05 12:38:30 -07:00
Andrew Kane
32e95a8598 Fixed warnings with Postgres 18 [skip ci] 2025-04-05 12:13:38 -07:00
Andrew Kane
a03dc5b7d0 Added fields to IndexAmRoutine for Postgres 18 [skip ci] 2025-04-05 11:31:57 -07:00
Andrew Kane
d365aac370 Added note about index options to index build time docs - #807 [skip ci] 2025-03-26 11:56:38 -07:00
Andrew Kane
05182479a2 Added ARM to CI [skip ci] 2025-03-24 19:26:52 -07:00
Andrew Kane
cc0958dec5 Removed deprecated runner image [skip ci] 2025-03-24 19:18:01 -07:00
Andrew Kane
4af2b06dc5 Improved installation note [skip ci] 2025-03-23 14:59:28 -07:00
38 changed files with 619 additions and 183 deletions

View File

@@ -1,8 +0,0 @@
/.git/
/dist/
/results/
/tmp_check/
/sql/vector--?.?.?.sql
regression.*
*.o
*.so

View File

@@ -8,27 +8,29 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 19
os: ubuntu-24.04
- postgres: 18
os: ubuntu-24.04
- postgres: 17
os: ubuntu-24.04
- postgres: 16
os: ubuntu-22.04
os: ubuntu-24.04-arm
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-20.04
os: ubuntu-22.04-arm
- postgres: 13
os: ubuntu-20.04
os: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
@@ -46,18 +48,18 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 17
os: macos-15
- postgres: 18
os: macos-26
- postgres: 14
os: macos-13
os: macos-15-intel
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
- run: make install
- run: make installcheck
- if: ${{ failure() }}
@@ -70,27 +72,35 @@ jobs:
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 17 && 'REL_17_2' || 'REL_14_15' }}
TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
env:
LLVM_VERSION: ${{ matrix.os == 'macos-15' && 18 || 15 }}
LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'mac') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: windows-2025
- postgres: 14
os: windows-2022
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
postgres-version: ${{ matrix.postgres }}
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
@@ -123,10 +133,10 @@ jobs:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 17
postgres-version: 18
check-ub: yes
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install

View File

@@ -1,3 +1,14 @@
## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build
- Improved `install` target on Windows
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
## 0.8.1 (2025-09-04)
- Added support for Postgres 18 rc1
- Improved performance of `binary_quantize` function
## 0.8.0 (2024-10-30)
- Added support for iterative index scans

View File

@@ -1,8 +1,11 @@
# syntax=docker/dockerfile:1
ARG PG_MAJOR=17
FROM postgres:$PG_MAJOR
ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR
COPY . /tmp/pgvector
ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
RUN apt-get update && \
apt-mark hold locales && \

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.0",
"version": "0.8.2",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.8.0",
"version": "0.8.2",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.0
EXTVERSION = 0.8.2
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)
@@ -27,6 +27,11 @@ ifneq ($(filter ppc64%, $(shell uname -m)), )
OPTFLAGS =
endif
# RISC-V64 doesn't support -march=native
ifeq ($(shell uname -m), riscv64)
OPTFLAGS =
endif
# For auto-vectorization:
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -76,4 +81,9 @@ docker:
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
.PHONY: docker-release-trixie
docker-release-trixie:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.0
EXTVERSION = 0.8.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
@@ -31,6 +31,9 @@ LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
# Use $(PGROOT)\bin\pg_regress for Postgres < 17
PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
@@ -54,11 +57,11 @@ install: all
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
installcheck:
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
uninstall:
del /f "$(PKGLIBDIR)\$(SHLIB)"

108
README.md
View File

@@ -21,7 +21,7 @@ Compile and install the extension (supports Postgres 13+)
```sh
cd /tmp
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -29,24 +29,16 @@ make install # may need sudo
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
```cmd
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
```
Note: The exact path will vary depending on your Visual Studio version and edition
Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
set "PGROOT=C:\Program Files\PostgreSQL\18"
cd %TEMP%
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -318,7 +310,9 @@ You can also speed up index creation by increasing the number of parallel worker
SET max_parallel_maintenance_workers = 7; -- plus leader
```
For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
### Indexing Progress
@@ -475,8 +469,6 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate
## Iterative Index Scans
*Added in 0.8.0*
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
Iterative scans can use strict or relaxed ordering.
@@ -500,9 +492,11 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o
```sql
WITH relaxed_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
) SELECT * FROM relaxed_results ORDER BY distance;
) SELECT * FROM relaxed_results ORDER BY distance + 0;
```
Note: `+ 0` is needed for Postgres 17+
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
```sql
@@ -701,10 +695,10 @@ CREATE INDEX CONCURRENTLY ...
### Querying
Use `EXPLAIN ANALYZE` to debug performance.
Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
```sql
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
#### Exact Search
@@ -754,8 +748,6 @@ SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_t
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Monitor recall by comparing results from approximate search with exact search.
```sql
@@ -779,9 +771,12 @@ Use pgvector from any language with a Postgres client. You can even generate and
Language | Libraries / Examples
--- | ---
Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol)
C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
@@ -793,19 +788,23 @@ Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl)
Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig)
## Frequently Asked Questions
@@ -820,11 +819,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
#### Can I store vectors with different dimensions in the same column?
You can use `vector` as the type (instead of `vector(3)`).
You can use `vector` as the type (instead of `vector(n)`).
```sql
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -1072,7 +1071,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1083,11 +1082,11 @@ sudo --preserve-env=PG_CONFIG make install
A few common paths on Mac are:
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Missing Header
@@ -1096,10 +1095,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use:
```sh
sudo apt install postgresql-server-dev-17
sudo apt install postgresql-server-dev-18
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Missing SDK
@@ -1129,7 +1128,7 @@ If compilation fails with `Cannot open include file: 'postgres.h': No such file
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure `vcvars64.bat` was called.
If compilation fails with `error C2196: case value '4' already used`, make sure youre using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
@@ -1146,17 +1145,32 @@ If installation fails with `Access is denied`, re-run the installation instructi
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh
docker pull pgvector/pgvector:pg17
docker pull pgvector/pgvector:pg18-trixie
```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
Supported tags are:
- `pg18-trixie`, `0.8.2-pg18-trixie`
- `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
- `pg17-trixie`, `0.8.2-pg17-trixie`
- `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
- `pg16-trixie`, `0.8.2-pg16-trixie`
- `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
- `pg15-trixie`, `0.8.2-pg15-trixie`
- `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
- `pg14-trixie`, `0.8.2-pg14-trixie`
- `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
- `pg13-trixie`, `0.8.2-pg13-trixie`
- `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
You can also build the image manually:
```sh
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
@@ -1173,7 +1187,7 @@ With Homebrew Postgres, you can use:
brew install pgvector
```
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
### PGXN
@@ -1188,29 +1202,29 @@ pgxn install vector
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh
sudo apt install postgresql-17-pgvector
sudo apt install postgresql-18-pgvector
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh
sudo yum install pgvector_17
sudo yum install pgvector_18
# or
sudo dnf install pgvector_17
sudo dnf install pgvector_18
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql16-pgvector
pkg install postgresql17-pgvector
```
or the port with:
@@ -1220,6 +1234,14 @@ cd /usr/ports/databases/pgvector
make install
```
### APK
Install the Alpine package with:
```sh
apk add postgresql-pgvector
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit

View File

@@ -2,6 +2,7 @@
#include "bitutils.h"
#include "bitvec.h"
#include "fmgr.h"
#include "utils/varbit.h"
#include "vector.h"

View File

@@ -1,5 +1,7 @@
#include "postgres.h"
#include <math.h>
#include "halfutils.h"
#include "halfvec.h"

View File

@@ -13,12 +13,20 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/numeric.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -129,9 +137,9 @@ InitHalfVector(int dim)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define halfvec_isspace(ch) scanner_isspace(ch)
#else
static inline bool
halfvec_isspace(char ch)
{
@@ -144,6 +152,7 @@ halfvec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -898,8 +907,21 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
half *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized on aarch64 */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -1,18 +1,24 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -52,12 +58,20 @@ HnswInitLockTranche(void)
sizeof(int) * 1,
&found);
if (!found)
{
#if PG_VERSION_NUM >= 190000
tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
#else
tranche_ids[0] = LWLockNewTrancheId();
#endif
}
hnsw_lock_tranche_id = tranche_ids[0];
LWLockRelease(AddinShmemInitLock);
#if PG_VERSION_NUM < 190000
/* Per-backend registration of the tranche ID */
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
#endif
}
/*
@@ -130,7 +144,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NULL)
if (path->indexorderbys == NIL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -252,6 +266,64 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
Datum
hnswhandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 3,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = hnswbuild,
.ambuildempty = hnswbuildempty,
.aminsert = hnswinsert,
.aminsertcleanup = NULL,
.ambulkdelete = hnswbulkdelete,
.amvacuumcleanup = hnswvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = hnswcostestimate,
.amgettreeheight = NULL,
.amoptions = hnswoptions,
.amproperty = NULL,
.ambuildphasename = hnswbuildphasename,
.amvalidate = hnswvalidate,
.amadjustmembers = NULL,
.ambeginscan = hnswbeginscan,
.amrescan = hnswrescan,
.amgettuple = hnswgettuple,
.amgetbitmap = NULL,
.amendscan = hnswendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -259,6 +331,11 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -291,6 +368,9 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL;
amroutine->amcostestimate = hnswcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = hnswoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = hnswbuildphasename;
@@ -311,5 +391,11 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -3,6 +3,8 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "lib/pairingheap.h"
@@ -12,6 +14,10 @@
#include "utils/sampling.h"
#include "vector.h"
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000
@@ -424,7 +430,7 @@ void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);

View File

@@ -36,11 +36,12 @@
*/
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
@@ -48,11 +49,18 @@
#include "commands/progress.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "tcop/tcopprot.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -69,6 +77,8 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
/*
* Create the metapage
*/
@@ -398,7 +408,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
* Update graph in memory
*/
static void
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
{
HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea;
@@ -460,7 +470,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
/* Update graph in memory */
UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);
UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
/* Release entry lock */
LWLockRelease(entryLock);
@@ -481,6 +491,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
LWLock *flushLock = &graph->flushLock;
char *base = buildstate->hnswarea;
Datum value;
Size memoryMargin;
/* Form index value */
if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
@@ -489,6 +500,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Get datum size */
valueSize = VARSIZE_ANY(DatumGetPointer(value));
/* In a parallel build, add a margin so allocations never fail */
memoryMargin = base == NULL ? 0 : 1024 * 1024;
/* Ensure graph not flushed when inserting */
LWLockAcquire(flushLock, LW_SHARED);
@@ -510,7 +524,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Check that we have enough memory available for the new element now that
* we have the allocator lock, and flush pages if needed.
*/
if (graph->memoryUsed >= graph->memoryTotal)
if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
{
LWLockRelease(&graph->allocatorLock);
@@ -545,7 +559,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Copy the datum */
memcpy(valuePtr, DatumGetPointer(value), valueSize);
HnswPtrStore(base, element->value, valuePtr);
HnswPtrStore(base, element->value, (char *) valuePtr);
/* Create a lock for the element */
LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -603,7 +617,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
graph->memoryTotal = memoryTotal;
graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
graph->flushed = false;
graph->indtuples = 0;
SpinLockInit(&graph->lock);
@@ -644,9 +658,17 @@ static void *
HnswSharedMemoryAlloc(Size size, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
void *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
Size alignedSize = MAXALIGN(size);
void *chunk;
buildstate->graph->memoryUsed += MAXALIGN(size);
if (alignedSize > 1024 * 1024)
elog(ERROR, "hnsw allocation too large");
if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
elog(ERROR, "hnsw allocator out of memory");
chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
buildstate->graph->memoryUsed += alignedSize;
return chunk;
}
@@ -932,6 +954,8 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
if (esthnswarea > estother)
esthnswarea -= estother;
esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
shm_toc_estimate_keys(&pcxt->estimator, 2);
@@ -974,8 +998,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
snapshot);
hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
/*
* Avoid base address for relptr for Postgres < 14.5
@@ -1054,7 +1077,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
* Build graph
*/
static void
BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
BuildGraph(HnswBuildState * buildstate)
{
int parallel_workers = 0;
@@ -1102,7 +1125,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
InitBuildState(buildstate, heap, index, indexInfo, forkNum);
BuildGraph(buildstate, forkNum);
BuildGraph(buildstate);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);

View File

@@ -1,13 +1,18 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "hnsw.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Get the insert page
@@ -660,7 +665,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk
*/
static void
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
{
BlockNumber newInsertPage = InvalidBlockNumber;
@@ -708,7 +713,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
/* Create an element */
element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
HnswPtrStore(base, element->value, DatumGetPointer(value));
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
/* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -728,7 +733,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
/* Update graph on disk */
UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);
UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);

View File

@@ -1,12 +1,21 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/relscan.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/float.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Algorithm 5 from paper
@@ -193,6 +202,10 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -2,18 +2,24 @@
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "catalog/pg_type.h"
#include "catalog/pg_type_d.h"
#include "common/hashfn.h"
#include "fmgr.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "nodes/pg_list.h"
#include "port/atomics.h"
#include "sparsevec.h"
#include "storage/bufmgr.h"
#include "utils/datum.h"
#include "utils/memdebug.h"
#include "utils/rel.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM < 170000
static inline uint64
@@ -256,7 +262,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
HnswInitNeighbors(base, element, m, allocator);
HnswPtrStore(base, element->value, (Pointer) NULL);
HnswPtrStore(base, element->value, (char *) NULL);
return element;
}
@@ -282,7 +288,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
element->blkno = blkno;
element->offno = offno;
HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL);
HnswPtrStore(base, element->value, (Pointer) NULL);
HnswPtrStore(base, element->value, (char *) NULL);
return element;
}
@@ -508,7 +514,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
char *base = NULL;
Datum value = datumCopy(PointerGetDatum(&etup->data), false, -1);
HnswPtrStore(base, element->value, DatumGetPointer(value));
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
}
}
@@ -922,7 +928,7 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
continue;
}
if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
if (!(eDistance < f->distance || alwaysAdd))
{
if (discarded != NULL)
{

View File

@@ -1,13 +1,18 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "nodes/pg_list.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
@@ -174,7 +179,12 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */
if (!needsUpdated)
{
/* Keep clang-tidy happy */
Assert(ntup->count > 0);
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
}
UnlockReleaseBuffer(buf);
@@ -524,8 +534,9 @@ MarkDeleted(HnswVacuumState * vacuumstate)
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Overwrite element */
/* Use memset instead of MemSet to keep clang-tidy happy */
etup->deleted = 1;
MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
memset(&etup->data, 0, VARSIZE_ANY(&etup->data));
/* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++)

View File

@@ -2,23 +2,36 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/parallel.h"
#include "access/xact.h"
#include "bitvec.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "halfvec.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "tcop/tcopprot.h"
#include "utils/memutils.h"
#include "vector.h"
#include "utils/rel.h"
#include "utils/sampling.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -138,7 +151,7 @@ SampleRows(IvfflatBuildState * buildstate)
* Add tuple to sort
*/
static void
AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
{
double distance;
double minDistance = DBL_MAX;
@@ -215,7 +228,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add tuple to sort */
AddTupleToSort(index, tid, values, buildstate);
AddTupleToSort(tid, values, buildstate);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
@@ -470,8 +483,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
* Create list pages
*/
static void
CreateListPages(Relation index, VectorArray centers, int dimensions,
int lists, ForkNumber forkNum, ListInfo * *listInfo)
CreateListPages(Relation index, VectorArray centers, int lists,
ForkNumber forkNum, ListInfo * *listInfo)
{
Buffer buf;
Page page;
@@ -1004,7 +1017,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
/* Create pages */
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
@@ -1023,6 +1036,10 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
IndexBuildResult *result;
IvfflatBuildState buildstate;
#ifdef IVFFLAT_BENCH
SeedRandom(42);
#endif
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

View File

@@ -3,14 +3,19 @@
#include <float.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/pg_list.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -92,7 +97,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NULL)
if (path->indexorderbys == NIL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -179,6 +184,64 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 5,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = ivfflatbuild,
.ambuildempty = ivfflatbuildempty,
.aminsert = ivfflatinsert,
.aminsertcleanup = NULL,
.ambulkdelete = ivfflatbulkdelete,
.amvacuumcleanup = ivfflatvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = ivfflatcostestimate,
.amgettreeheight = NULL,
.amoptions = ivfflatoptions,
.amproperty = NULL,
.ambuildphasename = ivfflatbuildphasename,
.amvalidate = ivfflatvalidate,
.amadjustmembers = NULL,
.ambeginscan = ivfflatbeginscan,
.amrescan = ivfflatrescan,
.amgettuple = ivfflatgettuple,
.amgetbitmap = NULL,
.amendscan = ivfflatendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -186,6 +249,11 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -218,6 +286,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
amroutine->amcostestimate = ivfflatcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = ivfflatoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -238,5 +309,11 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -13,6 +13,10 @@
#include "utils/tuplesort.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
@@ -21,6 +25,10 @@
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define IVFFLAT_MAX_DIM 2000
/* Support functions */
@@ -73,9 +81,11 @@
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#define SeedRandom(seed) srandom(seed)
#endif
/* Variables */

View File

@@ -2,11 +2,16 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
/*
* Find the list that minimizes the distance function
@@ -65,7 +70,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
* Insert a tuple into the index
*/
static void
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
{
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
IndexTuple itup;
@@ -204,7 +209,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
oldCtx = MemoryContextSwitchTo(insertCtx);
/* Insert tuple */
InsertTuple(index, values, isnull, heap_tid, heap);
InsertTuple(index, values, isnull, heap_tid);
/* Delete memory context */
MemoryContextSwitchTo(oldCtx);

View File

@@ -1,17 +1,19 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "bitvec.h"
#include "halfutils.h"
#include "halfvec.h"
#include "access/genam.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "vector.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Initialize with kmeans++

View File

@@ -2,15 +2,26 @@
#include <float.h>
#include "access/genam.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/tupdesc.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "fmgr.h"
#include "lib/pairingheap.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
@@ -355,6 +366,10 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -1,13 +1,19 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "bitvec.h"
#include "catalog/pg_type.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate a vector array
@@ -259,8 +265,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = x[k];
for (int i = 0; i < dimensions; i++)
vec->x[i] = x[i];
}
static void
@@ -271,8 +277,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = Float4ToHalfUnchecked(x[k]);
for (int i = 0; i < dimensions; i++)
vec->x[i] = Float4ToHalfUnchecked(x[i]);
}
static void
@@ -284,29 +290,33 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (uint32 k = 0; k < VARBITBYTES(vec); k++)
nx[k] = 0;
for (uint32 i = 0; i < VARBITBYTES(vec); i++)
nx[i] = 0;
for (int k = 0; k < dimensions; k++)
nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
for (int i = 0; i < dimensions; i++)
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
}
static void
VectorSumCenter(Pointer v, float *x)
{
Vector *vec = (Vector *) v;
int dim = vec->dim;
for (int k = 0; k < vec->dim; k++)
x[k] += vec->x[k];
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
x[i] += vec->x[i];
}
static void
HalfvecSumCenter(Pointer v, float *x)
{
HalfVector *vec = (HalfVector *) v;
int dim = vec->dim;
for (int k = 0; k < vec->dim; k++)
x[k] += HalfToFloat4(vec->x[k]);
/* Auto-vectorized on aarch64 */
for (int i = 0; i < dim; i++)
x[i] += HalfToFloat4(vec->x[i]);
}
static void
@@ -314,8 +324,8 @@ BitSumCenter(Pointer v, float *x)
{
VarBit *vec = (VarBit *) v;
for (int k = 0; k < VARBITLEN(vec); k++)
x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
for (int i = 0; i < VARBITLEN(vec); i++)
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
}
/*

View File

@@ -1,9 +1,12 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "commands/vacuum.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)

View File

@@ -5,18 +5,27 @@
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
typedef struct SparseInputElement
{
int32 index;
@@ -150,9 +159,9 @@ InitSparseVector(int dim, int nnz)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define sparsevec_isspace(ch) scanner_isspace(ch)
#else
static inline bool
sparsevec_isspace(char ch)
{
@@ -165,6 +174,7 @@ sparsevec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Compare indices

View File

@@ -16,16 +16,20 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/numeric.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -35,7 +39,11 @@
#define VECTOR_TARGET_CLONES
#endif
#if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.2");
#else
PG_MODULE_MAGIC;
#endif
/*
* Initialize index options and variables
@@ -125,9 +133,9 @@ InitVector(int dim)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define vector_isspace(ch) scanner_isspace(ch)
#else
static inline bool
vector_isspace(char ch)
{
@@ -140,6 +148,7 @@ vector_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -920,11 +929,13 @@ vector_concat(PG_FUNCTION_ARGS)
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++)
result->x[i] = a->x[i];
for (int i = 0; i < b->dim; i++)
result->x[i + a->dim] = b->x[i];
/* Auto-vectorized */
for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
result->x[i + start] = b->x[i];
PG_RETURN_POINTER(result);
}
@@ -940,8 +951,21 @@ binary_quantize(PG_FUNCTION_ARGS)
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (ax[i + j] > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -540,6 +540,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec)
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
subvector
-----------

View File

@@ -576,6 +576,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------

View File

@@ -121,6 +121,7 @@ SELECT l2_normalize('[65504]'::halfvec);
SELECT binary_quantize('[1,0,-1]'::halfvec);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);

View File

@@ -128,6 +128,7 @@ SELECT l2_normalize('[3e38]'::vector);
SELECT binary_quantize('[1,0,-1]'::vector);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);

View File

@@ -16,29 +16,19 @@ $node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
for my $i (0 .. $#operators)
# Test 100% recall
for (1 .. 20)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test 100% recall
for (1 .. 20)
{
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
done_testing();

View File

@@ -0,0 +1,29 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 1000) i;"
);
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '3073kB';
ALTER TABLE tst SET (parallel_workers = 1);
CREATE INDEX ON tst USING hnsw (v vector_l2_ops);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem after 0 tuples/);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.0'
default_version = '0.8.2'
module_pathname = '$libdir/vector'
relocatable = true