Compare commits

...

179 Commits

Author SHA1 Message Date
Andrew Kane
0e557b1d18 Updated readme [skip ci] 2026-07-10 15:16:36 -07:00
Andrew Kane
769a60884c Updated readme [skip ci] 2026-07-10 14:20:26 -07:00
Andrew Kane
8711840058 Added section on multitenancy [skip ci] 2026-07-10 14:13:13 -07:00
Andrew Kane
159b79aaad Version bump to 0.8.5 [skip ci] 2026-07-08 16:03:40 -07:00
Andrew Kane
c5a277a975 Hardened VectorArrayInit 2026-07-08 10:15:21 -07:00
Andrew Kane
124f6c61a4 Added Postgres 20 to CI [skip ci] 2026-07-07 16:15:39 -07:00
Andrew Kane
5ca52d12b6 Updated checkout action [skip ci] 2026-07-07 15:46:41 -07:00
Andrew Kane
1f61d1111b Added ubuntu-26.04 to CI [skip ci] 2026-07-07 15:46:06 -07:00
Andrew Kane
fb1b8966eb Added comment to IVFFlat vacuuming test [skip ci] 2026-07-01 17:21:26 -07:00
Andrew Kane
71ce9d3311 Added test for concurrent INSERTs, DELETEs, SELECTs, and VACUUM with IVFFlat [skip ci] 2026-07-01 17:18:34 -07:00
Andrew Kane
d19cc0d371 Improved HNSW vacuuming test [skip ci] 2026-07-01 17:14:53 -07:00
Andrew Kane
89fda3e100 Removed slow tests [skip ci] 2026-07-01 16:05:53 -07:00
Andrew Kane
30d8654b47 Added tests for HNSW index dimensions for bit type [skip ci] 2026-07-01 15:49:07 -07:00
Andrew Kane
a846385cc7 Added tests for index dimensions for bit type [skip ci] 2026-07-01 15:47:05 -07:00
Andrew Kane
98d7c4124e Added IVFFlat memory tests for bit [skip ci] 2026-07-01 15:45:18 -07:00
Andrew Kane
2a2b4a0b58 Updated changelog [skip ci] 2026-07-01 12:35:46 -07:00
Andrew Kane
f15a50387f Moved logic for calculating number of samples [skip ci] 2026-07-01 12:27:03 -07:00
Andrew Kane
971b7d7fd6 Added IVFFlat memory tests for halfvec [skip ci] 2026-07-01 12:21:08 -07:00
Andrew Kane
f51d8ed989 Improved readability of options tests [skip ci] 2026-07-01 12:17:50 -07:00
Andrew Kane
a76a18d526 Added tests for index dimensions [skip ci] 2026-07-01 12:15:27 -07:00
Andrew Kane
b383e4d191 Reduced memory usage for small tables for IVFFlat index builds - resolves #995 and resolves #996
Co-authored-by: Itai Spiegel <itai@mave.com>
2026-07-01 11:54:21 -07:00
Andrew Kane
1d458ad5d7 Version bump to 0.8.4 [skip ci] 2026-06-30 15:24:14 -07:00
Andrew Kane
9fa17c10b8 Added comments to HNSW vacuuming tests [skip ci] 2026-06-30 13:40:32 -07:00
Andrew Kane
14149b19f5 Added SELECTs to HNSW vacuuming test for good measure [skip ci] 2026-06-30 13:34:11 -07:00
Andrew Kane
34d796fbab Added test for concurrent INSERTs, DELETEs, and VACUUM with HNSW - #993 2026-06-30 13:23:04 -07:00
Andrew Kane
53341bb6c7 Simplified test [skip ci] 2026-06-30 13:16:17 -07:00
Andrew Kane
0d9720f440 Added test for concurrent SELECTs and VACUUM [skip ci] 2026-06-30 13:12:47 -07:00
Andrew Kane
cb246cb72d Improved naming [skip ci] 2026-06-30 02:46:31 -07:00
Andrew Kane
d053de2d94 Removed deletion list check in MarkDeleted (does not help safety) [skip ci] 2026-06-30 02:31:54 -07:00
Andrew Kane
83bac90869 Added checks for deleted tuples rather than relying on ItemPointerIsValid [skip ci] 2026-06-30 01:40:30 -07:00
Andrew Kane
4eca5024df Updated comment [skip ci] 2026-06-30 01:35:07 -07:00
Andrew Kane
a31771bc45 Changed log message to assertion [skip ci] 2026-06-30 01:25:36 -07:00
Andrew Kane
ecddde963a Added check to confirm in deletion list before marking as deleted 2026-06-30 01:15:38 -07:00
Andrew Kane
ecd413d0fe Improved naming [skip ci] 2026-06-30 01:03:08 -07:00
Andrew Kane
497db7976c Fixed hnsw graph not repaired error with HNSW vacuuming - fixes #993 2026-06-30 00:34:01 -07:00
Andrew Kane
f1dd4e3b03 Updated urls [skip ci] 2026-06-29 01:08:14 -07:00
Andrew Kane
7d067d7b83 Updated changelog [skip ci] 2026-06-24 11:44:04 -07:00
Andrew Kane
d4dd73d970 Moved repair confirmation after lock wait to catch more potential issues 2026-06-23 11:39:16 -07:00
Bhagyesh Chaturvedi
ffe28bb954 Fix HNSW insert and vacuum race 2026-06-23 06:00:23 +00:00
Andrew Kane
0dbc1a27c0 Removed redundant check [skip ci] 2026-06-18 12:56:57 -07:00
Andrew Kane
08c4e7ff10 Hardened VectorArrayGet and VectorArraySet [skip ci] 2026-06-18 12:56:12 -07:00
Andrew Kane
6731c49811 Fixed overflow check (should never be hit) [skip ci] 2026-06-18 12:51:38 -07:00
Andrew Kane
f2617f02d1 Updated style to be consistent with latest Postgres [skip ci] 2026-06-18 12:44:11 -07:00
Andrew Kane
bdf19077db Ensure centers and samples fit into maintenance_work_mem before allocating for IVFFlat index builds - closes #986 2026-06-18 12:29:58 -07:00
Andrew Kane
90cd2b4ee5 Improved memory tracking for IVFFlat index builds [skip ci] 2026-06-18 11:57:03 -07:00
Andrew Kane
b44d1b4c5f Added todo [skip ci] 2026-06-18 11:51:37 -07:00
Andrew Kane
cc5b865c33 Added itemsize to IvfflatBuildState [skip ci] 2026-06-18 11:49:29 -07:00
Andrew Kane
4895021088 Hardened NeedsUpdated [skip ci] 2026-06-18 11:19:27 -07:00
Andrew Kane
eda77b3492 DRY normalize code for IVFFlat index builds 2026-06-18 11:16:51 -07:00
Andrew Kane
a2364b1793 Switched to VectorArraySet for NormCenters [skip ci] 2026-06-18 11:05:37 -07:00
Andrew Kane
a0eaf70d17 Hardened VectorArraySet [skip ci] 2026-06-18 11:03:09 -07:00
Andrew Kane
586e7515ba Version bump to 0.8.3 [skip ci] 2026-06-17 21:26:13 -07:00
Andrew Kane
a619d0b34d Simplified logic for HNSW vacuum progress [skip ci] 2026-06-17 12:18:17 -07:00
Andrew Kane
ad222abf48 Improved HNSW vacuum progress for round block sizes [skip ci] 2026-06-16 14:21:23 -07:00
Andrew Kane
8a81a3fe3a Added HNSW_VACUUM_PROGRESS flag [skip ci] 2026-06-16 13:53:40 -07:00
Andrew Kane
fc804925a4 Added todo [skip ci] 2026-06-16 12:33:25 -07:00
Andrew Kane
98dc4aa6d0 Added pass to confirm graph was repaired for HNSW vacuuming 2026-06-16 10:51:54 -07:00
Andrew Kane
7b72aeeff3 Added check for deleted element [skip ci] 2026-06-16 10:13:40 -07:00
Andrew Kane
4682d51e5d Added memory usage for HNSW vacuuming [skip ci] 2026-06-16 10:10:13 -07:00
Andrew Kane
91fe2e62e7 Added benchmarking for HNSW vacuuming [skip ci] 2026-06-16 09:53:47 -07:00
Andrew Kane
eedba7ee14 Fixed possible index corruption with HNSW vacuuming - resolves #988
Co-authored-by: Bhagyesh Chaturvedi <bhagyeshc@google.com>
2026-06-16 09:44:34 -07:00
Andrew Kane
32284ba28a Updated CI for latest windows-2025 image [skip ci] 2026-06-15 19:24:02 -07:00
Andrew Kane
1f68c73c96 Updated changelog [skip ci] 2026-06-10 20:20:55 -07:00
Andrew Kane
f15bc0904f Updated changelog [skip ci] 2026-06-10 20:13:55 -07:00
Andrew Kane
2b7ad083dc Always use inline pg_popcount64 for Postgres 19+ - #985 2026-06-10 20:08:03 -07:00
Andrew Kane
421a35fcdc Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18 - fixes #985 2026-06-10 19:55:21 -07:00
Andrew Kane
12368bd79c Updated readme [skip ci] 2026-05-30 11:58:10 -07:00
Andrew Kane
88a0085459 Improved casting [skip ci] 2026-05-26 14:09:19 -07:00
Andrew Kane
ea23884efd Ran latest pgindent [skip ci] 2026-05-26 13:39:55 -07:00
Andrew Kane
3351f3d43e Simplified comment [skip ci] 2026-05-26 13:39:20 -07:00
Andrew Kane
d238409bec Merge pull request #979 from xingtanzjr/fix_ivf_create_concurrently
Fix CREATE INDEX CONCURRENTLY crash on ivfflat with assertion-enabled PostgreSQL
2026-04-27 03:28:52 -07:00
Zhang Jinrui
529f37175b Fix CREATE INDEX CONCURRENTLY crash on ivfflat with assertion-enabled PostgreSQL by passing anyvisible=false in ivfflat SampleRows 2026-04-27 10:15:32 +00:00
Andrew Kane
bce3946392 Added link to PgHero [skip ci] 2026-04-27 02:11:17 -07:00
Andrew Kane
13cb253d30 Added link to PgDog [skip ci] 2026-04-27 00:09:20 -07:00
Andrew Kane
41b3cdc011 Updated readme [skip ci] 2026-04-26 14:47:17 -07:00
Andrew Kane
ec02a96239 Updated readme [skip ci] 2026-04-26 14:45:12 -07:00
Andrew Kane
610d95b8d2 Moved scaling section [skip ci] 2026-04-26 13:12:28 -07:00
Andrew Kane
609d01f4c6 Added more scaling advice to readme [skip ci] 2026-04-26 12:56:07 -07:00
Andrew Kane
a7551a61ca Updated style [skip ci] 2026-04-14 16:21:32 -07:00
lychen0613
5c9a97af3b optimize: defer sample normalization when building ivfflat 2026-04-14 16:14:17 -07:00
Andrew Kane
8af675cd25 Added comment [skip ci] 2026-04-14 16:10:45 -07:00
Andrew Kane
97e0ed4464 Fixed compilation errors with Postgres 19 2026-04-14 15:46:10 -07:00
Andrew Kane
533ad160e0 Fixed CI 2026-04-14 13:51:04 -07:00
Andrew Kane
dfbd724a1f Changed rowstoskip to double to match reservoir_get_next_S 2026-04-14 13:38:20 -07:00
lychen0613
5f270c9663 Add benchmark for ivfflat sample rows 2026-04-14 21:56:10 +08:00
lychen0613
352ec5be29 fix: correctly use reservoir_get_next_S when building ivfflat 2026-04-14 17:57:40 +08:00
Andrew Kane
17916cad00 Improved includes [skip ci] 2026-02-25 17:35:23 -08:00
Andrew Kane
284f2f0fec Improved casting for HNSW scan memory limit [skip ci] 2026-02-25 12:59:17 -08:00
Andrew Kane
edd49863ba Improved limit for HNSW scan memory 2026-02-25 12:52:29 -08:00
Andrew Kane
411291189d Updated changelog [skip ci] 2026-02-25 11:28:43 -08:00
Andrew Kane
cab9da72c0 Version bump to 0.8.2 [skip ci] 2026-02-25 10:46:57 -08:00
Andrew Kane
d702ee31f2 Added test case for low memory HNSW index builds 2026-02-25 10:41:17 -08:00
Andrew Kane
9ed39fb94b Hardened shared memory allocator
Co-authored-by: chungkn from OneMount Group <kimngocchung.k2a@gmail.com>
2026-02-25 10:39:01 -08:00
Andrew Kane
b7e680d41a Fixed integer wraparound leading to buffer overflow with parallel HNSW index build
Reported-by: chungkn from OneMount Group <kimngocchung.k2a@gmail.com>
2026-02-25 10:35:14 -08:00
Andrew Kane
f5841f07fd Updated CI 2026-02-23 14:06:35 -08:00
Andrew Kane
35ab919bf5 Switched to statically-allocated IndexAmRoutine for Postgres 19 [skip ci] 2026-01-21 16:40:35 -08:00
Andrew Kane
5c08be0291 Updated checkout action [skip ci] 2026-01-15 20:45:56 -08:00
Jack Huang
544686feb1 Add RISC-V64 architecture support in Makefile (#948) 2026-01-14 06:11:58 -08:00
Andrew Kane
1a8ebb1129 Updated license year [skip ci] 2026-01-05 13:01:58 -08:00
Andrew Kane
aca89aec73 Switched to scanner_isspace for Postgres 17+ - resolves #942 2026-01-05 12:54:04 -08:00
Andrew Kane
c9875f5150 Simplified test case - resolves #943 2026-01-05 12:42:14 -08:00
Andrew Kane
df2c3d301f Added link to pgvector-cobol [skip ci] 2025-12-13 11:33:35 -08:00
Andrew Kane
d557f1c9b1 Added link to pgvector-tcl [skip ci] 2025-12-11 23:22:56 -08:00
Andrew Kane
3d6dd20469 Removed unneeded check 2025-12-11 16:28:45 -08:00
Andrew Kane
c711da411c Improved includes for indexes 2025-12-11 15:35:37 -08:00
Andrew Kane
3975e212cc Improved includes for ivfkmeans [skip ci] 2025-12-11 13:26:36 -08:00
Andrew Kane
2b741ef068 Improved includes for types 2025-12-11 00:35:18 -08:00
Andrew Kane
f85a7d3fd0 Keep clang-tidy happy [skip ci] 2025-12-10 20:03:26 -08:00
Andrew Kane
5c25f81789 Fixed inconsistent parameter names [skip ci] 2025-12-10 18:49:00 -08:00
Andrew Kane
d72ee71f23 Fixed compilation errors with Postgres 19 2025-12-10 14:59:29 -08:00
Andrew Kane
b7c0085afe Added instructions for APK [skip ci] 2025-12-01 22:52:23 -08:00
Andrew Kane
2b742576b4 Added links [skip ci] 2025-11-24 12:41:30 -08:00
Andrew Kane
a126c02184 Added link to pgvector-ada [skip ci] 2025-10-30 22:29:36 -07:00
Andrew Kane
744362305b Updated changelog [skip ci] 2025-10-30 11:48:44 -07:00
Andrew Kane
69e78e36a3 Added link to pgvetor-pascal [skip ci] 2025-10-30 11:40:29 -07:00
Andrew Kane
8dd37d4fb2 Improved install target for Windows - #917 [skip ci] 2025-10-30 11:03:59 -07:00
Andrew Kane
a24125839a Updated readme [skip ci] 2025-10-28 10:25:34 -07:00
Andrew Kane
2ee3113417 Updated readme [skip ci] 2025-10-22 13:10:58 -07:00
Andrew Kane
3ebb9a506e Added varatt headers for Postgres 16+ 2025-10-22 11:56:05 -07:00
Andrew Kane
727d4836d2 Fixed Index Searches in EXPLAIN output for Postgres 18 2025-10-22 11:47:11 -07:00
Andrew Kane
d823c44591 Removed unused parameter [skip ci] 2025-09-27 15:58:18 -07:00
Andrew Kane
8f4aa0751c Removed unused parameters [skip ci] 2025-09-27 15:51:32 -07:00
Andrew Kane
1ff30720c7 Updated readme [skip ci] 2025-09-26 13:02:37 -07:00
Andrew Kane
4cef9213ec Added Docker images for Postgres 18 [skip ci] 2025-09-25 20:00:10 -07:00
Andrew Kane
90add68d6f Updated style to be consistent with Mac [skip ci] 2025-09-25 15:55:30 -07:00
Andrew Kane
bc05dbf312 Updated CI for Postgres 18 [skip ci] 2025-09-25 15:54:37 -07:00
Andrew Kane
02fefc0e3f Updated readme for Postgres 18 [skip ci] 2025-09-25 15:46:28 -07:00
Andrew Kane
e7899b1250 Updated checkout action [skip ci] 2025-09-22 11:55:01 -07:00
Andrew Kane
9e29dff78c Updated CI [skip ci] 2025-09-19 19:16:08 -07:00
Andrew Kane
13c0dbf530 Test with macos-15-intel on CI [skip ci] 2025-09-19 18:34:51 -07:00
Andrew Kane
c205a12107 Updated CI [skip ci] 2025-09-05 17:03:37 -07:00
Andrew Kane
db3755a58f Added windows-2025 to CI [skip ci] 2025-09-05 16:41:22 -07:00
Andrew Kane
778dacf20c Version bump to 0.8.1 [skip ci] 2025-09-04 17:51:09 -07:00
Andrew Kane
3f687687ee Fixed compilation error with Postgres 19 2025-09-04 15:58:23 -07:00
Andrew Kane
815f48e489 Updated changelog [skip ci] 2025-09-04 15:36:18 -07:00
Andrew Kane
67e648b13e Added another test for binary_quantize function [skip ci] 2025-08-29 01:09:18 -07:00
Andrew Kane
c3ff955231 Reordered supported Docker tags [skip ci] 2025-08-23 19:57:14 -07:00
Andrew Kane
6f46a1035d Added new Docker images to readme [skip ci] 2025-08-23 19:46:07 -07:00
Andrew Kane
bbe66e821b Added Docker images for Debian 13 / Trixie [skip ci] 2025-08-23 19:34:11 -07:00
Julien Rouhaud
dd3a1e9137 Use NIL for empty lists (#890)
Postgres standard way to check for list emptiness is to compare a pointer to
NIL rather than NULL.
2025-08-23 03:31:22 -07:00
Andrew Kane
ea4746f6c0 Updated comment [skip ci] 2025-08-19 14:26:36 -07:00
Andrew Kane
6aec80ccdb Improved performance of binary_quantize function for halfvec 2025-08-19 13:47:11 -07:00
Luca Giacchino
0c9070ba82 Add autovectorized implementation of binary quantize 2025-08-19 13:18:50 -07:00
Andrew Kane
30382418da Ran pgindent [skip ci] 2025-08-19 12:44:54 -07:00
Andrew Kane
26b50e536f Updated FreeBSD package name in readme [skip ci] 2025-08-09 12:09:29 -07:00
Andrew Kane
e29fc3aa1a Switched to PG_MODULE_MAGIC_EXT for Postgres 18+ [skip ci] 2025-07-31 19:59:27 -07:00
Andrew Kane
6ef7fccb5c Added Postgres 19 to CI [skip ci] 2025-07-27 19:50:46 -07:00
Andrew Kane
5b8b68ba1d Use consistent style [skip ci] 2025-07-27 18:19:28 -07:00
Andrew Kane
3be8693c13 Added supported Docker tags to readme [skip ci] 2025-07-27 13:44:59 -07:00
Andrew Kane
247c8dc8a5 Updated changelog [skip ci] 2025-07-27 13:25:51 -07:00
Andrew Kane
3600ab93e5 Added Docker images with -bookworm suffix [skip ci] 2025-07-27 13:14:29 -07:00
Andrew Kane
665db75a3c Updated Dockerfile to use release [skip ci] 2025-07-27 12:34:23 -07:00
Andrew Kane
44163d0a97 Revert "Added OS to Dockerfile [skip ci]"
This reverts commit 33ca8a61e2.
2025-07-09 22:01:34 -07:00
Andrew Kane
33ca8a61e2 Added OS to Dockerfile [skip ci] 2025-07-09 16:25:05 -07:00
Andrew Kane
742e2d1d28 Synced .dockerignore with .gitignore [skip ci] 2025-07-09 16:22:43 -07:00
Andrew Kane
a7c49d8283 Updated readme [skip ci] 2025-07-01 11:13:29 -07:00
Andrew Kane
ae9ee81e4d Fixed relaxed results example for Postgres 17 - #862 [skip ci] 2025-07-01 10:11:50 -07:00
Andrew Kane
fa1dee4e3b Updated link [skip ci] 2025-07-01 03:35:26 -07:00
Andrew Kane
e6bad96a03 Ran pgindent [skip ci] 2025-06-18 20:07:46 -07:00
Jon Daniel
3a49d141b3 Vectorizing vector_concat for improved performance (#861)
* Vectorizing vector_concat for improved performance

On an ARM chip this should generate SIMD instructions to copy the two
incoming vectors to the new vector as opposed to doing it all in
software.

* Moving declarations to above CheckDim

* Removing const from dims

* Formatting
2025-06-18 20:06:32 -07:00
Andrew Kane
ce09c9a27a Improved variable names [skip ci] 2025-06-18 17:32:37 -07:00
Andrew Kane
870ca6724d Fixed CI [skip ci] 2025-06-18 16:26:30 -07:00
Andrew Kane
8ad680f009 Ran pgindent [skip ci] 2025-06-18 16:11:15 -07:00
Jon Daniel
fe697e8788 vectorize: optimize VectorSumCenter and HalfvecSumCenter (#860)
* vectorize: optimize VectorSumCenter and HalfvecSumCenter

The functions VectorSumCenter and HalfvecSumCenter were not being
vectorized by the compiler. A few slight changes will allow these
optimizations to take place and get a performance boost by utilizing
SIMD instructions.

This optimization helps improve performance of vector operations in IVF
index building and updating.

* Removing const, commenting that it is only vectoirzed on ARM
2025-06-18 16:09:43 -07:00
Andrew Kane
bf28ed8176 Set random seed for IVFFLAT_BENCH [skip ci] 2025-06-18 10:57:18 -07:00
Andrew Kane
799cfebf70 Updated readme [skip ci] 2025-06-04 14:29:53 -07:00
Andrew Kane
3cd1f09f66 Updated Windows installation instructions - #593 [skip ci] 2025-06-04 14:18:53 -07:00
Andrew Kane
e2efe62fe5 Updated readme [skip ci] 2025-05-12 13:11:43 -07:00
Andrew Kane
7b58352336 Updated readme [skip ci] 2025-05-06 21:39:51 -07:00
Andrew Kane
83d410eae9 Updated readme [skip ci] 2025-04-13 13:07:31 -07:00
Andrew Kane
ebbfe8dba0 Fixed CI for Postgres 18 [skip ci] 2025-04-05 13:05:27 -07:00
Andrew Kane
e575866297 Revert "Fixed warnings with Postgres 18 [skip ci]"
This reverts commit 32e95a8598.
2025-04-05 12:56:00 -07:00
Andrew Kane
35f4f7fc80 Improved warning check [skip ci] 2025-04-05 12:38:30 -07:00
Andrew Kane
32e95a8598 Fixed warnings with Postgres 18 [skip ci] 2025-04-05 12:13:38 -07:00
Andrew Kane
a03dc5b7d0 Added fields to IndexAmRoutine for Postgres 18 [skip ci] 2025-04-05 11:31:57 -07:00
Andrew Kane
d365aac370 Added note about index options to index build time docs - #807 [skip ci] 2025-03-26 11:56:38 -07:00
Andrew Kane
05182479a2 Added ARM to CI [skip ci] 2025-03-24 19:26:52 -07:00
Andrew Kane
cc0958dec5 Removed deprecated runner image [skip ci] 2025-03-24 19:18:01 -07:00
Andrew Kane
4af2b06dc5 Improved installation note [skip ci] 2025-03-23 14:59:28 -07:00
57 changed files with 1359 additions and 313 deletions

View File

@@ -1,8 +0,0 @@
/.git/
/dist/
/results/
/tmp_check/
/sql/vector--?.?.?.sql
regression.*
*.o
*.so

View File

@@ -8,27 +8,31 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 20
os: ubuntu-26.04
- postgres: 19
os: ubuntu-26.04
- postgres: 18
os: ubuntu-24.04
os: ubuntu-26.04-arm
- postgres: 17
os: ubuntu-24.04
- postgres: 16
os: ubuntu-22.04
os: ubuntu-24.04-arm
- postgres: 15
os: ubuntu-22.04
- postgres: 14
os: ubuntu-20.04
os: ubuntu-22.04-arm
- postgres: 13
os: ubuntu-20.04
os: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
@@ -46,18 +50,18 @@ jobs:
fail-fast: false
matrix:
include:
- postgres: 17
os: macos-15
- postgres: 18
os: macos-26
- postgres: 14
os: macos-13
os: macos-15-intel
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- run: make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }}
- run: make install
- run: make installcheck
- if: ${{ failure() }}
@@ -70,27 +74,35 @@ jobs:
tar xf $TAG.tar.gz
mv postgres-$TAG postgres
env:
TAG: ${{ matrix.postgres == 17 && 'REL_17_2' || 'REL_14_15' }}
TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
env:
PERL5LIB: /Users/runner/perl5/lib/perl5
- run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make
env:
LLVM_VERSION: ${{ matrix.os == 'macos-15' && 18 || 15 }}
LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }}
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
runs-on: ${{ matrix.os }}
if: ${{ !startsWith(github.ref_name, 'mac') }}
strategy:
fail-fast: false
matrix:
include:
- postgres: 17
os: windows-2025
- postgres: 14
os: windows-2022
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
postgres-version: ${{ matrix.postgres }}
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
@@ -123,10 +135,10 @@ jobs:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 17
postgres-version: 18
check-ub: yes
- run: make OPTFLAGS=""
- run: sudo --preserve-env=PG_CONFIG make install

View File

@@ -1,3 +1,29 @@
## 0.8.5 (2026-07-08)
- Reduced memory usage for small tables for IVFFlat index builds
## 0.8.4 (2026-06-30)
- Fixed `hnsw graph not repaired` error with HNSW vacuuming
- Fixed possible error with inserts during HNSW vacuuming
- Fixed memory exceeding `maintenance_work_mem` with IVFFlat index builds
## 0.8.3 (2026-06-17)
- Fixed possible index corruption with HNSW vacuuming
- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)
- Improved `install` target on Windows
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
## 0.8.1 (2025-09-04)
- Added support for Postgres 18 rc1
- Improved performance of `binary_quantize` function
## 0.8.0 (2024-10-30)
- Added support for iterative index scans

View File

@@ -1,8 +1,11 @@
# syntax=docker/dockerfile:1
ARG PG_MAJOR=17
FROM postgres:$PG_MAJOR
ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR
COPY . /tmp/pgvector
ADD https://github.com/pgvector/pgvector.git#v0.8.5 /tmp/pgvector
RUN apt-get update && \
apt-mark hold locales && \

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,12 +2,12 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.0",
"version": "0.8.5",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
"license": {
"PostgreSQL": "http://www.postgresql.org/about/licence"
"PostgreSQL": "https://www.postgresql.org/about/licence"
},
"prereqs": {
"runtime": {
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.8.0",
"version": "0.8.5",
"abstract": "Open-source vector similarity search for Postgres"
}
},
@@ -38,7 +38,7 @@
"generated_by": "Andrew Kane",
"meta-spec": {
"version": "1.0.0",
"url": "http://pgxn.org/meta/spec.txt"
"url": "https://pgxn.org/meta/spec.txt"
},
"tags": [
"vectors",

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.0
EXTVERSION = 0.8.5
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)
@@ -27,6 +27,11 @@ ifneq ($(filter ppc64%, $(shell uname -m)), )
OPTFLAGS =
endif
# RISC-V64 doesn't support -march=native
ifeq ($(shell uname -m), riscv64)
OPTFLAGS =
endif
# For auto-vectorization:
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -76,4 +81,9 @@ docker:
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm .
.PHONY: docker-release-trixie
docker-release-trixie:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie .

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.0
EXTVERSION = 0.8.5
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
@@ -31,6 +31,9 @@ LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
# Use $(PGROOT)\bin\pg_regress for Postgres < 17
PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
@@ -54,11 +57,11 @@ install: all
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)"
installcheck:
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
"$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
uninstall:
del /f "$(PKGLIBDIR)\$(SHLIB)"

175
README.md
View File

@@ -11,6 +11,8 @@ Store your vectors with the rest of your data. Supports:
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
Have a lot of vectors? Use [quantization](#scaling) to scale
[![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions)
## Installation
@@ -21,7 +23,7 @@ Compile and install the extension (supports Postgres 13+)
```sh
cd /tmp
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -29,24 +31,16 @@ make install # may need sudo
See the [installation notes](#installation-notes---linux-and-mac) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run:
Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build:
```cmd
call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
```
Note: The exact path will vary depending on your Visual Studio version and edition
Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
set "PGROOT=C:\Program Files\PostgreSQL\18"
cd %TEMP%
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -318,7 +312,11 @@ You can also speed up index creation by increasing the number of parallel worker
SET max_parallel_maintenance_workers = 7; -- plus leader
```
For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default)
For a large number of workers, you may need to increase `max_parallel_workers` (8 by default)
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
Use [binary quantization](#binary-quantization) for faster build times at scale
### Indexing Progress
@@ -449,13 +447,7 @@ Exact indexes work well for conditions that match a low percentage of rows. Othe
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
```sql
SET hnsw.ef_search = 200;
```
Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
```sql
SET hnsw.iterative_scan = strict_order;
@@ -473,9 +465,17 @@ If filtering by many different values, consider [partitioning](https://www.postg
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
```
## Iterative Index Scans
## Multitenancy
*Added in 0.8.0*
For applications with multiple tenants, sharing an approximate index between tenants means vectors from one tenant can affect recall (and speed) for other tenants.
For tenant isolation, use [list partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) or separate tables.
```sql
CREATE TABLE items (customer_id int, embedding vector(3)) PARTITION BY LIST(customer_id);
```
## Iterative Index Scans
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
@@ -500,9 +500,11 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o
```sql
WITH relaxed_results AS MATERIALIZED (
SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5
) SELECT * FROM relaxed_results ORDER BY distance;
) SELECT * FROM relaxed_results ORDER BY distance + 0;
```
Note: `+ 0` is needed for Postgres 17+
For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor)
```sql
@@ -679,6 +681,10 @@ SHOW shared_buffers;
Be sure to restart Postgres for changes to take effect.
### Storing
Use the `halfvec` type instead of `vector` for a smaller working set.
### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
@@ -693,6 +699,8 @@ Add any indexes *after* loading the initial data for best performance.
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
In production environments, create indexes concurrently to avoid blocking writes.
```sql
@@ -701,10 +709,10 @@ CREATE INDEX CONCURRENTLY ...
### Querying
Use `EXPLAIN ANALYZE` to debug performance.
Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance.
```sql
EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
```
#### Exact Search
@@ -723,6 +731,8 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
#### Approximate Search
Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
```sql
@@ -738,23 +748,20 @@ REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name;
```
## Scaling
For a smaller working set:
1. Use the `halfvec` type instead of `vector` for tables
2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Monitoring
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
```sql
CREATE EXTENSION pg_stat_statements;
```
Get the most time-consuming queries with:
```sql
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13
Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance.
Monitor recall by comparing results from approximate search with exact search.
@@ -765,23 +772,18 @@ SELECT ...
COMMIT;
```
## Scaling
Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
Language | Libraries / Examples
--- | ---
Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada)
Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol)
C | [pgvector-c](https://github.com/pgvector/pgvector-c)
C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp)
C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet)
COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol)
Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal)
D | [pgvector-d](https://github.com/pgvector/pgvector-d)
Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart)
@@ -793,19 +795,23 @@ Go | [pgvector-go](https://github.com/pgvector/pgvector-go)
Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell)
Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java)
JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node)
Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia)
Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl)
Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp)
Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua)
Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim)
OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml)
Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal)
Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl)
PHP | [pgvector-php](https://github.com/pgvector/pgvector-php)
Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog)
Python | [pgvector-python](https://github.com/pgvector/pgvector-python)
R | [pgvector-r](https://github.com/pgvector/pgvector-r)
Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket)
Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku)
Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor)
Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust)
Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift)
Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl)
Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig)
## Frequently Asked Questions
@@ -820,11 +826,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions?
You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction).
#### Can I store vectors with different dimensions in the same column?
You can use `vector` as the type (instead of `vector(3)`).
You can use `vector` as the type (instead of `vector(n)`).
```sql
CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id));
@@ -879,6 +885,8 @@ No, but like other index types, youll likely see better performance if they d
SELECT pg_size_pretty(pg_relation_size('index_name'));
```
Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
## Troubleshooting
#### Why isnt a query using an index?
@@ -1072,7 +1080,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0
If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with:
```sh
export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config
export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config
```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
@@ -1083,11 +1091,11 @@ sudo --preserve-env=PG_CONFIG make install
A few common paths on Mac are:
- EDB installer - `/Library/PostgreSQL/17/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config`
- EDB installer - `/Library/PostgreSQL/18/bin/pg_config`
- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config`
- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config`
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Missing Header
@@ -1096,10 +1104,10 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use:
```sh
sudo apt install postgresql-server-dev-17
sudo apt install postgresql-server-dev-18
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Missing SDK
@@ -1129,7 +1137,7 @@ If compilation fails with `Cannot open include file: 'postgres.h': No such file
### Mismatched Architecture
If compilation fails with `error C2196: case value '4' already used`, make sure `vcvars64.bat` was called.
If compilation fails with `error C2196: case value '4' already used`, make sure youre using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions.
### Missing Symbol
@@ -1146,17 +1154,32 @@ If installation fails with `Access is denied`, re-run the installation instructi
Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with:
```sh
docker pull pgvector/pgvector:pg17
docker pull pgvector/pgvector:pg18-trixie
```
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way).
This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way).
Supported tags are:
- `pg18-trixie`, `0.8.5-pg18-trixie`
- `pg18-bookworm`, `0.8.5-pg18-bookworm`, `pg18`, `0.8.5-pg18`
- `pg17-trixie`, `0.8.5-pg17-trixie`
- `pg17-bookworm`, `0.8.5-pg17-bookworm`, `pg17`, `0.8.5-pg17`
- `pg16-trixie`, `0.8.5-pg16-trixie`
- `pg16-bookworm`, `0.8.5-pg16-bookworm`, `pg16`, `0.8.5-pg16`
- `pg15-trixie`, `0.8.5-pg15-trixie`
- `pg15-bookworm`, `0.8.5-pg15-bookworm`, `pg15`, `0.8.5-pg15`
- `pg14-trixie`, `0.8.5-pg14-trixie`
- `pg14-bookworm`, `0.8.5-pg14-bookworm`, `pg14`, `0.8.5-pg14`
- `pg13-trixie`, `0.8.5-pg13-trixie`
- `pg13-bookworm`, `0.8.5-pg13-bookworm`, `pg13`, `0.8.5-pg13`
You can also build the image manually:
```sh
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector .
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
```
If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds.
@@ -1173,7 +1196,7 @@ With Homebrew Postgres, you can use:
brew install pgvector
```
Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas
Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas
### PGXN
@@ -1188,29 +1211,29 @@ pgxn install vector
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh
sudo apt install postgresql-17-pgvector
sudo apt install postgresql-18-pgvector
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
```sh
sudo yum install pgvector_17
sudo yum install pgvector_18
# or
sudo dnf install pgvector_17
sudo dnf install pgvector_18
```
Note: Replace `17` with your Postgres server version
Note: Replace `18` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql16-pgvector
pkg install postgresql17-pgvector
```
or the port with:
@@ -1220,6 +1243,14 @@ cd /usr/ports/databases/pgvector
make install
```
### APK
Install the Alpine package with:
```sh
apk add postgresql-pgvector
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
@@ -1308,7 +1339,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
To enable benchmarking:
```sh
make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install
```
To show memory usage:

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.5'" to load this file. \quit

View File

@@ -31,10 +31,12 @@
#define BIT_TARGET_CLONES
#endif
/* Use built-ins when possible for inlining */
#if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
/* Use built-ins when possible for Postgres < 19 for inlining */
#if PG_VERSION_NUM >= 190000
#define popcount64(x) pg_popcount64(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
#define popcount64(x) __builtin_popcountl(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8)
#define popcount64(x) __builtin_popcountll(x)
#elif !defined(_MSC_VER)
/* Fails to resolve with MSVC */
@@ -169,7 +171,7 @@ BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char
#endif
TARGET_XSAVE static bool
SupportsAvx512Popcount()
SupportsAvx512Popcount(void)
{
unsigned int exx[4] = {0, 0, 0, 0};

View File

@@ -2,6 +2,7 @@
#include "bitutils.h"
#include "bitvec.h"
#include "fmgr.h"
#include "utils/varbit.h"
#include "vector.h"

View File

@@ -1,5 +1,7 @@
#include "postgres.h"
#include <math.h>
#include "halfutils.h"
#include "halfvec.h"

View File

@@ -13,12 +13,20 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/numeric.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -129,9 +137,9 @@ InitHalfVector(int dim)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define halfvec_isspace(ch) scanner_isspace(ch)
#else
static inline bool
halfvec_isspace(char ch)
{
@@ -144,6 +152,7 @@ halfvec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -898,8 +907,21 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS)
half *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized on aarch64 */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -1,18 +1,25 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "storage/lwlock.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -52,12 +59,20 @@ HnswInitLockTranche(void)
sizeof(int) * 1,
&found);
if (!found)
{
#if PG_VERSION_NUM >= 190000
tranche_ids[0] = LWLockNewTrancheId("HnswBuild");
#else
tranche_ids[0] = LWLockNewTrancheId();
#endif
}
hnsw_lock_tranche_id = tranche_ids[0];
LWLockRelease(AddinShmemInitLock);
#if PG_VERSION_NUM < 190000
/* Per-backend registration of the tranche ID */
LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild");
#endif
}
/*
@@ -130,7 +145,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NULL)
if (path->indexorderbys == NIL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -252,6 +267,64 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(hnswhandler);
Datum
hnswhandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 3,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = hnswbuild,
.ambuildempty = hnswbuildempty,
.aminsert = hnswinsert,
.aminsertcleanup = NULL,
.ambulkdelete = hnswbulkdelete,
.amvacuumcleanup = hnswvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = hnswcostestimate,
.amgettreeheight = NULL,
.amoptions = hnswoptions,
.amproperty = NULL,
.ambuildphasename = hnswbuildphasename,
.amvalidate = hnswvalidate,
.amadjustmembers = NULL,
.ambeginscan = hnswbeginscan,
.amrescan = hnswrescan,
.amgettuple = hnswgettuple,
.amgetbitmap = NULL,
.amendscan = hnswendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -259,6 +332,11 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -291,6 +369,9 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = hnswvacuumcleanup;
amroutine->amcanreturn = NULL;
amroutine->amcostestimate = hnswcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = hnswoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = hnswbuildphasename;
@@ -311,5 +392,11 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -3,15 +3,29 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/bufpage.h"
#include "storage/condition_variable.h"
#include "storage/lwlock.h"
#include "storage/s_lock.h"
#include "utils/relptr.h"
#include "utils/sampling.h"
#include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000
@@ -68,6 +82,21 @@
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -398,10 +427,11 @@ typedef struct HnswVacuumState
HnswSupport support;
/* Variables */
struct tidhash_hash *deleted;
struct tidhash_hash *deleting;
BufferAccessStrategy bas;
HnswNeighborTuple ntup;
HnswElementData highestPoint;
HnswElementData fallbackPoint;
/* Memory */
MemoryContext tmpCtx;
@@ -424,7 +454,7 @@ void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec);
HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);

View File

@@ -36,11 +36,14 @@
*/
#include "postgres.h"
#include <math.h>
#include <limits.h>
#include "access/genam.h"
#include "access/parallel.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/xact.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
@@ -48,11 +51,19 @@
#include "commands/progress.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -69,6 +80,8 @@
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
/*
* Create the metapage
*/
@@ -398,7 +411,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
* Update graph in memory
*/
static void
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate)
{
HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea;
@@ -460,7 +473,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false);
/* Update graph in memory */
UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate);
UpdateGraphInMemory(support, element, m, entryPoint, buildstate);
/* Release entry lock */
LWLockRelease(entryLock);
@@ -481,6 +494,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
LWLock *flushLock = &graph->flushLock;
char *base = buildstate->hnswarea;
Datum value;
Size memoryMargin;
/* Form index value */
if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
@@ -489,6 +503,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Get datum size */
valueSize = VARSIZE_ANY(DatumGetPointer(value));
/* In a parallel build, add a margin so allocations never fail */
memoryMargin = base == NULL ? 0 : 1024 * 1024;
/* Ensure graph not flushed when inserting */
LWLockAcquire(flushLock, LW_SHARED);
@@ -510,7 +527,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
* Check that we have enough memory available for the new element now that
* we have the allocator lock, and flush pages if needed.
*/
if (graph->memoryUsed >= graph->memoryTotal)
if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
{
LWLockRelease(&graph->allocatorLock);
@@ -545,7 +562,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
/* Copy the datum */
memcpy(valuePtr, DatumGetPointer(value), valueSize);
HnswPtrStore(base, element->value, valuePtr);
HnswPtrStore(base, element->value, (char *) valuePtr);
/* Create a lock for the element */
LWLockInitialize(&element->lock, hnsw_lock_tranche_id);
@@ -603,7 +620,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
graph->memoryTotal = memoryTotal;
graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
graph->flushed = false;
graph->indtuples = 0;
SpinLockInit(&graph->lock);
@@ -644,9 +661,17 @@ static void *
HnswSharedMemoryAlloc(Size size, void *state)
{
HnswBuildState *buildstate = (HnswBuildState *) state;
void *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
Size alignedSize = MAXALIGN(size);
void *chunk;
buildstate->graph->memoryUsed += MAXALIGN(size);
if (alignedSize > 1024 * 1024)
elog(ERROR, "hnsw allocation too large");
if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
elog(ERROR, "hnsw allocator out of memory");
chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
buildstate->graph->memoryUsed += alignedSize;
return chunk;
}
@@ -694,7 +719,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */
HnswInitSupport(&buildstate->support, index);
InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024);
buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -778,7 +803,11 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
buildstate.hnswarea = hnswarea;
InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
scan = table_beginscan_parallel(heapRel,
ParallelTableScanFromHnswShared(hnswshared));
ParallelTableScanFromHnswShared(hnswshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -927,11 +956,13 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Leave space for other objects in shared memory */
/* Docker has a default limit of 64 MB for shm_size */
/* which happens to be the default value of maintenance_work_mem */
esthnswarea = maintenance_work_mem * 1024L;
esthnswarea = maintenance_work_mem * (Size) 1024;
estother = 3 * 1024 * 1024;
if (esthnswarea > estother)
esthnswarea -= estother;
esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
shm_toc_estimate_keys(&pcxt->estimator, 2);
@@ -974,8 +1005,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
snapshot);
hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
/*
* Avoid base address for relptr for Postgres < 14.5
@@ -1054,7 +1084,7 @@ ComputeParallelWorkers(Relation heap, Relation index)
* Build graph
*/
static void
BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum)
BuildGraph(HnswBuildState * buildstate)
{
int parallel_workers = 0;
@@ -1102,7 +1132,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
InitBuildState(buildstate, heap, index, indexInfo, forkNum);
BuildGraph(buildstate, forkNum);
BuildGraph(buildstate);
if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM)
log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true);

View File

@@ -1,13 +1,19 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "hnsw.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "storage/lwlock.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Get the insert page
@@ -660,7 +666,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk
*/
static void
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building)
{
BlockNumber newInsertPage = InvalidBlockNumber;
@@ -708,7 +714,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
/* Create an element */
element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL);
HnswPtrStore(base, element->value, DatumGetPointer(value));
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
/* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -728,7 +734,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo
HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false);
/* Update graph on disk */
UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building);
UpdateGraphOnDisk(index, support, element, m, entryPoint, building);
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);

View File

@@ -1,12 +1,23 @@
#include "postgres.h"
#include <limits.h>
#include "access/genam.h"
#include "access/relscan.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/float.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/snapmgr.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Algorithm 5 from paper
@@ -142,7 +153,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
/* Calculate max memory */
/* Add 256 extra bytes to fill last block when close */
maxMemory = (double) work_mem * hnsw_scan_mem_multiplier * 1024.0 + 256;
so->maxMemory = Min(maxMemory, (double) SIZE_MAX);
so->maxMemory = Min(maxMemory, (double) (SIZE_MAX / 2));
scan->opaque = so;
@@ -193,6 +204,10 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -2,18 +2,24 @@
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "catalog/pg_type.h"
#include "catalog/pg_type_d.h"
#include "common/hashfn.h"
#include "fmgr.h"
#include "hnsw.h"
#include "lib/pairingheap.h"
#include "nodes/pg_list.h"
#include "port/atomics.h"
#include "sparsevec.h"
#include "storage/bufmgr.h"
#include "utils/datum.h"
#include "utils/memdebug.h"
#include "utils/rel.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM < 170000
static inline uint64
@@ -256,7 +262,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel,
HnswInitNeighbors(base, element, m, allocator);
HnswPtrStore(base, element->value, (Pointer) NULL);
HnswPtrStore(base, element->value, (char *) NULL);
return element;
}
@@ -282,7 +288,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
element->blkno = blkno;
element->offno = offno;
HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL);
HnswPtrStore(base, element->value, (Pointer) NULL);
HnswPtrStore(base, element->value, (char *) NULL);
return element;
}
@@ -508,7 +514,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
char *base = NULL;
Datum value = datumCopy(PointerGetDatum(&etup->data), false, -1);
HnswPtrStore(base, element->value, DatumGetPointer(value));
HnswPtrStore(base, element->value, (char *) DatumGetPointer(value));
}
}
@@ -540,6 +546,9 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, Hns
Assert(HnswIsElementTuple(etup));
if (unlikely(etup->deleted))
elog(ERROR, "cannot load deleted element");
/* Calculate distance */
if (distance != NULL)
{
@@ -922,7 +931,7 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
continue;
}
if (eElement == NULL || !(eDistance < f->distance || alwaysAdd))
if (!(eDistance < f->distance || alwaysAdd))
{
if (discarded != NULL)
{

View File

@@ -1,25 +1,30 @@
#include "postgres.h"
#include <math.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "nodes/pg_list.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)
#endif
/*
* Check if deleted list contains an index TID
* Check if deletion list contains an element
*/
static bool
DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
{
return tidhash_lookup(deleted, *indextid) != NULL;
return tidhash_lookup(deleting, *indextid) != NULL;
}
/*
@@ -32,17 +37,20 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
IndexBulkDeleteResult *stats = vacuumstate->stats;
/* Store separately since highestPoint.level is uint8 */
/* Store separately since HnswElement level is uint8 */
int highestLevel = -1;
int fallbackLevel = -1;
/* Initialize highest point */
/* Initialize highest point and fallback point */
highestPoint->blkno = InvalidBlockNumber;
highestPoint->offno = InvalidOffsetNumber;
fallbackPoint->blkno = InvalidBlockNumber;
fallbackPoint->offno = InvalidOffsetNumber;
while (BlockNumberIsValid(blkno))
{
@@ -72,6 +80,14 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/*
* Skip deleted tuples. It is important they are not added to the
* deletion list to avoid false positives in NeedsUpdated and
* ConfirmRepaired.
*/
if (etup->deleted)
continue;
if (ItemPointerIsValid(&etup->heaptids[0]))
{
for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -105,23 +121,40 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!ItemPointerIsValid(&etup->heaptids[0]))
{
ItemPointerData ip;
ItemPointerData indextid;
bool found;
/* Add to deleted list */
ItemPointerSet(&ip, blkno, offno);
/* Add to deletion list */
ItemPointerSet(&indextid, blkno, offno);
tidhash_insert(vacuumstate->deleted, ip, &found);
tidhash_insert(vacuumstate->deleting, indextid, &found);
Assert(!found);
}
else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
else if (etup->level > highestLevel)
{
/* Keep track of highest non-entry point */
if (BlockNumberIsValid(highestPoint->blkno))
{
/* Current highest point becomes fallback */
fallbackPoint->blkno = highestPoint->blkno;
fallbackPoint->offno = highestPoint->offno;
fallbackPoint->level = highestPoint->level;
fallbackLevel = highestLevel;
}
/* Keep track of highest point */
highestPoint->blkno = blkno;
highestPoint->offno = offno;
highestPoint->level = etup->level;
highestLevel = etup->level;
}
else if (etup->level > fallbackLevel)
{
/* Keep track of second highest point */
fallbackPoint->blkno = blkno;
fallbackPoint->offno = offno;
fallbackPoint->level = etup->level;
fallbackLevel = etup->level;
}
}
blkno = HnswPageGetOpaque(page)->nextblkno;
@@ -133,6 +166,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
UnlockReleaseBuffer(buf);
}
#ifdef HNSW_MEMORY
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
#endif
}
/*
@@ -163,8 +200,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deleted list */
if (DeletedContains(vacuumstate->deleted, indextid))
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
{
needsUpdated = true;
break;
@@ -173,7 +210,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */
if (!needsUpdated)
/* There should always be more than zero indextids, but check for safety */
if (!needsUpdated && ntup->count > 0)
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
UnlockReleaseBuffer(buf);
@@ -259,12 +297,27 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
/* Get a shared lock */
LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Get latest entry point */
entryPoint = HnswGetEntryPoint(index);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
/* Use fallback point if highest point is entry point */
if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno)
{
highestPoint = &vacuumstate->fallbackPoint;
if (!BlockNumberIsValid(highestPoint->blkno))
highestPoint = NULL;
}
if (highestPoint != NULL)
{
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, entryPoint);
}
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -282,7 +335,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
if (DeletedContains(vacuumstate->deleted, &epData))
if (DeletingElement(vacuumstate->deleting, &epData))
{
/*
* Replace the entry point with the highest point. If highest
@@ -368,6 +421,10 @@ RepairGraph(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip updating neighbors if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
@@ -431,6 +488,103 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(vacuumstate->tmpCtx);
#ifdef HNSW_VACUUM_PROGRESS
if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
{
BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
}
#endif
}
}
/*
* Confirm graph was repaired
*/
static void
ConfirmRepaired(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
while (BlockNumberIsValid(blkno))
{
Buffer buf;
Page page;
OffsetNumber offno;
OffsetNumber maxoffno;
vacuum_delay_point();
buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
OffsetNumber neighborOffno;
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
if (neighborPage == blkno)
{
nbuf = buf;
npage = page;
}
else
{
nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
LockBuffer(nbuf, BUFFER_LOCK_SHARE);
npage = BufferGetPage(nbuf);
}
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Check neighbors */
for (int i = 0; i < ntup->count; i++)
{
ItemPointer indextid = &ntup->indextids[i];
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
elog(ERROR, "hnsw graph not repaired");
}
if (nbuf != buf)
UnlockReleaseBuffer(nbuf);
}
blkno = HnswPageGetOpaque(page)->nextblkno;
UnlockReleaseBuffer(buf);
}
}
@@ -446,10 +600,15 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BufferAccessStrategy bas = vacuumstate->bas;
/*
* Wait for index scans to complete. Scans before this point may contain
* tuples about to be deleted. Scans after this point will not, since the
* graph has been repaired.
* Wait for inserts and index scans to complete. Inserts and scans before
* this point may visit tuples about to be deleted. Inserts and scans
* after this point will not, since the graph has been repaired.
*/
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
ConfirmRepaired(vacuumstate);
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
@@ -524,8 +683,9 @@ MarkDeleted(HnswVacuumState * vacuumstate)
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Overwrite element */
/* Use memset instead of MemSet to keep clang-tidy happy */
etup->deleted = 1;
MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data));
memset(&etup->data, 0, VARSIZE_ANY(&etup->data));
/* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++)
@@ -596,7 +756,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
/* Create hash table */
vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
}
/*
@@ -605,7 +765,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
static void
FreeVacuumState(HnswVacuumState * vacuumstate)
{
tidhash_destroy(vacuumstate->deleted);
tidhash_destroy(vacuumstate->deleting);
FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx);
@@ -623,13 +783,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
/* Pass 1: Remove heap TIDs */
RemoveHeapTids(&vacuumstate);
HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate));
/* Pass 2: Repair graph */
RepairGraph(&vacuumstate);
HnswBench("RepairGraph", RepairGraph(&vacuumstate));
/* Pass 3: Mark as deleted */
MarkDeleted(&vacuumstate);
/* Passes 3 and 4: Confirm repaired and mark as deleted */
HnswBench("MarkDeleted", MarkDeleted(&vacuumstate));
FreeVacuumState(&vacuumstate);

View File

@@ -2,23 +2,37 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/tupdesc.h"
#include "access/parallel.h"
#include "access/xact.h"
#include "bitvec.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "commands/progress.h"
#include "halfvec.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "optimizer/optimizer.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h"
#include "utils/memutils.h"
#include "vector.h"
#include "utils/rel.h"
#include "utils/sampling.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 140000
#include "utils/backend_progress.h"
@@ -49,15 +63,13 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/*
* Normalize with KMEANS_NORM_PROC since spherical distance function
* expects unit vectors
* Check with KMEANS_NORM_PROC that the value can be normalized since
* spherical distance function expects unit vectors
*/
if (buildstate->kmeansnormprocinfo != NULL)
{
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
}
if (samples->length < targsamples)
@@ -68,7 +80,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
else
{
if (buildstate->rowstoskip < 0)
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples);
if (buildstate->rowstoskip <= 0)
{
@@ -84,6 +96,9 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
buildstate->rowstoskip -= 1;
}
/* Increment after reservoir_get_next_S */
buildstate->samplerows += 1;
}
/*
@@ -120,6 +135,7 @@ SampleRows(IvfflatBuildState * buildstate)
int targsamples = buildstate->samples->maxlen;
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
buildstate->samplerows = 0;
buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
@@ -129,16 +145,21 @@ SampleRows(IvfflatBuildState * buildstate)
{
BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
/* Set anyvisible to false like table_index_build_scan */
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
}
/* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL)
IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx);
}
/*
* Add tuple to sort
*/
static void
AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate)
{
double distance;
double minDistance = DBL_MAX;
@@ -215,7 +236,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values,
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
/* Add tuple to sort */
AddTupleToSort(index, tid, values, buildstate);
AddTupleToSort(tid, values, buildstate);
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
@@ -361,10 +382,20 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(buildstate->sortdesc);
#endif
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
buildstate->memoryUsed = 0;
buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
/* TODO Move allocation to page creation */
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -407,22 +438,30 @@ ComputeCenters(IvfflatBuildState * buildstate)
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
/* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50;
if (numSamples < 10000)
numSamples = 10000;
/* Skip samples for unlogged table */
if (buildstate->heap == NULL)
numSamples = 1;
else
{
int64 maxTuples = (int64) RelationGetNumberOfBlocks(buildstate->heap) * MaxHeapTuplesPerPage;
/* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50;
if (numSamples < 10000)
numSamples = 10000;
/* Save memory since will not have more than max tuples */
numSamples = Max(Min(numSamples, maxTuples), 1);
}
/* Sample rows */
/* TODO Ensure within maintenance_work_mem */
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
if (buildstate->heap != NULL)
{
SampleRows(buildstate);
IvfflatBench("sample rows", SampleRows(buildstate));
if (buildstate->samples->length < buildstate->lists)
{
@@ -434,7 +473,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
}
/* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed));
/* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples);
@@ -470,8 +509,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum)
* Create list pages
*/
static void
CreateListPages(Relation index, VectorArray centers, int dimensions,
int lists, ForkNumber forkNum, ListInfo * *listInfo)
CreateListPages(Relation index, VectorArray centers, int lists,
ForkNumber forkNum, ListInfo * *listInfo)
{
Buffer buf;
Page page;
@@ -637,7 +676,11 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared));
ParallelTableScanFromIvfflatShared(ivfshared)
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
true, progress, BuildCallback,
(void *) &buildstate, scan);
@@ -1004,7 +1047,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
/* Create pages */
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum);
/* Write WAL for initialization fork since GenericXLog functions do not */
@@ -1023,6 +1066,10 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo)
IndexBuildResult *result;
IvfflatBuildState buildstate;
#ifdef IVFFLAT_BENCH
SeedRandom(42);
#endif
BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM);
result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));

View File

@@ -3,14 +3,19 @@
#include <float.h>
#include "access/amapi.h"
#include "access/genam.h"
#include "access/reloptions.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/pg_list.h"
#include "utils/float.h"
#include "utils/guc.h"
#include "utils/relcache.h"
#include "utils/selfuncs.h"
#include "utils/spccache.h"
#include "vector.h"
#if PG_VERSION_NUM < 150000
#define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x)
@@ -92,7 +97,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Relation index;
/* Never use index without order */
if (path->indexorderbys == NULL)
if (path->indexorderbys == NIL)
{
*indexStartupCost = get_float8_infinity();
*indexTotalCost = get_float8_infinity();
@@ -179,6 +184,64 @@ FUNCTION_PREFIX PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{
#if PG_VERSION_NUM >= 190000
static const IndexAmRoutine amroutine = {
.type = T_IndexAmRoutine,
.amstrategies = 0,
.amsupport = 5,
.amoptsprocnum = 0,
.amcanorder = false,
.amcanorderbyop = true,
.amcanhash = false,
.amconsistentequality = false,
.amconsistentordering = false,
.amcanbackward = false,
.amcanunique = false,
.amcanmulticol = false,
.amoptionalkey = true,
.amsearcharray = false,
.amsearchnulls = false,
.amstorage = false,
.amclusterable = false,
.ampredlocks = false,
.amcanparallel = false,
.amcanbuildparallel = true,
.amcaninclude = false,
.amusemaintenanceworkmem = false,
.amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid,
.ambuild = ivfflatbuild,
.ambuildempty = ivfflatbuildempty,
.aminsert = ivfflatinsert,
.aminsertcleanup = NULL,
.ambulkdelete = ivfflatbulkdelete,
.amvacuumcleanup = ivfflatvacuumcleanup,
.amcanreturn = NULL,
.amcostestimate = ivfflatcostestimate,
.amgettreeheight = NULL,
.amoptions = ivfflatoptions,
.amproperty = NULL,
.ambuildphasename = ivfflatbuildphasename,
.amvalidate = ivfflatvalidate,
.amadjustmembers = NULL,
.ambeginscan = ivfflatbeginscan,
.amrescan = ivfflatrescan,
.amgettuple = ivfflatgettuple,
.amgetbitmap = NULL,
.amendscan = ivfflatendscan,
.ammarkpos = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL,
.aminitparallelscan = NULL,
.amparallelrescan = NULL,
.amtranslatestrategy = NULL,
.amtranslatecmptype = NULL,
};
PG_RETURN_POINTER(&amroutine);
#else
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
@@ -186,6 +249,11 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amoptsprocnum = 0;
amroutine->amcanorder = false;
amroutine->amcanorderbyop = true;
#if PG_VERSION_NUM >= 180000
amroutine->amcanhash = false;
amroutine->amconsistentequality = false;
amroutine->amconsistentordering = false;
#endif
amroutine->amcanbackward = false; /* can change direction mid-scan */
amroutine->amcanunique = false;
amroutine->amcanmulticol = false;
@@ -218,6 +286,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amvacuumcleanup = ivfflatvacuumcleanup;
amroutine->amcanreturn = NULL; /* tuple not included in heapsort */
amroutine->amcostestimate = ivfflatcostestimate;
#if PG_VERSION_NUM >= 180000
amroutine->amgettreeheight = NULL;
#endif
amroutine->amoptions = ivfflatoptions;
amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */
amroutine->ambuildphasename = ivfflatbuildphasename;
@@ -238,5 +309,11 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = NULL;
#if PG_VERSION_NUM >= 180000
amroutine->amtranslatestrategy = NULL;
amroutine->amtranslatecmptype = NULL;
#endif
PG_RETURN_POINTER(amroutine);
#endif
}

View File

@@ -9,10 +9,15 @@
#include "lib/pairingheap.h"
#include "nodes/execnodes.h"
#include "port.h" /* for random() */
#include "storage/condition_variable.h"
#include "utils/sampling.h"
#include "utils/tuplesort.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
@@ -21,6 +26,10 @@
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
#define IVFFLAT_MAX_DIM 2000
/* Support functions */
@@ -73,9 +82,11 @@
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#define SeedRandom(seed) srandom(seed)
#endif
/* Variables */
@@ -193,6 +204,7 @@ typedef struct IvfflatBuildState
VectorArray samples;
VectorArray centers;
ListInfo *listInfo;
Size itemsize;
#ifdef IVFFLAT_KMEANS_DEBUG
double inertia;
@@ -203,7 +215,8 @@ typedef struct IvfflatBuildState
/* Sampling */
BlockSamplerData bs;
ReservoirStateData rstate;
int rowstoskip;
double samplerows;
double rowstoskip;
/* Sorting */
Tuplesortstate *sortstate;
@@ -211,6 +224,7 @@ typedef struct IvfflatBuildState
TupleTableSlot *slot;
/* Memory */
Size memoryUsed;
MemoryContext tmpCtx;
/* Parallel builds */
@@ -291,22 +305,32 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
static inline Pointer
VectorArrayGet(VectorArray arr, int offset)
{
if (offset >= arr->maxlen)
elog(ERROR, "safety check failed");
return ((char *) arr->items) + (offset * arr->itemsize);
}
static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val)
{
memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
Size size = VARSIZE_ANY(val);
if (size > arr->itemsize)
elog(ERROR, "safety check failed");
memcpy(VectorArrayGet(arr, offset), val, size);
}
/* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
void IvfflatCheckMemoryUsage(Size totalSize);
int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);

View File

@@ -2,11 +2,16 @@
#include <float.h>
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "nodes/execnodes.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
/*
* Find the list that minimizes the distance function
@@ -65,7 +70,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo
* Insert a tuple into the index
*/
static void
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel)
InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid)
{
const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index);
IndexTuple itup;
@@ -204,7 +209,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid,
oldCtx = MemoryContextSwitchTo(insertCtx);
/* Insert tuple */
InsertTuple(index, values, isnull, heap_tid, heap);
InsertTuple(index, values, isnull, heap_tid);
/* Delete memory context */
MemoryContextSwitchTo(oldCtx);

View File

@@ -1,17 +1,19 @@
#include "postgres.h"
#include <float.h>
#include <limits.h>
#include <math.h>
#include "bitvec.h"
#include "halfutils.h"
#include "halfvec.h"
#include "access/genam.h"
#include "fmgr.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/memutils.h"
#include "vector.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Initialize with kmeans++
@@ -97,22 +99,8 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context",
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(normCtx);
for (int j = 0; j < centers->length; j++)
{
Datum center = PointerGetDatum(VectorArrayGet(centers, j));
Datum newCenter = IvfflatNormValue(typeInfo, collation, center);
Size size = VARSIZE_ANY(DatumGetPointer(newCenter));
if (size > centers->itemsize)
elog(ERROR, "safety check failed");
memcpy(DatumGetPointer(center), DatumGetPointer(newCenter), size);
MemoryContextReset(normCtx);
}
MemoryContextSwitchTo(oldCtx);
IvfflatNormVectors(typeInfo, collation, centers, normCtx);
MemoryContextDelete(normCtx);
}
@@ -256,7 +244,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/
static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
{
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
@@ -275,8 +263,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *newcdist;
/* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size centerCountsSize = sizeof(int) * numCenters;
@@ -288,18 +274,13 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */
Size totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
Size totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */
/* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
IvfflatCheckMemoryUsage(totalSize);
/* Ensure indexing does not overflow */
if (numCenters * numCenters > INT_MAX)
if (numCenters > INT_MAX / numCenters)
elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */
@@ -560,7 +541,7 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine
*/
void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
{
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context",
@@ -570,7 +551,7 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
if (samples->length == 0)
RandomCenters(index, centers, typeInfo);
else
ElkanKmeans(index, samples, centers, typeInfo);
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
CheckCenters(index, centers, typeInfo);

View File

@@ -2,15 +2,26 @@
#include <float.h>
#include "access/genam.h"
#include "access/itup.h"
#include "access/relscan.h"
#include "access/tupdesc.h"
#include "catalog/pg_operator_d.h"
#include "catalog/pg_type_d.h"
#include "fmgr.h"
#include "lib/pairingheap.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
#include "utils/tuplesort.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr)
#define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr)
@@ -287,6 +298,9 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->tupdesc = CreateTemplateTupleDesc(2);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(so->tupdesc);
#endif
/* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc);
@@ -355,6 +369,10 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
/* Count index scan for stats */
pgstat_count_index_scan(scan->indexRelation);
#if PG_VERSION_NUM >= 180000
if (scan->instrument)
scan->instrument->nsearches++;
#endif
/* Safety check */
if (scan->orderByData == NULL)

View File

@@ -1,13 +1,21 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "bitvec.h"
#include "catalog/pg_type.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
/*
* Allocate a vector array
@@ -15,11 +23,15 @@
VectorArray
VectorArrayInit(int maxlen, int dimensions, Size itemsize)
{
VectorArray res = palloc(sizeof(VectorArrayData));
VectorArray res;
if (maxlen < 1 || dimensions < 1)
elog(ERROR, "safety check failed");
/* Ensure items are aligned to prevent UB */
itemsize = MAXALIGN(itemsize);
res = palloc(sizeof(VectorArrayData));
res->length = 0;
res->maxlen = maxlen;
res->dim = dimensions;
@@ -82,6 +94,40 @@ IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
}
/*
* Normalize vectors
*/
void
IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
{
MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
for (int i = 0; i < arr->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(arr, i));
Datum newValue = IvfflatNormValue(typeInfo, collation, value);
VectorArraySet(arr, i, DatumGetPointer(newValue));
MemoryContextReset(tmpCtx);
}
MemoryContextSwitchTo(oldCtx);
}
/*
* Check memory usage
*/
void
IvfflatCheckMemoryUsage(Size totalSize)
{
/* Add one to error message to ceil */
if (totalSize > maintenance_work_mem * (Size) 1024)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
}
/*
* New buffer
*/
@@ -259,8 +305,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VECTOR_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = x[k];
for (int i = 0; i < dimensions; i++)
vec->x[i] = x[i];
}
static void
@@ -271,8 +317,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, HALFVEC_SIZE(dimensions));
vec->dim = dimensions;
for (int k = 0; k < dimensions; k++)
vec->x[k] = Float4ToHalfUnchecked(x[k]);
for (int i = 0; i < dimensions; i++)
vec->x[i] = Float4ToHalfUnchecked(x[i]);
}
static void
@@ -284,29 +330,33 @@ BitUpdateCenter(Pointer v, int dimensions, float *x)
SET_VARSIZE(vec, VARBITTOTALLEN(dimensions));
VARBITLEN(vec) = dimensions;
for (uint32 k = 0; k < VARBITBYTES(vec); k++)
nx[k] = 0;
for (uint32 i = 0; i < VARBITBYTES(vec); i++)
nx[i] = 0;
for (int k = 0; k < dimensions; k++)
nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8));
for (int i = 0; i < dimensions; i++)
nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8));
}
static void
VectorSumCenter(Pointer v, float *x)
{
Vector *vec = (Vector *) v;
int dim = vec->dim;
for (int k = 0; k < vec->dim; k++)
x[k] += vec->x[k];
/* Auto-vectorized */
for (int i = 0; i < dim; i++)
x[i] += vec->x[i];
}
static void
HalfvecSumCenter(Pointer v, float *x)
{
HalfVector *vec = (HalfVector *) v;
int dim = vec->dim;
for (int k = 0; k < vec->dim; k++)
x[k] += HalfToFloat4(vec->x[k]);
/* Auto-vectorized on aarch64 */
for (int i = 0; i < dim; i++)
x[i] += HalfToFloat4(vec->x[i]);
}
static void
@@ -314,8 +364,8 @@ BitSumCenter(Pointer v, float *x)
{
VarBit *vec = (VarBit *) v;
for (int k = 0; k < VARBITLEN(vec); k++)
x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01);
for (int i = 0; i < VARBITLEN(vec); i++)
x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01);
}
/*

View File

@@ -1,9 +1,12 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/generic_xlog.h"
#include "access/itup.h"
#include "commands/vacuum.h"
#include "ivfflat.h"
#include "storage/bufmgr.h"
#include "utils/relcache.h"
#if PG_VERSION_NUM >= 180000
#define vacuum_delay_point() vacuum_delay_point(false)

View File

@@ -5,18 +5,27 @@
#include "catalog/pg_type.h"
#include "common/shortest_dec.h"
#include "common/string.h"
#include "fmgr.h"
#include "halfutils.h"
#include "halfvec.h"
#include "lib/stringinfo.h"
#include "libpq/pqformat.h"
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
typedef struct SparseInputElement
{
int32 index;
@@ -150,9 +159,9 @@ InitSparseVector(int dim, int nnz)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define sparsevec_isspace(ch) scanner_isspace(ch)
#else
static inline bool
sparsevec_isspace(char ch)
{
@@ -165,6 +174,7 @@ sparsevec_isspace(char ch)
return true;
return false;
}
#endif
/*
* Compare indices
@@ -172,10 +182,10 @@ sparsevec_isspace(char ch)
static int
CompareIndices(const void *a, const void *b)
{
if (((SparseInputElement *) a)->index < ((SparseInputElement *) b)->index)
if (((const SparseInputElement *) a)->index < ((const SparseInputElement *) b)->index)
return -1;
if (((SparseInputElement *) a)->index > ((SparseInputElement *) b)->index)
if (((const SparseInputElement *) a)->index > ((const SparseInputElement *) b)->index)
return 1;
return 0;

View File

@@ -16,16 +16,20 @@
#include "port.h" /* for strtof() */
#include "sparsevec.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "utils/numeric.h"
#include "utils/varbit.h"
#include "vector.h"
#if PG_VERSION_NUM >= 160000
#include "varatt.h"
#endif
#if PG_VERSION_NUM >= 170000
#include "parser/scansup.h"
#endif
#define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1)
#define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1))
@@ -35,7 +39,11 @@
#define VECTOR_TARGET_CLONES
#endif
#if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.5");
#else
PG_MODULE_MAGIC;
#endif
/*
* Initialize index options and variables
@@ -125,9 +133,9 @@ InitVector(int dim)
return result;
}
/*
* Check for whitespace, since array_isspace() is static
*/
#if PG_VERSION_NUM >= 170000
#define vector_isspace(ch) scanner_isspace(ch)
#else
static inline bool
vector_isspace(char ch)
{
@@ -140,6 +148,7 @@ vector_isspace(char ch)
return true;
return false;
}
#endif
/*
* Check state array
@@ -920,11 +929,13 @@ vector_concat(PG_FUNCTION_ARGS)
CheckDim(dim);
result = InitVector(dim);
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++)
result->x[i] = a->x[i];
for (int i = 0; i < b->dim; i++)
result->x[i + a->dim] = b->x[i];
/* Auto-vectorized */
for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++)
result->x[i + start] = b->x[i];
PG_RETURN_POINTER(result);
}
@@ -940,8 +951,21 @@ binary_quantize(PG_FUNCTION_ARGS)
float *ax = a->x;
VarBit *result = InitBitVector(a->dim);
unsigned char *rx = VARBITS(result);
int i = 0;
int count = (a->dim / 8) * 8;
for (int i = 0; i < a->dim; i++)
/* Auto-vectorized */
for (; i < count; i += 8)
{
unsigned char result_byte = 0;
for (int j = 0; j < 8; j++)
result_byte |= (ax[i + j] > 0) << (7 - j);
rx[i / 8] = result_byte;
}
for (; i < a->dim; i++)
rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8));
PG_RETURN_VARBIT_P(result);

View File

@@ -540,6 +540,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec)
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
subvector
-----------

View File

@@ -49,3 +49,11 @@ CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -100,3 +100,11 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t
(1 row)
DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -161,6 +161,7 @@ ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m
DROP TABLE t;
SHOW hnsw.ef_search;
hnsw.ef_search
----------------
@@ -198,4 +199,11 @@ SET hnsw.scan_mem_multiplier = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
SET hnsw.scan_mem_multiplier = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -35,3 +35,32 @@ NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -82,3 +82,32 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t
(1 row)
DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -143,6 +143,7 @@ DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768".
DROP TABLE t;
SHOW ivfflat.probes;
ivfflat.probes
----------------
@@ -172,4 +173,32 @@ SET ivfflat.max_probes = 0;
ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
SET ivfflat.max_probes = 32769;
ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -576,6 +576,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
01001110101
(1 row)
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
binary_quantize
---------------------
1110110110011011011
(1 row)
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
subvector
-----------

View File

@@ -121,6 +121,7 @@ SELECT l2_normalize('[65504]'::halfvec);
SELECT binary_quantize('[1,0,-1]'::halfvec);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec);
SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3);
SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2);

View File

@@ -33,3 +33,13 @@ CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;

View File

@@ -56,3 +56,13 @@ SELECT * FROM t ORDER BY val <+> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;

View File

@@ -95,23 +95,28 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
DROP TABLE t;
SHOW hnsw.ef_search;
SET hnsw.ef_search = 0;
SET hnsw.ef_search = 1001;
SHOW hnsw.iterative_scan;
SET hnsw.iterative_scan = on;
SHOW hnsw.max_scan_tuples;
SET hnsw.max_scan_tuples = 0;
SHOW hnsw.scan_mem_multiplier;
SET hnsw.scan_mem_multiplier = 0;
SET hnsw.scan_mem_multiplier = 1001;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;

View File

@@ -21,3 +21,28 @@ CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1)
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -43,3 +43,28 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -81,19 +81,40 @@ DROP TABLE t;
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
DROP TABLE t;
SHOW ivfflat.probes;
SET ivfflat.probes = 0;
SET ivfflat.probes = 32769;
SHOW ivfflat.iterative_scan;
SET ivfflat.iterative_scan = on;
SHOW ivfflat.max_probes;
SET ivfflat.max_probes = 0;
SET ivfflat.max_probes = 32769;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -128,6 +128,7 @@ SELECT l2_normalize('[3e38]'::vector);
SELECT binary_quantize('[1,0,-1]'::vector);
SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector);
SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector);
SELECT subvector('[1,2,3,4,5]'::vector, 1, 3);
SELECT subvector('[1,2,3,4,5]'::vector, 3, 2);

View File

@@ -16,29 +16,19 @@ $node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 100000) i;"
);
# Check each index type
my @operators = ("<->", "<#>", "<=>");
my @opclasses = ("vector_l2_ops", "vector_ip_ops", "vector_cosine_ops");
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops);");
for my $i (0 .. $#operators)
# Test 100% recall
for (1 .. 20)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test 100% recall
for (1 .. 20)
{
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
my $id = int(rand() * 100000);
my $query = $node->safe_psql("postgres", "SELECT v FROM tst WHERE i = $id;");
my $res = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT v FROM tst ORDER BY v <-> '$query' LIMIT 1;
));
is($res, $query);
}
done_testing();

View File

@@ -0,0 +1,29 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 1000) i;"
);
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
SET client_min_messages = DEBUG;
SET maintenance_work_mem = '3073kB';
ALTER TABLE tst SET (parallel_workers = 1);
CREATE INDEX ON tst USING hnsw (v vector_l2_ops);
));
is($ret, 0, $stderr);
like($stderr, qr/using \d+ parallel workers/);
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem after 0 tuples/);
done_testing();

View File

@@ -0,0 +1,38 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "DELETE FROM tst");
# Test HNSW_SCAN_LOCK at the beginning of MarkDeleted is effective
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent SELECTs and VACUUM",
{
"046_hnsw_vacuum_scan_select\@1000" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"046_hnsw_vacuum_scan_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -0,0 +1,39 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
# Test no "hnsw graph not repaired" errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
"047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -0,0 +1,39 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 10);");
# Test no errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1500",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"048_ivfflat_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"048_ivfflat_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"048_ivfflat_vacuum_insert_select\@500" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
"048_ivfflat_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.0'
default_version = '0.8.5'
module_pathname = '$libdir/vector'
relocatable = true