Compare commits

...

57 Commits

Author SHA1 Message Date
Andrew Kane
1d458ad5d7 Version bump to 0.8.4 [skip ci] 2026-06-30 15:24:14 -07:00
Andrew Kane
9fa17c10b8 Added comments to HNSW vacuuming tests [skip ci] 2026-06-30 13:40:32 -07:00
Andrew Kane
14149b19f5 Added SELECTs to HNSW vacuuming test for good measure [skip ci] 2026-06-30 13:34:11 -07:00
Andrew Kane
34d796fbab Added test for concurrent INSERTs, DELETEs, and VACUUM with HNSW - #993 2026-06-30 13:23:04 -07:00
Andrew Kane
53341bb6c7 Simplified test [skip ci] 2026-06-30 13:16:17 -07:00
Andrew Kane
0d9720f440 Added test for concurrent SELECTs and VACUUM [skip ci] 2026-06-30 13:12:47 -07:00
Andrew Kane
cb246cb72d Improved naming [skip ci] 2026-06-30 02:46:31 -07:00
Andrew Kane
d053de2d94 Removed deletion list check in MarkDeleted (does not help safety) [skip ci] 2026-06-30 02:31:54 -07:00
Andrew Kane
83bac90869 Added checks for deleted tuples rather than relying on ItemPointerIsValid [skip ci] 2026-06-30 01:40:30 -07:00
Andrew Kane
4eca5024df Updated comment [skip ci] 2026-06-30 01:35:07 -07:00
Andrew Kane
a31771bc45 Changed log message to assertion [skip ci] 2026-06-30 01:25:36 -07:00
Andrew Kane
ecddde963a Added check to confirm in deletion list before marking as deleted 2026-06-30 01:15:38 -07:00
Andrew Kane
ecd413d0fe Improved naming [skip ci] 2026-06-30 01:03:08 -07:00
Andrew Kane
497db7976c Fixed hnsw graph not repaired error with HNSW vacuuming - fixes #993 2026-06-30 00:34:01 -07:00
Andrew Kane
f1dd4e3b03 Updated urls [skip ci] 2026-06-29 01:08:14 -07:00
Andrew Kane
7d067d7b83 Updated changelog [skip ci] 2026-06-24 11:44:04 -07:00
Andrew Kane
d4dd73d970 Moved repair confirmation after lock wait to catch more potential issues 2026-06-23 11:39:16 -07:00
Bhagyesh Chaturvedi
ffe28bb954 Fix HNSW insert and vacuum race 2026-06-23 06:00:23 +00:00
Andrew Kane
0dbc1a27c0 Removed redundant check [skip ci] 2026-06-18 12:56:57 -07:00
Andrew Kane
08c4e7ff10 Hardened VectorArrayGet and VectorArraySet [skip ci] 2026-06-18 12:56:12 -07:00
Andrew Kane
6731c49811 Fixed overflow check (should never be hit) [skip ci] 2026-06-18 12:51:38 -07:00
Andrew Kane
f2617f02d1 Updated style to be consistent with latest Postgres [skip ci] 2026-06-18 12:44:11 -07:00
Andrew Kane
bdf19077db Ensure centers and samples fit into maintenance_work_mem before allocating for IVFFlat index builds - closes #986 2026-06-18 12:29:58 -07:00
Andrew Kane
90cd2b4ee5 Improved memory tracking for IVFFlat index builds [skip ci] 2026-06-18 11:57:03 -07:00
Andrew Kane
b44d1b4c5f Added todo [skip ci] 2026-06-18 11:51:37 -07:00
Andrew Kane
cc5b865c33 Added itemsize to IvfflatBuildState [skip ci] 2026-06-18 11:49:29 -07:00
Andrew Kane
4895021088 Hardened NeedsUpdated [skip ci] 2026-06-18 11:19:27 -07:00
Andrew Kane
eda77b3492 DRY normalize code for IVFFlat index builds 2026-06-18 11:16:51 -07:00
Andrew Kane
a2364b1793 Switched to VectorArraySet for NormCenters [skip ci] 2026-06-18 11:05:37 -07:00
Andrew Kane
a0eaf70d17 Hardened VectorArraySet [skip ci] 2026-06-18 11:03:09 -07:00
Andrew Kane
586e7515ba Version bump to 0.8.3 [skip ci] 2026-06-17 21:26:13 -07:00
Andrew Kane
a619d0b34d Simplified logic for HNSW vacuum progress [skip ci] 2026-06-17 12:18:17 -07:00
Andrew Kane
ad222abf48 Improved HNSW vacuum progress for round block sizes [skip ci] 2026-06-16 14:21:23 -07:00
Andrew Kane
8a81a3fe3a Added HNSW_VACUUM_PROGRESS flag [skip ci] 2026-06-16 13:53:40 -07:00
Andrew Kane
fc804925a4 Added todo [skip ci] 2026-06-16 12:33:25 -07:00
Andrew Kane
98dc4aa6d0 Added pass to confirm graph was repaired for HNSW vacuuming 2026-06-16 10:51:54 -07:00
Andrew Kane
7b72aeeff3 Added check for deleted element [skip ci] 2026-06-16 10:13:40 -07:00
Andrew Kane
4682d51e5d Added memory usage for HNSW vacuuming [skip ci] 2026-06-16 10:10:13 -07:00
Andrew Kane
91fe2e62e7 Added benchmarking for HNSW vacuuming [skip ci] 2026-06-16 09:53:47 -07:00
Andrew Kane
eedba7ee14 Fixed possible index corruption with HNSW vacuuming - resolves #988
Co-authored-by: Bhagyesh Chaturvedi <bhagyeshc@google.com>
2026-06-16 09:44:34 -07:00
Andrew Kane
32284ba28a Updated CI for latest windows-2025 image [skip ci] 2026-06-15 19:24:02 -07:00
Andrew Kane
1f68c73c96 Updated changelog [skip ci] 2026-06-10 20:20:55 -07:00
Andrew Kane
f15bc0904f Updated changelog [skip ci] 2026-06-10 20:13:55 -07:00
Andrew Kane
2b7ad083dc Always use inline pg_popcount64 for Postgres 19+ - #985 2026-06-10 20:08:03 -07:00
Andrew Kane
421a35fcdc Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18 - fixes #985 2026-06-10 19:55:21 -07:00
Andrew Kane
12368bd79c Updated readme [skip ci] 2026-05-30 11:58:10 -07:00
Andrew Kane
88a0085459 Improved casting [skip ci] 2026-05-26 14:09:19 -07:00
Andrew Kane
ea23884efd Ran latest pgindent [skip ci] 2026-05-26 13:39:55 -07:00
Andrew Kane
3351f3d43e Simplified comment [skip ci] 2026-05-26 13:39:20 -07:00
Andrew Kane
d238409bec Merge pull request #979 from xingtanzjr/fix_ivf_create_concurrently
Fix CREATE INDEX CONCURRENTLY crash on ivfflat with assertion-enabled PostgreSQL
2026-04-27 03:28:52 -07:00
Zhang Jinrui
529f37175b Fix CREATE INDEX CONCURRENTLY crash on ivfflat with assertion-enabled PostgreSQL by passing anyvisible=false in ivfflat SampleRows 2026-04-27 10:15:32 +00:00
Andrew Kane
bce3946392 Added link to PgHero [skip ci] 2026-04-27 02:11:17 -07:00
Andrew Kane
13cb253d30 Added link to PgDog [skip ci] 2026-04-27 00:09:20 -07:00
Andrew Kane
41b3cdc011 Updated readme [skip ci] 2026-04-26 14:47:17 -07:00
Andrew Kane
ec02a96239 Updated readme [skip ci] 2026-04-26 14:45:12 -07:00
Andrew Kane
610d95b8d2 Moved scaling section [skip ci] 2026-04-26 13:12:28 -07:00
Andrew Kane
609d01f4c6 Added more scaling advice to readme [skip ci] 2026-04-26 12:56:07 -07:00
23 changed files with 433 additions and 144 deletions

View File

@@ -97,7 +97,7 @@ jobs:
with:
postgres-version: ${{ matrix.postgres }}
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^

View File

@@ -1,3 +1,13 @@
## 0.8.4 (2026-06-30)
- Fixed `hnsw graph not repaired` error with HNSW vacuuming
- Fixed possible error with inserts during HNSW vacuuming
## 0.8.3 (2026-06-17)
- Fixed possible index corruption with HNSW vacuuming
- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)

View File

@@ -5,7 +5,7 @@ ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR
ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
ADD https://github.com/pgvector/pgvector.git#v0.8.4 /tmp/pgvector
RUN apt-get update && \
apt-mark hold locales && \

View File

@@ -2,12 +2,12 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.2",
"version": "0.8.4",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
"license": {
"PostgreSQL": "http://www.postgresql.org/about/licence"
"PostgreSQL": "https://www.postgresql.org/about/licence"
},
"prereqs": {
"runtime": {
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.8.2",
"version": "0.8.4",
"abstract": "Open-source vector similarity search for Postgres"
}
},
@@ -38,7 +38,7 @@
"generated_by": "Andrew Kane",
"meta-spec": {
"version": "1.0.0",
"url": "http://pgxn.org/meta/spec.txt"
"url": "https://pgxn.org/meta/spec.txt"
},
"tags": [
"vectors",

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.2
EXTVERSION = 0.8.4
MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.8.2
EXTVERSION = 0.8.4
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

View File

@@ -11,6 +11,8 @@ Store your vectors with the rest of your data. Supports:
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
Have a lot of vectors? Use [quantization](#scaling) to scale
[![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions)
## Installation
@@ -21,7 +23,7 @@ Compile and install the extension (supports Postgres 13+)
```sh
cd /tmp
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -38,7 +40,7 @@ Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/buil
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\18"
cd %TEMP%
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -314,6 +316,8 @@ For a large number of workers, you may need to increase `max_parallel_workers` (
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
Use [binary quantization](#binary-quantization) for faster build times at scale
### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
@@ -443,13 +447,7 @@ Exact indexes work well for conditions that match a low percentage of rows. Othe
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
```
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
```sql
SET hnsw.ef_search = 200;
```
Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
```sql
SET hnsw.iterative_scan = strict_order;
@@ -673,6 +671,10 @@ SHOW shared_buffers;
Be sure to restart Postgres for changes to take effect.
### Storing
Use the `halfvec` type instead of `vector` for a smaller working set.
### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
@@ -687,6 +689,8 @@ Add any indexes *after* loading the initial data for best performance.
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
In production environments, create indexes concurrently to avoid blocking writes.
```sql
@@ -717,6 +721,8 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
#### Approximate Search
Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
```sql
@@ -732,21 +738,20 @@ REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name;
```
## Scaling
For a smaller working set:
1. Use the `halfvec` type instead of `vector` for tables
2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Monitoring
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
```sql
CREATE EXTENSION pg_stat_statements;
```
Get the most time-consuming queries with:
```sql
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance.
Monitor recall by comparing results from approximate search with exact search.
@@ -757,14 +762,6 @@ SELECT ...
COMMIT;
```
## Scaling
Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -878,6 +875,8 @@ No, but like other index types, youll likely see better performance if they d
SELECT pg_size_pretty(pg_relation_size('index_name'));
```
Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
## Troubleshooting
#### Why isnt a query using an index?
@@ -1152,23 +1151,23 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
Supported tags are:
- `pg18-trixie`, `0.8.2-pg18-trixie`
- `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
- `pg17-trixie`, `0.8.2-pg17-trixie`
- `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
- `pg16-trixie`, `0.8.2-pg16-trixie`
- `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
- `pg15-trixie`, `0.8.2-pg15-trixie`
- `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
- `pg14-trixie`, `0.8.2-pg14-trixie`
- `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
- `pg13-trixie`, `0.8.2-pg13-trixie`
- `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
- `pg18-trixie`, `0.8.4-pg18-trixie`
- `pg18-bookworm`, `0.8.4-pg18-bookworm`, `pg18`, `0.8.4-pg18`
- `pg17-trixie`, `0.8.4-pg17-trixie`
- `pg17-bookworm`, `0.8.4-pg17-bookworm`, `pg17`, `0.8.4-pg17`
- `pg16-trixie`, `0.8.4-pg16-trixie`
- `pg16-bookworm`, `0.8.4-pg16-bookworm`, `pg16`, `0.8.4-pg16`
- `pg15-trixie`, `0.8.4-pg15-trixie`
- `pg15-bookworm`, `0.8.4-pg15-bookworm`, `pg15`, `0.8.4-pg15`
- `pg14-trixie`, `0.8.4-pg14-trixie`
- `pg14-bookworm`, `0.8.4-pg14-bookworm`, `pg14`, `0.8.4-pg14`
- `pg13-trixie`, `0.8.4-pg13-trixie`
- `pg13-bookworm`, `0.8.4-pg13-bookworm`, `pg13`, `0.8.4-pg13`
You can also build the image manually:
```sh
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
```
@@ -1330,7 +1329,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
To enable benchmarking:
```sh
make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install
```
To show memory usage:

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

View File

@@ -31,10 +31,12 @@
#define BIT_TARGET_CLONES
#endif
/* Use built-ins when possible for inlining */
#if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
/* Use built-ins when possible for Postgres < 19 for inlining */
#if PG_VERSION_NUM >= 190000
#define popcount64(x) pg_popcount64(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
#define popcount64(x) __builtin_popcountl(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8)
#define popcount64(x) __builtin_popcountll(x)
#elif !defined(_MSC_VER)
/* Fails to resolve with MSVC */

View File

@@ -18,6 +18,10 @@
#include "utils/sampling.h"
#include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000
typedef Pointer Item;
#endif
@@ -78,6 +82,21 @@ typedef Pointer Item;
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -408,10 +427,11 @@ typedef struct HnswVacuumState
HnswSupport support;
/* Variables */
struct tidhash_hash *deleted;
struct tidhash_hash *deleting;
BufferAccessStrategy bas;
HnswNeighborTuple ntup;
HnswElementData highestPoint;
HnswElementData fallbackPoint;
/* Memory */
MemoryContext tmpCtx;

View File

@@ -719,7 +719,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */
HnswInitSupport(&buildstate->support, index);
InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024);
buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -956,7 +956,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Leave space for other objects in shared memory */
/* Docker has a default limit of 64 MB for shm_size */
/* which happens to be the default value of maintenance_work_mem */
esthnswarea = maintenance_work_mem * 1024L;
esthnswarea = maintenance_work_mem * (Size) 1024;
estother = 3 * 1024 * 1024;
if (esthnswarea > estother)
esthnswarea -= estother;

View File

@@ -546,6 +546,9 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, Hns
Assert(HnswIsElementTuple(etup));
if (unlikely(etup->deleted))
elog(ERROR, "cannot load deleted element");
/* Calculate distance */
if (distance != NULL)
{

View File

@@ -19,12 +19,12 @@
#endif
/*
* Check if deleted list contains an index TID
* Check if deletion list contains an element
*/
static bool
DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
DeletingElement(tidhash_hash * deleting, ItemPointer indextid)
{
return tidhash_lookup(deleted, *indextid) != NULL;
return tidhash_lookup(deleting, *indextid) != NULL;
}
/*
@@ -37,17 +37,20 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
IndexBulkDeleteResult *stats = vacuumstate->stats;
/* Store separately since highestPoint.level is uint8 */
/* Store separately since HnswElement level is uint8 */
int highestLevel = -1;
int fallbackLevel = -1;
/* Initialize highest point */
/* Initialize highest point and fallback point */
highestPoint->blkno = InvalidBlockNumber;
highestPoint->offno = InvalidOffsetNumber;
fallbackPoint->blkno = InvalidBlockNumber;
fallbackPoint->offno = InvalidOffsetNumber;
while (BlockNumberIsValid(blkno))
{
@@ -77,6 +80,14 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/*
* Skip deleted tuples. It is important they are not added to the
* deletion list to avoid false positives in NeedsUpdated and
* ConfirmRepaired.
*/
if (etup->deleted)
continue;
if (ItemPointerIsValid(&etup->heaptids[0]))
{
for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -110,23 +121,40 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!ItemPointerIsValid(&etup->heaptids[0]))
{
ItemPointerData ip;
ItemPointerData indextid;
bool found;
/* Add to deleted list */
ItemPointerSet(&ip, blkno, offno);
/* Add to deletion list */
ItemPointerSet(&indextid, blkno, offno);
tidhash_insert(vacuumstate->deleted, ip, &found);
tidhash_insert(vacuumstate->deleting, indextid, &found);
Assert(!found);
}
else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
else if (etup->level > highestLevel)
{
/* Keep track of highest non-entry point */
if (BlockNumberIsValid(highestPoint->blkno))
{
/* Current highest point becomes fallback */
fallbackPoint->blkno = highestPoint->blkno;
fallbackPoint->offno = highestPoint->offno;
fallbackPoint->level = highestPoint->level;
fallbackLevel = highestLevel;
}
/* Keep track of highest point */
highestPoint->blkno = blkno;
highestPoint->offno = offno;
highestPoint->level = etup->level;
highestLevel = etup->level;
}
else if (etup->level > fallbackLevel)
{
/* Keep track of second highest point */
fallbackPoint->blkno = blkno;
fallbackPoint->offno = offno;
fallbackPoint->level = etup->level;
fallbackLevel = etup->level;
}
}
blkno = HnswPageGetOpaque(page)->nextblkno;
@@ -138,6 +166,10 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
UnlockReleaseBuffer(buf);
}
#ifdef HNSW_MEMORY
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
#endif
}
/*
@@ -168,8 +200,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deleted list */
if (DeletedContains(vacuumstate->deleted, indextid))
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
{
needsUpdated = true;
break;
@@ -178,13 +210,9 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */
if (!needsUpdated)
{
/* Keep clang-tidy happy */
Assert(ntup->count > 0);
/* There should always be more than zero indextids, but check for safety */
if (!needsUpdated && ntup->count > 0)
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
}
UnlockReleaseBuffer(buf);
@@ -269,12 +297,27 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
/* Get a shared lock */
LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Get latest entry point */
entryPoint = HnswGetEntryPoint(index);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
/* Use fallback point if highest point is entry point */
if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno)
{
highestPoint = &vacuumstate->fallbackPoint;
if (!BlockNumberIsValid(highestPoint->blkno))
highestPoint = NULL;
}
if (highestPoint != NULL)
{
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, entryPoint);
}
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -292,7 +335,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
if (DeletedContains(vacuumstate->deleted, &epData))
if (DeletingElement(vacuumstate->deleting, &epData))
{
/*
* Replace the entry point with the highest point. If highest
@@ -378,6 +421,10 @@ RepairGraph(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip updating neighbors if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
@@ -441,6 +488,103 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Reset memory context */
MemoryContextSwitchTo(oldCtx);
MemoryContextReset(vacuumstate->tmpCtx);
#ifdef HNSW_VACUUM_PROGRESS
if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
{
BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
}
#endif
}
}
/*
* Confirm graph was repaired
*/
static void
ConfirmRepaired(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
while (BlockNumberIsValid(blkno))
{
Buffer buf;
Page page;
OffsetNumber offno;
OffsetNumber maxoffno;
vacuum_delay_point();
buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
OffsetNumber neighborOffno;
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
if (neighborPage == blkno)
{
nbuf = buf;
npage = page;
}
else
{
nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
LockBuffer(nbuf, BUFFER_LOCK_SHARE);
npage = BufferGetPage(nbuf);
}
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Check neighbors */
for (int i = 0; i < ntup->count; i++)
{
ItemPointer indextid = &ntup->indextids[i];
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
elog(ERROR, "hnsw graph not repaired");
}
if (nbuf != buf)
UnlockReleaseBuffer(nbuf);
}
blkno = HnswPageGetOpaque(page)->nextblkno;
UnlockReleaseBuffer(buf);
}
}
@@ -456,10 +600,15 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BufferAccessStrategy bas = vacuumstate->bas;
/*
* Wait for index scans to complete. Scans before this point may contain
* tuples about to be deleted. Scans after this point will not, since the
* graph has been repaired.
* Wait for inserts and index scans to complete. Inserts and scans before
* this point may visit tuples about to be deleted. Inserts and scans
* after this point will not, since the graph has been repaired.
*/
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
ConfirmRepaired(vacuumstate);
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
@@ -607,7 +756,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
/* Create hash table */
vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
}
/*
@@ -616,7 +765,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
static void
FreeVacuumState(HnswVacuumState * vacuumstate)
{
tidhash_destroy(vacuumstate->deleted);
tidhash_destroy(vacuumstate->deleting);
FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx);
@@ -634,13 +783,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
/* Pass 1: Remove heap TIDs */
RemoveHeapTids(&vacuumstate);
HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate));
/* Pass 2: Repair graph */
RepairGraph(&vacuumstate);
HnswBench("RepairGraph", RepairGraph(&vacuumstate));
/* Pass 3: Mark as deleted */
MarkDeleted(&vacuumstate);
/* Passes 3 and 4: Confirm repaired and mark as deleted */
HnswBench("MarkDeleted", MarkDeleted(&vacuumstate));
FreeVacuumState(&vacuumstate);

View File

@@ -145,24 +145,14 @@ SampleRows(IvfflatBuildState * buildstate)
{
BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
/* Set anyvisible to false like table_index_build_scan */
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
}
/* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL)
{
VectorArray samples = buildstate->samples;
for (int i = 0; i < samples->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(samples, i));
Datum normValue = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
VectorArraySet(samples, i, DatumGetPointer(normValue));
pfree(DatumGetPointer(normValue));
}
}
IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx);
}
/*
@@ -398,7 +388,14 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
buildstate->memoryUsed = 0;
buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
/* TODO Move allocation to page creation */
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -452,8 +449,9 @@ ComputeCenters(IvfflatBuildState * buildstate)
numSamples = 1;
/* Sample rows */
/* TODO Ensure within maintenance_work_mem */
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
if (buildstate->heap != NULL)
{
IvfflatBench("sample rows", SampleRows(buildstate));
@@ -468,7 +466,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
}
/* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed));
/* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples);

View File

@@ -204,6 +204,7 @@ typedef struct IvfflatBuildState
VectorArray samples;
VectorArray centers;
ListInfo *listInfo;
Size itemsize;
#ifdef IVFFLAT_KMEANS_DEBUG
double inertia;
@@ -223,6 +224,7 @@ typedef struct IvfflatBuildState
TupleTableSlot *slot;
/* Memory */
Size memoryUsed;
MemoryContext tmpCtx;
/* Parallel builds */
@@ -303,22 +305,32 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
static inline Pointer
VectorArrayGet(VectorArray arr, int offset)
{
if (offset >= arr->maxlen)
elog(ERROR, "safety check failed");
return ((char *) arr->items) + (offset * arr->itemsize);
}
static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val)
{
memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
Size size = VARSIZE_ANY(val);
if (size > arr->itemsize)
elog(ERROR, "safety check failed");
memcpy(VectorArrayGet(arr, offset), val, size);
}
/* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
void IvfflatCheckMemoryUsage(Size totalSize);
int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);

View File

@@ -99,22 +99,8 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context",
ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(normCtx);
for (int j = 0; j < centers->length; j++)
{
Datum center = PointerGetDatum(VectorArrayGet(centers, j));
Datum newCenter = IvfflatNormValue(typeInfo, collation, center);
Size size = VARSIZE_ANY(DatumGetPointer(newCenter));
if (size > centers->itemsize)
elog(ERROR, "safety check failed");
memcpy(DatumGetPointer(center), DatumGetPointer(newCenter), size);
MemoryContextReset(normCtx);
}
MemoryContextSwitchTo(oldCtx);
IvfflatNormVectors(typeInfo, collation, centers, normCtx);
MemoryContextDelete(normCtx);
}
@@ -258,7 +244,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/
static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
{
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
@@ -277,8 +263,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *newcdist;
/* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size centerCountsSize = sizeof(int) * numCenters;
@@ -290,18 +274,13 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */
Size totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
Size totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */
/* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
IvfflatCheckMemoryUsage(totalSize);
/* Ensure indexing does not overflow */
if (numCenters * numCenters > INT_MAX)
if (numCenters > INT_MAX / numCenters)
elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */
@@ -562,7 +541,7 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine
*/
void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed)
{
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context",
@@ -572,7 +551,7 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
if (samples->length == 0)
RandomCenters(index, centers, typeInfo);
else
ElkanKmeans(index, samples, centers, typeInfo);
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed);
CheckCenters(index, centers, typeInfo);

View File

@@ -6,7 +6,9 @@
#include "halfutils.h"
#include "halfvec.h"
#include "ivfflat.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "utils/varbit.h"
#include "vector.h"
@@ -88,6 +90,40 @@ IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
}
/*
* Normalize vectors
*/
void
IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
{
MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
for (int i = 0; i < arr->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(arr, i));
Datum newValue = IvfflatNormValue(typeInfo, collation, value);
VectorArraySet(arr, i, DatumGetPointer(newValue));
MemoryContextReset(tmpCtx);
}
MemoryContextSwitchTo(oldCtx);
}
/*
* Check memory usage
*/
void
IvfflatCheckMemoryUsage(Size totalSize)
{
/* Add one to error message to ceil */
if (totalSize > maintenance_work_mem * (Size) 1024)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
}
/*
* New buffer
*/

View File

@@ -182,10 +182,10 @@ sparsevec_isspace(char ch)
static int
CompareIndices(const void *a, const void *b)
{
if (((SparseInputElement *) a)->index < ((SparseInputElement *) b)->index)
if (((const SparseInputElement *) a)->index < ((const SparseInputElement *) b)->index)
return -1;
if (((SparseInputElement *) a)->index > ((SparseInputElement *) b)->index)
if (((const SparseInputElement *) a)->index > ((const SparseInputElement *) b)->index)
return 1;
return 0;

View File

@@ -40,7 +40,7 @@
#endif
#if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.2");
PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.4");
#else
PG_MODULE_MAGIC;
#endif

View File

@@ -0,0 +1,38 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "DELETE FROM tst");
# Test HNSW_SCAN_LOCK at the beginning of MarkDeleted is effective
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent SELECTs and VACUUM",
{
"046_hnsw_vacuum_scan_select\@1000" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"046_hnsw_vacuum_scan_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -0,0 +1,39 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
# Test no "hnsw graph not repaired" errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.2'
default_version = '0.8.4'
module_pathname = '$libdir/vector'
relocatable = true