Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
434ef7a5ab Updated SparsevecInnerProduct [skip ci] 2026-03-17 14:31:58 -07:00
40 changed files with 175 additions and 785 deletions

View File

@@ -8,12 +8,10 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- postgres: 20
os: ubuntu-26.04
- postgres: 19 - postgres: 19
os: ubuntu-26.04 os: ubuntu-24.04
- postgres: 18 - postgres: 18
os: ubuntu-26.04-arm os: ubuntu-24.04
- postgres: 17 - postgres: 17
os: ubuntu-24.04 os: ubuntu-24.04
- postgres: 16 - postgres: 16
@@ -25,7 +23,7 @@ jobs:
- postgres: 13 - postgres: 13
os: ubuntu-22.04 os: ubuntu-22.04
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
@@ -55,7 +53,7 @@ jobs:
- postgres: 14 - postgres: 14
os: macos-15-intel os: macos-15-intel
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
@@ -94,12 +92,12 @@ jobs:
- postgres: 14 - postgres: 14
os: windows-2022 os: windows-2022
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v6
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
- run: | - run: |
call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^ nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^ nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^ nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^
@@ -135,7 +133,7 @@ jobs:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }} if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v6
- uses: ankane/setup-postgres-valgrind@v1 - uses: ankane/setup-postgres-valgrind@v1
with: with:
postgres-version: 18 postgres-version: 18

View File

@@ -1,18 +1,3 @@
## 0.8.5 (2026-07-08)
- Reduced memory usage for small tables for IVFFlat index builds
## 0.8.4 (2026-06-30)
- Fixed `hnsw graph not repaired` error with HNSW vacuuming
- Fixed possible error with inserts during HNSW vacuuming
- Fixed memory exceeding `maintenance_work_mem` with IVFFlat index builds
## 0.8.3 (2026-06-17)
- Fixed possible index corruption with HNSW vacuuming
- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
## 0.8.2 (2026-02-25) ## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959) - Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)

View File

@@ -5,7 +5,7 @@ ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR ARG PG_MAJOR
ADD https://github.com/pgvector/pgvector.git#v0.8.5 /tmp/pgvector ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
RUN apt-get update && \ RUN apt-get update && \
apt-mark hold locales && \ apt-mark hold locales && \

View File

@@ -2,12 +2,12 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.5", "version": "0.8.2",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
"license": { "license": {
"PostgreSQL": "https://www.postgresql.org/about/licence" "PostgreSQL": "http://www.postgresql.org/about/licence"
}, },
"prereqs": { "prereqs": {
"runtime": { "runtime": {
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.8.5", "version": "0.8.2",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },
@@ -38,7 +38,7 @@
"generated_by": "Andrew Kane", "generated_by": "Andrew Kane",
"meta-spec": { "meta-spec": {
"version": "1.0.0", "version": "1.0.0",
"url": "https://pgxn.org/meta/spec.txt" "url": "http://pgxn.org/meta/spec.txt"
}, },
"tags": [ "tags": [
"vectors", "vectors",

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.5 EXTVERSION = 0.8.2
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql) DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.5 EXTVERSION = 0.8.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

View File

@@ -11,8 +11,6 @@ Store your vectors with the rest of your data. Supports:
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
Have a lot of vectors? Use [quantization](#scaling) to scale
[![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions) [![Build Status](https://github.com/pgvector/pgvector/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector/actions)
## Installation ## Installation
@@ -23,7 +21,7 @@ Compile and install the extension (supports Postgres 13+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -40,7 +38,7 @@ Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/buil
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\18" set "PGROOT=C:\Program Files\PostgreSQL\18"
cd %TEMP% cd %TEMP%
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
@@ -316,8 +314,6 @@ For a large number of workers, you may need to increase `max_parallel_workers` (
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall) The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
Use [binary quantization](#binary-quantization) for faster build times at scale
### Indexing Progress ### Indexing Progress
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING) Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
@@ -447,7 +443,13 @@ Exact indexes work well for conditions that match a low percentage of rows. Othe
CREATE INDEX ON items USING hnsw (embedding vector_l2_ops); CREATE INDEX ON items USING hnsw (embedding vector_l2_ops);
``` ```
With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed. With approximate indexes, filtering is applied *after* the index is scanned. If a condition matches 10% of rows, with HNSW and the default `hnsw.ef_search` of 40, only 4 rows will match on average. For more rows, increase `hnsw.ef_search`.
```sql
SET hnsw.ef_search = 200;
```
Starting with 0.8.0, you can enable [iterative index scans](#iterative-index-scans), which will automatically scan more of the index when needed.
```sql ```sql
SET hnsw.iterative_scan = strict_order; SET hnsw.iterative_scan = strict_order;
@@ -465,16 +467,6 @@ If filtering by many different values, consider [partitioning](https://www.postg
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id); CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
``` ```
## Multitenancy
For applications with multiple tenants, sharing an approximate index between tenants means vectors from one tenant can affect recall (and speed) for other tenants.
For tenant isolation, use [list partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) or separate tables.
```sql
CREATE TABLE items (customer_id int, embedding vector(3)) PARTITION BY LIST(customer_id);
```
## Iterative Index Scans ## Iterative Index Scans
With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`).
@@ -681,10 +673,6 @@ SHOW shared_buffers;
Be sure to restart Postgres for changes to take effect. Be sure to restart Postgres for changes to take effect.
### Storing
Use the `halfvec` type instead of `vector` for a smaller working set.
### Loading ### Loading
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)). Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
@@ -699,8 +687,6 @@ Add any indexes *after* loading the initial data for best performance.
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1). See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
In production environments, create indexes concurrently to avoid blocking writes. In production environments, create indexes concurrently to avoid blocking writes.
```sql ```sql
@@ -731,8 +717,6 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
#### Approximate Search #### Approximate Search
Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall). To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
```sql ```sql
@@ -748,20 +732,21 @@ REINDEX INDEX CONCURRENTLY index_name;
VACUUM table_name; VACUUM table_name;
``` ```
## Scaling
For a smaller working set:
1. Use the `halfvec` type instead of `vector` for tables
2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Monitoring ## Monitoring
Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance. Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
```sql
CREATE EXTENSION pg_stat_statements;
```
Get the most time-consuming queries with:
```sql
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
```
Monitor recall by comparing results from approximate search with exact search. Monitor recall by comparing results from approximate search with exact search.
@@ -772,6 +757,14 @@ SELECT ...
COMMIT; COMMIT;
``` ```
## Scaling
Scale pgvector the same way you scale Postgres.
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
## Languages ## Languages
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another. Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
@@ -885,8 +878,6 @@ No, but like other index types, youll likely see better performance if they d
SELECT pg_size_pretty(pg_relation_size('index_name')); SELECT pg_size_pretty(pg_relation_size('index_name'));
``` ```
Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
## Troubleshooting ## Troubleshooting
#### Why isnt a query using an index? #### Why isnt a query using an index?
@@ -1161,23 +1152,23 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
Supported tags are: Supported tags are:
- `pg18-trixie`, `0.8.5-pg18-trixie` - `pg18-trixie`, `0.8.2-pg18-trixie`
- `pg18-bookworm`, `0.8.5-pg18-bookworm`, `pg18`, `0.8.5-pg18` - `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
- `pg17-trixie`, `0.8.5-pg17-trixie` - `pg17-trixie`, `0.8.2-pg17-trixie`
- `pg17-bookworm`, `0.8.5-pg17-bookworm`, `pg17`, `0.8.5-pg17` - `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
- `pg16-trixie`, `0.8.5-pg16-trixie` - `pg16-trixie`, `0.8.2-pg16-trixie`
- `pg16-bookworm`, `0.8.5-pg16-bookworm`, `pg16`, `0.8.5-pg16` - `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
- `pg15-trixie`, `0.8.5-pg15-trixie` - `pg15-trixie`, `0.8.2-pg15-trixie`
- `pg15-bookworm`, `0.8.5-pg15-bookworm`, `pg15`, `0.8.5-pg15` - `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
- `pg14-trixie`, `0.8.5-pg14-trixie` - `pg14-trixie`, `0.8.2-pg14-trixie`
- `pg14-bookworm`, `0.8.5-pg14-bookworm`, `pg14`, `0.8.5-pg14` - `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
- `pg13-trixie`, `0.8.5-pg13-trixie` - `pg13-trixie`, `0.8.2-pg13-trixie`
- `pg13-bookworm`, `0.8.5-pg13-bookworm`, `pg13`, `0.8.5-pg13` - `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.8.5 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector . docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
``` ```
@@ -1233,7 +1224,7 @@ Note: Replace `18` with your Postgres server version
Install the FreeBSD package with: Install the FreeBSD package with:
```sh ```sh
pkg install postgresql18-pgvector pkg install postgresql17-pgvector
``` ```
or the port with: or the port with:
@@ -1339,7 +1330,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
To enable benchmarking: To enable benchmarking:
```sh ```sh
make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
``` ```
To show memory usage: To show memory usage:

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.5'" to load this file. \quit

View File

@@ -31,12 +31,10 @@
#define BIT_TARGET_CLONES #define BIT_TARGET_CLONES
#endif #endif
/* Use built-ins when possible for Postgres < 19 for inlining */ /* Use built-ins when possible for inlining */
#if PG_VERSION_NUM >= 190000 #if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
#define popcount64(x) pg_popcount64(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
#define popcount64(x) __builtin_popcountl(x) #define popcount64(x) __builtin_popcountl(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8) #elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
#define popcount64(x) __builtin_popcountll(x) #define popcount64(x) __builtin_popcountll(x)
#elif !defined(_MSC_VER) #elif !defined(_MSC_VER)
/* Fails to resolve with MSVC */ /* Fails to resolve with MSVC */
@@ -171,7 +169,7 @@ BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char
#endif #endif
TARGET_XSAVE static bool TARGET_XSAVE static bool
SupportsAvx512Popcount(void) SupportsAvx512Popcount()
{ {
unsigned int exx[4] = {0, 0, 0, 0}; unsigned int exx[4] = {0, 0, 0, 0};

View File

@@ -13,7 +13,6 @@
#include "hnsw.h" #include "hnsw.h"
#include "miscadmin.h" #include "miscadmin.h"
#include "nodes/pg_list.h" #include "nodes/pg_list.h"
#include "storage/lwlock.h"
#include "utils/float.h" #include "utils/float.h"
#include "utils/guc.h" #include "utils/guc.h"
#include "utils/relcache.h" #include "utils/relcache.h"

View File

@@ -10,18 +10,10 @@
#include "lib/pairingheap.h" #include "lib/pairingheap.h"
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "port.h" /* for random() */ #include "port.h" /* for random() */
#include "storage/bufpage.h"
#include "storage/condition_variable.h"
#include "storage/lwlock.h"
#include "storage/s_lock.h"
#include "utils/relptr.h" #include "utils/relptr.h"
#include "utils/sampling.h" #include "utils/sampling.h"
#include "vector.h" #include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000 #if PG_VERSION_NUM >= 190000
typedef Pointer Item; typedef Pointer Item;
#endif #endif
@@ -82,21 +74,6 @@ typedef Pointer Item;
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page)) #define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page)) #define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000 #if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state) #define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed) #define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -427,11 +404,10 @@ typedef struct HnswVacuumState
HnswSupport support; HnswSupport support;
/* Variables */ /* Variables */
struct tidhash_hash *deleting; struct tidhash_hash *deleted;
BufferAccessStrategy bas; BufferAccessStrategy bas;
HnswNeighborTuple ntup; HnswNeighborTuple ntup;
HnswElementData highestPoint; HnswElementData highestPoint;
HnswElementData fallbackPoint;
/* Memory */ /* Memory */
MemoryContext tmpCtx; MemoryContext tmpCtx;

View File

@@ -54,7 +54,6 @@
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "optimizer/optimizer.h" #include "optimizer/optimizer.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h" #include "tcop/tcopprot.h"
#include "utils/datum.h" #include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
@@ -719,7 +718,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */ /* Get support functions */
HnswInitSupport(&buildstate->support, index); HnswInitSupport(&buildstate->support, index);
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024); InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
buildstate->graph = &buildstate->graphData; buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m); buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m); buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -803,11 +802,7 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
buildstate.hnswarea = hnswarea; buildstate.hnswarea = hnswarea;
InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate); InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
scan = table_beginscan_parallel(heapRel, scan = table_beginscan_parallel(heapRel,
ParallelTableScanFromHnswShared(hnswshared) ParallelTableScanFromHnswShared(hnswshared));
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
reltuples = table_index_build_scan(heapRel, indexRel, indexInfo, reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
true, progress, BuildCallback, true, progress, BuildCallback,
(void *) &buildstate, scan); (void *) &buildstate, scan);
@@ -956,7 +951,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Leave space for other objects in shared memory */ /* Leave space for other objects in shared memory */
/* Docker has a default limit of 64 MB for shm_size */ /* Docker has a default limit of 64 MB for shm_size */
/* which happens to be the default value of maintenance_work_mem */ /* which happens to be the default value of maintenance_work_mem */
esthnswarea = maintenance_work_mem * (Size) 1024; esthnswarea = maintenance_work_mem * 1024L;
estother = 3 * 1024 * 1024; estother = 3 * 1024 * 1024;
if (esthnswarea > estother) if (esthnswarea > estother)
esthnswarea -= estother; esthnswarea -= estother;

View File

@@ -6,7 +6,6 @@
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "storage/lmgr.h" #include "storage/lmgr.h"
#include "storage/lwlock.h"
#include "utils/datum.h" #include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#include "utils/rel.h" #include "utils/rel.h"

View File

@@ -546,9 +546,6 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, Hns
Assert(HnswIsElementTuple(etup)); Assert(HnswIsElementTuple(etup));
if (unlikely(etup->deleted))
elog(ERROR, "cannot load deleted element");
/* Calculate distance */ /* Calculate distance */
if (distance != NULL) if (distance != NULL)
{ {

View File

@@ -19,12 +19,12 @@
#endif #endif
/* /*
* Check if deletion list contains an element * Check if deleted list contains an index TID
*/ */
static bool static bool
DeletingElement(tidhash_hash * deleting, ItemPointer indextid) DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
{ {
return tidhash_lookup(deleting, *indextid) != NULL; return tidhash_lookup(deleted, *indextid) != NULL;
} }
/* /*
@@ -37,20 +37,17 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
{ {
BlockNumber blkno = HNSW_HEAD_BLKNO; BlockNumber blkno = HNSW_HEAD_BLKNO;
HnswElement highestPoint = &vacuumstate->highestPoint; HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
Relation index = vacuumstate->index; Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
IndexBulkDeleteResult *stats = vacuumstate->stats; IndexBulkDeleteResult *stats = vacuumstate->stats;
/* Store separately since HnswElement level is uint8 */ /* Store separately since highestPoint.level is uint8 */
int highestLevel = -1; int highestLevel = -1;
int fallbackLevel = -1;
/* Initialize highest point and fallback point */ /* Initialize highest point */
highestPoint->blkno = InvalidBlockNumber; highestPoint->blkno = InvalidBlockNumber;
highestPoint->offno = InvalidOffsetNumber; highestPoint->offno = InvalidOffsetNumber;
fallbackPoint->blkno = InvalidBlockNumber;
fallbackPoint->offno = InvalidOffsetNumber;
while (BlockNumberIsValid(blkno)) while (BlockNumberIsValid(blkno))
{ {
@@ -80,14 +77,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup)) if (!HnswIsElementTuple(etup))
continue; continue;
/*
* Skip deleted tuples. It is important they are not added to the
* deletion list to avoid false positives in NeedsUpdated and
* ConfirmRepaired.
*/
if (etup->deleted)
continue;
if (ItemPointerIsValid(&etup->heaptids[0])) if (ItemPointerIsValid(&etup->heaptids[0]))
{ {
for (int i = 0; i < HNSW_HEAPTIDS; i++) for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -121,40 +110,23 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!ItemPointerIsValid(&etup->heaptids[0])) if (!ItemPointerIsValid(&etup->heaptids[0]))
{ {
ItemPointerData indextid; ItemPointerData ip;
bool found; bool found;
/* Add to deletion list */ /* Add to deleted list */
ItemPointerSet(&indextid, blkno, offno); ItemPointerSet(&ip, blkno, offno);
tidhash_insert(vacuumstate->deleting, indextid, &found); tidhash_insert(vacuumstate->deleted, ip, &found);
Assert(!found); Assert(!found);
} }
else if (etup->level > highestLevel) else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
{ {
if (BlockNumberIsValid(highestPoint->blkno)) /* Keep track of highest non-entry point */
{
/* Current highest point becomes fallback */
fallbackPoint->blkno = highestPoint->blkno;
fallbackPoint->offno = highestPoint->offno;
fallbackPoint->level = highestPoint->level;
fallbackLevel = highestLevel;
}
/* Keep track of highest point */
highestPoint->blkno = blkno; highestPoint->blkno = blkno;
highestPoint->offno = offno; highestPoint->offno = offno;
highestPoint->level = etup->level; highestPoint->level = etup->level;
highestLevel = etup->level; highestLevel = etup->level;
} }
else if (etup->level > fallbackLevel)
{
/* Keep track of second highest point */
fallbackPoint->blkno = blkno;
fallbackPoint->offno = offno;
fallbackPoint->level = etup->level;
fallbackLevel = etup->level;
}
} }
blkno = HnswPageGetOpaque(page)->nextblkno; blkno = HnswPageGetOpaque(page)->nextblkno;
@@ -166,10 +138,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
#ifdef HNSW_MEMORY
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
#endif
} }
/* /*
@@ -200,8 +168,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
if (!ItemPointerIsValid(indextid)) if (!ItemPointerIsValid(indextid))
continue; continue;
/* Check if in deletion list */ /* Check if in deleted list */
if (DeletingElement(vacuumstate->deleting, indextid)) if (DeletedContains(vacuumstate->deleted, indextid))
{ {
needsUpdated = true; needsUpdated = true;
break; break;
@@ -210,9 +178,13 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */ /* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */ /* This could indicate too many candidates being deleted during insert */
/* There should always be more than zero indextids, but check for safety */ if (!needsUpdated)
if (!needsUpdated && ntup->count > 0) {
/* Keep clang-tidy happy */
Assert(ntup->count > 0);
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]); needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
}
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
@@ -297,27 +269,12 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
/* Get a shared lock */ /* Get a shared lock */
LockPage(index, HNSW_UPDATE_LOCK, ShareLock); LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Get latest entry point */
entryPoint = HnswGetEntryPoint(index);
/* Use fallback point if highest point is entry point */
if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno)
{
highestPoint = &vacuumstate->fallbackPoint;
if (!BlockNumberIsValid(highestPoint->blkno))
highestPoint = NULL;
}
if (highestPoint != NULL)
{
/* Load element */ /* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL); HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Repair if needed */ /* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint)) if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, entryPoint); RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
}
/* Release lock */ /* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock); UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -335,7 +292,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno); ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
if (DeletingElement(vacuumstate->deleting, &epData)) if (DeletedContains(vacuumstate->deleted, &epData))
{ {
/* /*
* Replace the entry point with the highest point. If highest * Replace the entry point with the highest point. If highest
@@ -421,10 +378,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup)) if (!HnswIsElementTuple(etup))
continue; continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip updating neighbors if being deleted */ /* Skip updating neighbors if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0])) if (!ItemPointerIsValid(&etup->heaptids[0]))
continue; continue;
@@ -488,103 +441,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Reset memory context */ /* Reset memory context */
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);
MemoryContextReset(vacuumstate->tmpCtx); MemoryContextReset(vacuumstate->tmpCtx);
#ifdef HNSW_VACUUM_PROGRESS
if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
{
BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
}
#endif
}
}
/*
* Confirm graph was repaired
*/
static void
ConfirmRepaired(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
while (BlockNumberIsValid(blkno))
{
Buffer buf;
Page page;
OffsetNumber offno;
OffsetNumber maxoffno;
vacuum_delay_point();
buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
OffsetNumber neighborOffno;
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
if (neighborPage == blkno)
{
nbuf = buf;
npage = page;
}
else
{
nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
LockBuffer(nbuf, BUFFER_LOCK_SHARE);
npage = BufferGetPage(nbuf);
}
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Check neighbors */
for (int i = 0; i < ntup->count; i++)
{
ItemPointer indextid = &ntup->indextids[i];
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
elog(ERROR, "hnsw graph not repaired");
}
if (nbuf != buf)
UnlockReleaseBuffer(nbuf);
}
blkno = HnswPageGetOpaque(page)->nextblkno;
UnlockReleaseBuffer(buf);
} }
} }
@@ -600,15 +456,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
/* /*
* Wait for inserts and index scans to complete. Inserts and scans before * Wait for index scans to complete. Scans before this point may contain
* this point may visit tuples about to be deleted. Inserts and scans * tuples about to be deleted. Scans after this point will not, since the
* after this point will not, since the graph has been repaired. * graph has been repaired.
*/ */
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
ConfirmRepaired(vacuumstate);
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock); LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock); UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
@@ -756,7 +607,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL); HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
/* Create hash table */ /* Create hash table */
vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL); vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
} }
/* /*
@@ -765,7 +616,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
static void static void
FreeVacuumState(HnswVacuumState * vacuumstate) FreeVacuumState(HnswVacuumState * vacuumstate)
{ {
tidhash_destroy(vacuumstate->deleting); tidhash_destroy(vacuumstate->deleted);
FreeAccessStrategy(vacuumstate->bas); FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup); pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx); MemoryContextDelete(vacuumstate->tmpCtx);
@@ -783,13 +634,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
InitVacuumState(&vacuumstate, info, stats, callback, callback_state); InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
/* Pass 1: Remove heap TIDs */ /* Pass 1: Remove heap TIDs */
HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate)); RemoveHeapTids(&vacuumstate);
/* Pass 2: Repair graph */ /* Pass 2: Repair graph */
HnswBench("RepairGraph", RepairGraph(&vacuumstate)); RepairGraph(&vacuumstate);
/* Passes 3 and 4: Confirm repaired and mark as deleted */ /* Pass 3: Mark as deleted */
HnswBench("MarkDeleted", MarkDeleted(&vacuumstate)); MarkDeleted(&vacuumstate);
FreeVacuumState(&vacuumstate); FreeVacuumState(&vacuumstate);

View File

@@ -22,7 +22,6 @@
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "optimizer/optimizer.h" #include "optimizer/optimizer.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "tcop/tcopprot.h" #include "tcop/tcopprot.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#include "utils/rel.h" #include "utils/rel.h"
@@ -63,13 +62,15 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* /*
* Check with KMEANS_NORM_PROC that the value can be normalized since * Normalize with KMEANS_NORM_PROC since spherical distance function
* spherical distance function expects unit vectors * expects unit vectors
*/ */
if (buildstate->kmeansnormprocinfo != NULL) if (buildstate->kmeansnormprocinfo != NULL)
{ {
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value)) if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
return; return;
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
} }
if (samples->length < targsamples) if (samples->length < targsamples)
@@ -80,7 +81,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
else else
{ {
if (buildstate->rowstoskip < 0) if (buildstate->rowstoskip < 0)
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples); buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
if (buildstate->rowstoskip <= 0) if (buildstate->rowstoskip <= 0)
{ {
@@ -96,9 +97,6 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
buildstate->rowstoskip -= 1; buildstate->rowstoskip -= 1;
} }
/* Increment after reservoir_get_next_S */
buildstate->samplerows += 1;
} }
/* /*
@@ -135,7 +133,6 @@ SampleRows(IvfflatBuildState * buildstate)
int targsamples = buildstate->samples->maxlen; int targsamples = buildstate->samples->maxlen;
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap); BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
buildstate->samplerows = 0;
buildstate->rowstoskip = -1; buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt()); BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
@@ -145,14 +142,9 @@ SampleRows(IvfflatBuildState * buildstate)
{ {
BlockNumber targblock = BlockSampler_Next(&buildstate->bs); BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
/* Set anyvisible to false like table_index_build_scan */
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo, table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL); false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
} }
/* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL)
IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx);
} }
/* /*
@@ -382,20 +374,10 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0); TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(buildstate->sortdesc);
#endif
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual); buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
buildstate->memoryUsed = 0; buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
/* TODO Move allocation to page creation */
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists); buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext, buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -438,30 +420,22 @@ ComputeCenters(IvfflatBuildState * buildstate)
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS); pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
/* Skip samples for unlogged table */
if (buildstate->heap == NULL)
numSamples = 1;
else
{
int64 maxTuples = (int64) RelationGetNumberOfBlocks(buildstate->heap) * MaxHeapTuplesPerPage;
/* Target 50 samples per list, with at least 10000 samples */ /* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */ /* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50; numSamples = buildstate->lists * 50;
if (numSamples < 10000) if (numSamples < 10000)
numSamples = 10000; numSamples = 10000;
/* Save memory since will not have more than max tuples */ /* Skip samples for unlogged table */
numSamples = Max(Min(numSamples, maxTuples), 1); if (buildstate->heap == NULL)
} numSamples = 1;
/* Sample rows */ /* Sample rows */
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize); /* TODO Ensure within maintenance_work_mem */
IvfflatCheckMemoryUsage(buildstate->memoryUsed); buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
{ {
IvfflatBench("sample rows", SampleRows(buildstate)); SampleRows(buildstate);
if (buildstate->samples->length < buildstate->lists) if (buildstate->samples->length < buildstate->lists)
{ {
@@ -473,7 +447,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
} }
/* Calculate centers */ /* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed)); IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
/* Free samples before we allocate more memory */ /* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples); VectorArrayFree(buildstate->samples);
@@ -676,11 +650,7 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate); ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
buildstate.sortstate = ivfspool->sortstate; buildstate.sortstate = ivfspool->sortstate;
scan = table_beginscan_parallel(ivfspool->heap, scan = table_beginscan_parallel(ivfspool->heap,
ParallelTableScanFromIvfflatShared(ivfshared) ParallelTableScanFromIvfflatShared(ivfshared));
#if PG_VERSION_NUM >= 190000
,SO_NONE
#endif
);
reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo, reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
true, progress, BuildCallback, true, progress, BuildCallback,
(void *) &buildstate, scan); (void *) &buildstate, scan);

View File

@@ -9,7 +9,6 @@
#include "lib/pairingheap.h" #include "lib/pairingheap.h"
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "port.h" /* for random() */ #include "port.h" /* for random() */
#include "storage/condition_variable.h"
#include "utils/sampling.h" #include "utils/sampling.h"
#include "utils/tuplesort.h" #include "utils/tuplesort.h"
#include "vector.h" #include "vector.h"
@@ -204,7 +203,6 @@ typedef struct IvfflatBuildState
VectorArray samples; VectorArray samples;
VectorArray centers; VectorArray centers;
ListInfo *listInfo; ListInfo *listInfo;
Size itemsize;
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
double inertia; double inertia;
@@ -215,8 +213,7 @@ typedef struct IvfflatBuildState
/* Sampling */ /* Sampling */
BlockSamplerData bs; BlockSamplerData bs;
ReservoirStateData rstate; ReservoirStateData rstate;
double samplerows; int rowstoskip;
double rowstoskip;
/* Sorting */ /* Sorting */
Tuplesortstate *sortstate; Tuplesortstate *sortstate;
@@ -224,7 +221,6 @@ typedef struct IvfflatBuildState
TupleTableSlot *slot; TupleTableSlot *slot;
/* Memory */ /* Memory */
Size memoryUsed;
MemoryContext tmpCtx; MemoryContext tmpCtx;
/* Parallel builds */ /* Parallel builds */
@@ -305,32 +301,22 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
static inline Pointer static inline Pointer
VectorArrayGet(VectorArray arr, int offset) VectorArrayGet(VectorArray arr, int offset)
{ {
if (offset >= arr->maxlen)
elog(ERROR, "safety check failed");
return ((char *) arr->items) + (offset * arr->itemsize); return ((char *) arr->items) + (offset * arr->itemsize);
} }
static inline void static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val) VectorArraySet(VectorArray arr, int offset, Pointer val)
{ {
Size size = VARSIZE_ANY(val); memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
if (size > arr->itemsize)
elog(ERROR, "safety check failed");
memcpy(VectorArrayGet(arr, offset), val, size);
} }
/* Methods */ /* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize); VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
void VectorArrayFree(VectorArray arr); void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed); void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum); FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value); Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value); bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
void IvfflatCheckMemoryUsage(Size totalSize);
int IvfflatGetLists(Relation index); int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions); void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum); void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);

View File

@@ -99,8 +99,22 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext, MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context", "Ivfflat norm temporary context",
ALLOCSET_DEFAULT_SIZES); ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(normCtx);
IvfflatNormVectors(typeInfo, collation, centers, normCtx); for (int j = 0; j < centers->length; j++)
{
Datum center = PointerGetDatum(VectorArrayGet(centers, j));
Datum newCenter = IvfflatNormValue(typeInfo, collation, center);
Size size = VARSIZE_ANY(DatumGetPointer(newCenter));
if (size > centers->itemsize)
elog(ERROR, "safety check failed");
memcpy(DatumGetPointer(center), DatumGetPointer(newCenter), size);
MemoryContextReset(normCtx);
}
MemoryContextSwitchTo(oldCtx);
MemoryContextDelete(normCtx); MemoryContextDelete(normCtx);
} }
@@ -244,7 +258,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf * https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/ */
static void static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed) ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{ {
FmgrInfo *procinfo; FmgrInfo *procinfo;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
@@ -263,6 +277,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *newcdist; float *newcdist;
/* Calculate allocation sizes */ /* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize); Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions; Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size centerCountsSize = sizeof(int) * numCenters; Size centerCountsSize = sizeof(int) * numCenters;
@@ -274,13 +290,18 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters; Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */ /* Calculate total size */
Size totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize; Size totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */ /* Check memory requirements */
IvfflatCheckMemoryUsage(totalSize); /* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
/* Ensure indexing does not overflow */ /* Ensure indexing does not overflow */
if (numCenters > INT_MAX / numCenters) if (numCenters * numCenters > INT_MAX)
elog(ERROR, "Indexing overflow detected. Please report a bug."); elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */ /* Set support functions */
@@ -541,7 +562,7 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine * We use spherical k-means for inner product and cosine
*/ */
void void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed) IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{ {
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext, MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context", "Ivfflat kmeans temporary context",
@@ -551,7 +572,7 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
if (samples->length == 0) if (samples->length == 0)
RandomCenters(index, centers, typeInfo); RandomCenters(index, centers, typeInfo);
else else
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed); ElkanKmeans(index, samples, centers, typeInfo);
CheckCenters(index, centers, typeInfo); CheckCenters(index, centers, typeInfo);

View File

@@ -298,9 +298,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
so->tupdesc = CreateTemplateTupleDesc(2); so->tupdesc = CreateTemplateTupleDesc(2);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0); TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
#if PG_VERSION_NUM >= 190000
TupleDescFinalize(so->tupdesc);
#endif
/* Prep sort */ /* Prep sort */
so->sortstate = InitScanSortState(so->tupdesc); so->sortstate = InitScanSortState(so->tupdesc);

View File

@@ -6,9 +6,7 @@
#include "halfutils.h" #include "halfutils.h"
#include "halfvec.h" #include "halfvec.h"
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/relcache.h" #include "utils/relcache.h"
#include "utils/varbit.h" #include "utils/varbit.h"
#include "vector.h" #include "vector.h"
@@ -23,15 +21,11 @@
VectorArray VectorArray
VectorArrayInit(int maxlen, int dimensions, Size itemsize) VectorArrayInit(int maxlen, int dimensions, Size itemsize)
{ {
VectorArray res; VectorArray res = palloc(sizeof(VectorArrayData));
if (maxlen < 1 || dimensions < 1 || itemsize == 0)
elog(ERROR, "cannot create vector array");
/* Ensure items are aligned to prevent UB */ /* Ensure items are aligned to prevent UB */
itemsize = MAXALIGN(itemsize); itemsize = MAXALIGN(itemsize);
res = palloc(sizeof(VectorArrayData));
res->length = 0; res->length = 0;
res->maxlen = maxlen; res->maxlen = maxlen;
res->dim = dimensions; res->dim = dimensions;
@@ -94,40 +88,6 @@ IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0; return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
} }
/*
* Normalize vectors
*/
void
IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
{
MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
for (int i = 0; i < arr->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(arr, i));
Datum newValue = IvfflatNormValue(typeInfo, collation, value);
VectorArraySet(arr, i, DatumGetPointer(newValue));
MemoryContextReset(tmpCtx);
}
MemoryContextSwitchTo(oldCtx);
}
/*
* Check memory usage
*/
void
IvfflatCheckMemoryUsage(Size totalSize)
{
/* Add one to error message to ceil */
if (totalSize > maintenance_work_mem * (Size) 1024)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
}
/* /*
* New buffer * New buffer
*/ */

View File

@@ -182,10 +182,10 @@ sparsevec_isspace(char ch)
static int static int
CompareIndices(const void *a, const void *b) CompareIndices(const void *a, const void *b)
{ {
if (((const SparseInputElement *) a)->index < ((const SparseInputElement *) b)->index) if (((SparseInputElement *) a)->index < ((SparseInputElement *) b)->index)
return -1; return -1;
if (((const SparseInputElement *) a)->index > ((const SparseInputElement *) b)->index) if (((SparseInputElement *) a)->index > ((SparseInputElement *) b)->index)
return 1; return 1;
return 0; return 0;
@@ -895,28 +895,24 @@ SparsevecInnerProduct(SparseVector * a, SparseVector * b)
float *ax = SPARSEVEC_VALUES(a); float *ax = SPARSEVEC_VALUES(a);
float *bx = SPARSEVEC_VALUES(b); float *bx = SPARSEVEC_VALUES(b);
float distance = 0.0; float distance = 0.0;
int bpos = 0; int i = 0;
int j = 0;
for (int i = 0; i < a->nnz; i++) while (i < a->nnz && j < b->nnz)
{ {
int ai = a->indices[i]; int ai = a->indices[i];
for (int j = bpos; j < b->nnz; j++)
{
int bi = b->indices[j]; int bi = b->indices[j];
/* Only update when the same index */
if (ai == bi) if (ai == bi)
{
distance += ax[i] * bx[j]; distance += ax[i] * bx[j];
i++;
/* Update start for next iteration */ j++;
if (ai >= bi)
bpos = j + 1;
/* Found or passed it */
if (bi >= ai)
break;
} }
else if (ai < bi)
i++;
else
j++;
} }
return distance; return distance;

View File

@@ -40,7 +40,7 @@
#endif #endif
#if PG_VERSION_NUM >= 180000 #if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.5"); PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.2");
#else #else
PG_MODULE_MAGIC; PG_MODULE_MAGIC;
#endif #endif

View File

@@ -49,11 +49,3 @@ CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -100,11 +100,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t
(1 row) (1 row)
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -161,7 +161,6 @@ ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000". DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m ERROR: ef_construction must be greater than or equal to 2 * m
DROP TABLE t;
SHOW hnsw.ef_search; SHOW hnsw.ef_search;
hnsw.ef_search hnsw.ef_search
---------------- ----------------
@@ -199,11 +198,4 @@ SET hnsw.scan_mem_multiplier = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000) ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
SET hnsw.scan_mem_multiplier = 1001; SET hnsw.scan_mem_multiplier = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000) ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for hnsw index
DROP TABLE t; DROP TABLE t;

View File

@@ -35,32 +35,3 @@ NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall. DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data. HINT: Drop the index until the table has more data.
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -82,32 +82,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t
(1 row) (1 row)
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -143,7 +143,6 @@ DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists" ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768". DETAIL: Valid values are between "1" and "32768".
DROP TABLE t;
SHOW ivfflat.probes; SHOW ivfflat.probes;
ivfflat.probes ivfflat.probes
---------------- ----------------
@@ -173,32 +172,4 @@ SET ivfflat.max_probes = 0;
ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768) ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
SET ivfflat.max_probes = 32769; SET ivfflat.max_probes = 32769;
ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768) ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t; DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -33,13 +33,3 @@ CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;

View File

@@ -56,13 +56,3 @@ SELECT * FROM t ORDER BY val <+> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;

View File

@@ -95,28 +95,23 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
DROP TABLE t;
SHOW hnsw.ef_search; SHOW hnsw.ef_search;
SET hnsw.ef_search = 0; SET hnsw.ef_search = 0;
SET hnsw.ef_search = 1001; SET hnsw.ef_search = 1001;
SHOW hnsw.iterative_scan; SHOW hnsw.iterative_scan;
SET hnsw.iterative_scan = on; SET hnsw.iterative_scan = on;
SHOW hnsw.max_scan_tuples; SHOW hnsw.max_scan_tuples;
SET hnsw.max_scan_tuples = 0; SET hnsw.max_scan_tuples = 0;
SHOW hnsw.scan_mem_multiplier; SHOW hnsw.scan_mem_multiplier;
SET hnsw.scan_mem_multiplier = 0; SET hnsw.scan_mem_multiplier = 0;
SET hnsw.scan_mem_multiplier = 1001; SET hnsw.scan_mem_multiplier = 1001;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t; DROP TABLE t;

View File

@@ -21,28 +21,3 @@ CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1)
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1); CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5); CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -43,28 +43,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -81,40 +81,19 @@ DROP TABLE t;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
DROP TABLE t;
SHOW ivfflat.probes; SHOW ivfflat.probes;
SET ivfflat.probes = 0; SET ivfflat.probes = 0;
SET ivfflat.probes = 32769; SET ivfflat.probes = 32769;
SHOW ivfflat.iterative_scan; SHOW ivfflat.iterative_scan;
SET ivfflat.iterative_scan = on; SET ivfflat.iterative_scan = on;
SHOW ivfflat.max_probes; SHOW ivfflat.max_probes;
SET ivfflat.max_probes = 0; SET ivfflat.max_probes = 0;
SET ivfflat.max_probes = 32769; SET ivfflat.max_probes = 32769;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t; DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -1,38 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "DELETE FROM tst");
# Test HNSW_SCAN_LOCK at the beginning of MarkDeleted is effective
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent SELECTs and VACUUM",
{
"046_hnsw_vacuum_scan_select\@1000" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"046_hnsw_vacuum_scan_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,39 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
# Test no "hnsw graph not repaired" errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
"047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,39 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v vector_l2_ops) WITH (lists = 10);");
# Test no errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1500",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"048_ivfflat_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"048_ivfflat_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"048_ivfflat_vacuum_insert_select\@500" => "SELECT i FROM tst ORDER BY v <-> (SELECT ARRAY[$array_sql]::vector) LIMIT 10;",
"048_ivfflat_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods' comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.5' default_version = '0.8.2'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true