Compare commits

..

10 Commits

Author SHA1 Message Date
Andrew Kane
89dc41ce2b Fixed crash with debugging disabled [skip ci] 2026-06-10 14:31:54 -07:00
Andrew Kane
06ab41c094 Removed extra whitespace [skip ci] 2026-06-10 13:49:39 -07:00
Andrew Kane
8a8bc90d6d Added support for amgetbatch for IVFFlat [skip ci] 2026-06-10 13:47:36 -07:00
Andrew Kane
91717c81b3 Improved batch size flexibility [skip ci] 2026-06-10 12:10:20 -07:00
Andrew Kane
98a7f94608 Added comment [skip ci] 2026-06-10 11:59:38 -07:00
Andrew Kane
7db3d67b2d Re-added check for iterative scans [skip ci] 2026-06-10 11:58:19 -07:00
Andrew Kane
1dd39f66de Minimized diff [skip ci] 2026-06-10 11:56:52 -07:00
Andrew Kane
cd431c51ca Removed unneeded check [skip ci] 2026-06-10 11:55:31 -07:00
Andrew Kane
73890f2f70 Fixed comment [skip ci] 2026-06-10 11:46:15 -07:00
Andrew Kane
18980387ec Added support for amgetbatch [skip ci] 2026-06-10 11:31:57 -07:00
38 changed files with 243 additions and 721 deletions

View File

@@ -97,7 +97,7 @@ jobs:
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
- run: | - run: |
call "C:\Program Files\Microsoft Visual Studio\${{ matrix.os == 'windows-2025' && 18 || 2022 }}\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
nmake /NOLOGO /F Makefile.win && ^ nmake /NOLOGO /F Makefile.win && ^
nmake /NOLOGO /F Makefile.win install && ^ nmake /NOLOGO /F Makefile.win install && ^
nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^ nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^

View File

@@ -1,18 +1,3 @@
## 0.8.5 (unreleased)
- Reduced memory usage for small tables for IVFFlat index builds
## 0.8.4 (2026-06-30)
- Fixed `hnsw graph not repaired` error with HNSW vacuuming
- Fixed possible error with inserts during HNSW vacuuming
- Fixed memory exceeding `maintenance_work_mem` with IVFFlat index builds
## 0.8.3 (2026-06-17)
- Fixed possible index corruption with HNSW vacuuming
- Fixed performance regression with Hamming distance and Jaccard distance with Postgres 18
## 0.8.2 (2026-02-25) ## 0.8.2 (2026-02-25)
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959) - Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)

View File

@@ -5,7 +5,7 @@ ARG DEBIAN_CODENAME=bookworm
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
ARG PG_MAJOR ARG PG_MAJOR
ADD https://github.com/pgvector/pgvector.git#v0.8.4 /tmp/pgvector ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
RUN apt-get update && \ RUN apt-get update && \
apt-mark hold locales && \ apt-mark hold locales && \

View File

@@ -2,12 +2,12 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.8.4", "version": "0.8.2",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
"license": { "license": {
"PostgreSQL": "https://www.postgresql.org/about/licence" "PostgreSQL": "http://www.postgresql.org/about/licence"
}, },
"prereqs": { "prereqs": {
"runtime": { "runtime": {
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.8.4", "version": "0.8.2",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },
@@ -38,7 +38,7 @@
"generated_by": "Andrew Kane", "generated_by": "Andrew Kane",
"meta-spec": { "meta-spec": {
"version": "1.0.0", "version": "1.0.0",
"url": "https://pgxn.org/meta/spec.txt" "url": "http://pgxn.org/meta/spec.txt"
}, },
"tags": [ "tags": [
"vectors", "vectors",

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.4 EXTVERSION = 0.8.2
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*--*.sql) DATA = $(wildcard sql/*--*--*.sql)

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.8.4 EXTVERSION = 0.8.2
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj

View File

@@ -23,7 +23,7 @@ Compile and install the extension (supports Postgres 13+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -40,7 +40,7 @@ Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/buil
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\18" set "PGROOT=C:\Program Files\PostgreSQL\18"
cd %TEMP% cd %TEMP%
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
@@ -1151,23 +1151,23 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
Supported tags are: Supported tags are:
- `pg18-trixie`, `0.8.4-pg18-trixie` - `pg18-trixie`, `0.8.2-pg18-trixie`
- `pg18-bookworm`, `0.8.4-pg18-bookworm`, `pg18`, `0.8.4-pg18` - `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
- `pg17-trixie`, `0.8.4-pg17-trixie` - `pg17-trixie`, `0.8.2-pg17-trixie`
- `pg17-bookworm`, `0.8.4-pg17-bookworm`, `pg17`, `0.8.4-pg17` - `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
- `pg16-trixie`, `0.8.4-pg16-trixie` - `pg16-trixie`, `0.8.2-pg16-trixie`
- `pg16-bookworm`, `0.8.4-pg16-bookworm`, `pg16`, `0.8.4-pg16` - `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
- `pg15-trixie`, `0.8.4-pg15-trixie` - `pg15-trixie`, `0.8.2-pg15-trixie`
- `pg15-bookworm`, `0.8.4-pg15-bookworm`, `pg15`, `0.8.4-pg15` - `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
- `pg14-trixie`, `0.8.4-pg14-trixie` - `pg14-trixie`, `0.8.2-pg14-trixie`
- `pg14-bookworm`, `0.8.4-pg14-bookworm`, `pg14`, `0.8.4-pg14` - `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
- `pg13-trixie`, `0.8.4-pg13-trixie` - `pg13-trixie`, `0.8.2-pg13-trixie`
- `pg13-bookworm`, `0.8.4-pg13-bookworm`, `pg13`, `0.8.4-pg13` - `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.8.4 https://github.com/pgvector/pgvector.git git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector . docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
``` ```
@@ -1329,7 +1329,7 @@ make clean && PG_CFLAGS="-DUSE_ASSERT_CHECKING" make && make install
To enable benchmarking: To enable benchmarking:
```sh ```sh
make clean && PG_CFLAGS="-DHNSW_BENCH -DIVFFLAT_BENCH" make && make install make clean && PG_CFLAGS="-DIVFFLAT_BENCH" make && make install
``` ```
To show memory usage: To show memory usage:

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.4'" to load this file. \quit

View File

@@ -31,12 +31,10 @@
#define BIT_TARGET_CLONES #define BIT_TARGET_CLONES
#endif #endif
/* Use built-ins when possible for Postgres < 19 for inlining */ /* Use built-ins when possible for inlining */
#if PG_VERSION_NUM >= 190000 #if defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_INT_64)
#define popcount64(x) pg_popcount64(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_INT_64) || SIZEOF_LONG == 8)
#define popcount64(x) __builtin_popcountl(x) #define popcount64(x) __builtin_popcountl(x)
#elif defined(HAVE__BUILTIN_POPCOUNT) && (defined(HAVE_LONG_LONG_INT_64) || SIZEOF_LONG_LONG == 8) #elif defined(HAVE__BUILTIN_POPCOUNT) && defined(HAVE_LONG_LONG_INT_64)
#define popcount64(x) __builtin_popcountll(x) #define popcount64(x) __builtin_popcountll(x)
#elif !defined(_MSC_VER) #elif !defined(_MSC_VER)
/* Fails to resolve with MSVC */ /* Fails to resolve with MSVC */

View File

@@ -258,6 +258,11 @@ hnswvalidate(Oid opclassoid)
return true; return true;
} }
static void
hnswunguardbatch(IndexScanDesc scan, IndexScanBatch batch)
{
}
/* /*
* Define index handler * Define index handler
* *
@@ -279,6 +284,7 @@ hnswhandler(PG_FUNCTION_ARGS)
.amconsistentequality = false, .amconsistentequality = false,
.amconsistentordering = false, .amconsistentordering = false,
.amcanbackward = false, .amcanbackward = false,
.amcanmarkpos = false,
.amcanunique = false, .amcanunique = false,
.amcanmulticol = false, .amcanmulticol = false,
.amoptionalkey = true, .amoptionalkey = true,
@@ -290,11 +296,7 @@ hnswhandler(PG_FUNCTION_ARGS)
.amcanparallel = false, .amcanparallel = false,
.amcanbuildparallel = true, .amcanbuildparallel = true,
.amcaninclude = false, .amcaninclude = false,
#if PG_VERSION_NUM >= 170000
.amusemaintenanceworkmem = true,
#else
.amusemaintenanceworkmem = false, .amusemaintenanceworkmem = false,
#endif
.amsummarizing = false, .amsummarizing = false,
.amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL, .amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL,
.amkeytype = InvalidOid, .amkeytype = InvalidOid,
@@ -315,11 +317,14 @@ hnswhandler(PG_FUNCTION_ARGS)
.amadjustmembers = NULL, .amadjustmembers = NULL,
.ambeginscan = hnswbeginscan, .ambeginscan = hnswbeginscan,
.amrescan = hnswrescan, .amrescan = hnswrescan,
.amgettuple = hnswgettuple, .amgettuple = NULL,
.amgetbatch = hnswgetbatch,
.amunguardbatch = hnswunguardbatch,
.amkillitemsbatch = NULL,
.amgettransform = NULL,
.amgetbitmap = NULL, .amgetbitmap = NULL,
.amendscan = hnswendscan, .amendscan = hnswendscan,
.ammarkpos = NULL, .amposreset = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL, .amestimateparallelscan = NULL,
.aminitparallelscan = NULL, .aminitparallelscan = NULL,
.amparallelrescan = NULL, .amparallelrescan = NULL,
@@ -355,11 +360,7 @@ hnswhandler(PG_FUNCTION_ARGS)
amroutine->amcanbuildparallel = true; amroutine->amcanbuildparallel = true;
#endif #endif
amroutine->amcaninclude = false; amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 170000 amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
amroutine->amusemaintenanceworkmem = true;
#else
amroutine->amusemaintenanceworkmem = false;
#endif
#if PG_VERSION_NUM >= 160000 #if PG_VERSION_NUM >= 160000
amroutine->amsummarizing = false; amroutine->amsummarizing = false;
#endif #endif

View File

@@ -18,20 +18,10 @@
#include "utils/sampling.h" #include "utils/sampling.h"
#include "vector.h" #include "vector.h"
#ifdef HNSW_BENCH
#include "portability/instr_time.h"
#endif
#if PG_VERSION_NUM >= 190000 #if PG_VERSION_NUM >= 190000
typedef Pointer Item; typedef Pointer Item;
#endif #endif
#if PG_VERSION_NUM >= 170000
#define HnswTidStore TidStore
#else
#define HnswTidStore tidhash_hash
#endif
#define HNSW_MAX_DIM 2000 #define HNSW_MAX_DIM 2000
#define HNSW_MAX_NNZ 1000 #define HNSW_MAX_NNZ 1000
@@ -88,21 +78,6 @@ typedef Pointer Item;
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page)) #define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
#define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page)) #define HnswPageGetMeta(page) ((HnswMetaPageData *) PageGetContents(page))
#ifdef HNSW_BENCH
#define HnswBench(name, code) \
do { \
instr_time start; \
instr_time duration; \
INSTR_TIME_SET_CURRENT(start); \
(code); \
INSTR_TIME_SET_CURRENT(duration); \
INSTR_TIME_SUBTRACT(duration, start); \
elog(INFO, "%s: %.3f ms", name, INSTR_TIME_GET_MILLISEC(duration)); \
} while (0)
#else
#define HnswBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000 #if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state) #define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed) #define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed)
@@ -433,11 +408,10 @@ typedef struct HnswVacuumState
HnswSupport support; HnswSupport support;
/* Variables */ /* Variables */
struct HnswTidStore *deleting; struct tidhash_hash *deleted;
BufferAccessStrategy bas; BufferAccessStrategy bas;
HnswNeighborTuple ntup; HnswNeighborTuple ntup;
HnswElementData highestPoint; HnswElementData highestPoint;
HnswElementData fallbackPoint;
/* Memory */ /* Memory */
MemoryContext tmpCtx; MemoryContext tmpCtx;
@@ -491,7 +465,7 @@ IndexBulkDeleteResult *hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResu
IndexBulkDeleteResult *hnswvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats); IndexBulkDeleteResult *hnswvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats);
IndexScanDesc hnswbeginscan(Relation index, int nkeys, int norderbys); IndexScanDesc hnswbeginscan(Relation index, int nkeys, int norderbys);
void hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys); void hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys);
bool hnswgettuple(IndexScanDesc scan, ScanDirection dir); IndexScanBatch hnswgetbatch(IndexScanDesc scan, IndexScanBatch priorbatch, ScanDirection dir);
void hnswendscan(IndexScanDesc scan); void hnswendscan(IndexScanDesc scan);
static inline HnswNeighborArray * static inline HnswNeighborArray *

View File

@@ -719,7 +719,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */ /* Get support functions */
HnswInitSupport(&buildstate->support, index); HnswInitSupport(&buildstate->support, index);
InitGraph(&buildstate->graphData, NULL, maintenance_work_mem * (Size) 1024); InitGraph(&buildstate->graphData, NULL, (Size) maintenance_work_mem * 1024L);
buildstate->graph = &buildstate->graphData; buildstate->graph = &buildstate->graphData;
buildstate->ml = HnswGetMl(buildstate->m); buildstate->ml = HnswGetMl(buildstate->m);
buildstate->maxLevel = HnswGetMaxLevel(buildstate->m); buildstate->maxLevel = HnswGetMaxLevel(buildstate->m);
@@ -956,7 +956,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Leave space for other objects in shared memory */ /* Leave space for other objects in shared memory */
/* Docker has a default limit of 64 MB for shm_size */ /* Docker has a default limit of 64 MB for shm_size */
/* which happens to be the default value of maintenance_work_mem */ /* which happens to be the default value of maintenance_work_mem */
esthnswarea = maintenance_work_mem * (Size) 1024; esthnswarea = maintenance_work_mem * 1024L;
estother = 3 * 1024 * 1024; estother = 3 * 1024 * 1024;
if (esthnswarea > estother) if (esthnswarea > estother)
esthnswarea -= estother; esthnswarea -= estother;

View File

@@ -3,6 +3,7 @@
#include <limits.h> #include <limits.h>
#include "access/genam.h" #include "access/genam.h"
#include "access/indexbatch.h"
#include "access/relscan.h" #include "access/relscan.h"
#include "hnsw.h" #include "hnsw.h"
#include "lib/pairingheap.h" #include "lib/pairingheap.h"
@@ -135,6 +136,11 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
double maxMemory; double maxMemory;
scan = RelationGetIndexScan(index, nkeys, norderbys); scan = RelationGetIndexScan(index, nkeys, norderbys);
scan->maxitemsbatch = hnsw_ef_search * HNSW_HEAPTIDS;
/* unused but must be > 0 */
scan->batch_index_opaque_static = MAXALIGN(1);
scan->batch_index_opaque_dyn = 0;
scan->batch_tuples_workspace = 0;
so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData)); so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData));
so->typeInfo = HnswGetTypeInfo(index); so->typeInfo = HnswGetTypeInfo(index);
@@ -184,12 +190,13 @@ hnswrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int no
} }
/* /*
* Fetch the next tuple in the given scan * Fetch the next batch in the given scan
*/ */
bool IndexScanBatch
hnswgettuple(IndexScanDesc scan, ScanDirection dir) hnswgetbatch(IndexScanDesc scan, IndexScanBatch priorbatch, ScanDirection dir)
{ {
HnswScanOpaque so = (HnswScanOpaque) scan->opaque; HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
IndexScanBatch batch = indexam_util_alloc_batch(scan);
MemoryContext oldCtx = MemoryContextSwitchTo(so->tmpCtx); MemoryContext oldCtx = MemoryContextSwitchTo(so->tmpCtx);
/* /*
@@ -245,6 +252,7 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
HnswSearchCandidate *sc; HnswSearchCandidate *sc;
HnswElement element; HnswElement element;
ItemPointer heaptid; ItemPointer heaptid;
int nitems = 0;
if (list_length(so->w) == 0) if (list_length(so->w) == 0)
{ {
@@ -290,44 +298,64 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
break; break;
} }
sc = llast(so->w); while (list_length(so->w) != 0)
element = HnswPtrAccess(base, sc->element);
/* Move to next element if no valid heap TIDs */
if (element->heaptidsLength == 0)
{ {
so->w = list_delete_last(so->w); sc = llast(so->w);
element = HnswPtrAccess(base, sc->element);
/* Mark memory as free for next iteration */ /* Move to next element if no valid heap TIDs */
if (hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF) if (element->heaptidsLength == 0)
{ {
pfree(element); so->w = list_delete_last(so->w);
pfree(sc);
/* Mark memory as free for next iteration */
if (hnsw_iterative_scan != HNSW_ITERATIVE_SCAN_OFF)
{
pfree(element);
pfree(sc);
}
continue;
} }
heaptid = &element->heaptids[--element->heaptidsLength];
if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_STRICT)
{
if (sc->distance < so->previousDistance)
continue;
so->previousDistance = sc->distance;
}
batch->items[nitems].tableTid = *heaptid;
batch->items[nitems].indexOffset = -1;
batch->items[nitems].tupleOffset = 0;
nitems++;
/* Keep batch size flexible */
if (nitems == scan->maxitemsbatch)
break;
}
/* Needed for strict iterative scans */
if (nitems == 0)
continue; continue;
}
heaptid = &element->heaptids[--element->heaptidsLength];
if (hnsw_iterative_scan == HNSW_ITERATIVE_SCAN_STRICT)
{
if (sc->distance < so->previousDistance)
continue;
so->previousDistance = sc->distance;
}
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);
scan->xs_heaptid = *heaptid;
scan->xs_recheck = false; scan->xs_recheck = false;
scan->xs_recheckorderby = false; scan->xs_recheckorderby = false;
return true;
batch->firstItem = 0;
batch->lastItem = nitems - 1;
batch->dir = ForwardScanDirection;
return batch;
} }
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);
return false; indexam_util_release_batch(scan, batch);
return NULL;
} }
/* /*

View File

@@ -546,9 +546,6 @@ HnswLoadElementImpl(BlockNumber blkno, OffsetNumber offno, double *distance, Hns
Assert(HnswIsElementTuple(etup)); Assert(HnswIsElementTuple(etup));
if (unlikely(etup->deleted))
elog(ERROR, "cannot load deleted element");
/* Calculate distance */ /* Calculate distance */
if (distance != NULL) if (distance != NULL)
{ {

View File

@@ -10,12 +10,6 @@
#include "utils/memutils.h" #include "utils/memutils.h"
#include "utils/rel.h" #include "utils/rel.h"
#if PG_VERSION_NUM >= 170000
#include "access/tidstore.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#endif
#if PG_VERSION_NUM >= 160000 #if PG_VERSION_NUM >= 160000
#include "varatt.h" #include "varatt.h"
#endif #endif
@@ -25,16 +19,12 @@
#endif #endif
/* /*
* Check if deletion list contains an element * Check if deleted list contains an index TID
*/ */
static bool static bool
DeletingElement(HnswTidStore * deleting, ItemPointer indextid) DeletedContains(tidhash_hash * deleted, ItemPointer indextid)
{ {
#if PG_VERSION_NUM >= 170000 return tidhash_lookup(deleted, *indextid) != NULL;
return TidStoreIsMember(deleting, indextid);
#else
return tidhash_lookup(deleting, *indextid) != NULL;
#endif
} }
/* /*
@@ -47,20 +37,17 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
{ {
BlockNumber blkno = HNSW_HEAD_BLKNO; BlockNumber blkno = HNSW_HEAD_BLKNO;
HnswElement highestPoint = &vacuumstate->highestPoint; HnswElement highestPoint = &vacuumstate->highestPoint;
HnswElement fallbackPoint = &vacuumstate->fallbackPoint;
Relation index = vacuumstate->index; Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
HnswElement entryPoint = HnswGetEntryPoint(vacuumstate->index);
IndexBulkDeleteResult *stats = vacuumstate->stats; IndexBulkDeleteResult *stats = vacuumstate->stats;
/* Store separately since HnswElement level is uint8 */ /* Store separately since highestPoint.level is uint8 */
int highestLevel = -1; int highestLevel = -1;
int fallbackLevel = -1;
/* Initialize highest point and fallback point */ /* Initialize highest point */
highestPoint->blkno = InvalidBlockNumber; highestPoint->blkno = InvalidBlockNumber;
highestPoint->offno = InvalidOffsetNumber; highestPoint->offno = InvalidOffsetNumber;
fallbackPoint->blkno = InvalidBlockNumber;
fallbackPoint->offno = InvalidOffsetNumber;
while (BlockNumberIsValid(blkno)) while (BlockNumberIsValid(blkno))
{ {
@@ -70,10 +57,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
OffsetNumber offno; OffsetNumber offno;
OffsetNumber maxoffno; OffsetNumber maxoffno;
bool updated = false; bool updated = false;
#if PG_VERSION_NUM >= 170000
OffsetNumber deletedoffs[MaxOffsetNumber];
int ndeletedoffs = 0;
#endif
vacuum_delay_point(); vacuum_delay_point();
@@ -94,14 +77,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup)) if (!HnswIsElementTuple(etup))
continue; continue;
/*
* Skip deleted tuples. It is important they are not added to the
* deletion list to avoid false positives in NeedsUpdated and
* ConfirmRepaired.
*/
if (etup->deleted)
continue;
if (ItemPointerIsValid(&etup->heaptids[0])) if (ItemPointerIsValid(&etup->heaptids[0]))
{ {
for (int i = 0; i < HNSW_HEAPTIDS; i++) for (int i = 0; i < HNSW_HEAPTIDS; i++)
@@ -135,50 +110,25 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (!ItemPointerIsValid(&etup->heaptids[0])) if (!ItemPointerIsValid(&etup->heaptids[0]))
{ {
#if PG_VERSION_NUM >= 170000 ItemPointerData ip;
deletedoffs[ndeletedoffs++] = offno;
#else
ItemPointerData indextid;
bool found; bool found;
/* Add to deletion list */ /* Add to deleted list */
ItemPointerSet(&indextid, blkno, offno); ItemPointerSet(&ip, blkno, offno);
tidhash_insert(vacuumstate->deleting, indextid, &found); tidhash_insert(vacuumstate->deleted, ip, &found);
Assert(!found); Assert(!found);
#endif
} }
else if (etup->level > highestLevel) else if (etup->level > highestLevel && !(entryPoint != NULL && blkno == entryPoint->blkno && offno == entryPoint->offno))
{ {
if (BlockNumberIsValid(highestPoint->blkno)) /* Keep track of highest non-entry point */
{
/* Current highest point becomes fallback */
fallbackPoint->blkno = highestPoint->blkno;
fallbackPoint->offno = highestPoint->offno;
fallbackPoint->level = highestPoint->level;
fallbackLevel = highestLevel;
}
/* Keep track of highest point */
highestPoint->blkno = blkno; highestPoint->blkno = blkno;
highestPoint->offno = offno; highestPoint->offno = offno;
highestPoint->level = etup->level; highestPoint->level = etup->level;
highestLevel = etup->level; highestLevel = etup->level;
} }
else if (etup->level > fallbackLevel)
{
/* Keep track of second highest point */
fallbackPoint->blkno = blkno;
fallbackPoint->offno = offno;
fallbackPoint->level = etup->level;
fallbackLevel = etup->level;
}
} }
#if PG_VERSION_NUM >= 170000
TidStoreSetBlockOffsets(vacuumstate->deleting, blkno, deletedoffs, ndeletedoffs);
#endif
blkno = HnswPageGetOpaque(page)->nextblkno; blkno = HnswPageGetOpaque(page)->nextblkno;
if (updated) if (updated)
@@ -188,10 +138,6 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
#ifdef HNSW_MEMORY
elog(INFO, "memory: %zu KB", MemoryContextMemAllocated(CurrentMemoryContext, true) / 1024);
#endif
} }
/* /*
@@ -222,8 +168,8 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
if (!ItemPointerIsValid(indextid)) if (!ItemPointerIsValid(indextid))
continue; continue;
/* Check if in deletion list */ /* Check if in deleted list */
if (DeletingElement(vacuumstate->deleting, indextid)) if (DeletedContains(vacuumstate->deleted, indextid))
{ {
needsUpdated = true; needsUpdated = true;
break; break;
@@ -232,9 +178,13 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element)
/* Also update if layer 0 is not full */ /* Also update if layer 0 is not full */
/* This could indicate too many candidates being deleted during insert */ /* This could indicate too many candidates being deleted during insert */
/* There should always be more than zero indextids, but check for safety */ if (!needsUpdated)
if (!needsUpdated && ntup->count > 0) {
/* Keep clang-tidy happy */
Assert(ntup->count > 0);
needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]); needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]);
}
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
@@ -319,27 +269,12 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
/* Get a shared lock */ /* Get a shared lock */
LockPage(index, HNSW_UPDATE_LOCK, ShareLock); LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Get latest entry point */ /* Load element */
entryPoint = HnswGetEntryPoint(index); HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Use fallback point if highest point is entry point */ /* Repair if needed */
if (entryPoint != NULL && entryPoint->blkno == highestPoint->blkno && entryPoint->offno == highestPoint->offno) if (NeedsUpdated(vacuumstate, highestPoint))
{ RepairGraphElement(vacuumstate, highestPoint, HnswGetEntryPoint(index));
highestPoint = &vacuumstate->fallbackPoint;
if (!BlockNumberIsValid(highestPoint->blkno))
highestPoint = NULL;
}
if (highestPoint != NULL)
{
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, support, true, NULL);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
RepairGraphElement(vacuumstate, highestPoint, entryPoint);
}
/* Release lock */ /* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock); UnlockPage(index, HNSW_UPDATE_LOCK, ShareLock);
@@ -357,7 +292,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno); ItemPointerSet(&epData, entryPoint->blkno, entryPoint->offno);
if (DeletingElement(vacuumstate->deleting, &epData)) if (DeletedContains(vacuumstate->deleted, &epData))
{ {
/* /*
* Replace the entry point with the highest point. If highest * Replace the entry point with the highest point. If highest
@@ -443,10 +378,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
if (!HnswIsElementTuple(etup)) if (!HnswIsElementTuple(etup))
continue; continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip updating neighbors if being deleted */ /* Skip updating neighbors if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0])) if (!ItemPointerIsValid(&etup->heaptids[0]))
continue; continue;
@@ -510,103 +441,6 @@ RepairGraph(HnswVacuumState * vacuumstate)
/* Reset memory context */ /* Reset memory context */
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);
MemoryContextReset(vacuumstate->tmpCtx); MemoryContextReset(vacuumstate->tmpCtx);
#ifdef HNSW_VACUUM_PROGRESS
if (!BlockNumberIsValid(blkno) || (blkno - HNSW_HEAD_BLKNO) % 1000 == 0)
{
BlockNumber totalBlocks = RelationGetNumberOfBlocks(index);
BlockNumber currentBlocks = BlockNumberIsValid(blkno) ? blkno : totalBlocks;
elog(INFO, "hnsw vacuum progress: %.1f%%", 100.0 * currentBlocks / totalBlocks);
}
#endif
}
}
/*
* Confirm graph was repaired
*/
static void
ConfirmRepaired(HnswVacuumState * vacuumstate)
{
BlockNumber blkno = HNSW_HEAD_BLKNO;
Relation index = vacuumstate->index;
BufferAccessStrategy bas = vacuumstate->bas;
while (BlockNumberIsValid(blkno))
{
Buffer buf;
Page page;
OffsetNumber offno;
OffsetNumber maxoffno;
vacuum_delay_point();
buf = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bas);
LockBuffer(buf, BUFFER_LOCK_SHARE);
page = BufferGetPage(buf);
maxoffno = PageGetMaxOffsetNumber(page);
/* Iterate over nodes */
for (offno = FirstOffsetNumber; offno <= maxoffno; offno = OffsetNumberNext(offno))
{
HnswElementTuple etup = (HnswElementTuple) PageGetItem(page, PageGetItemId(page, offno));
HnswNeighborTuple ntup;
Buffer nbuf;
Page npage;
BlockNumber neighborPage;
OffsetNumber neighborOffno;
/* Skip neighbor tuples */
if (!HnswIsElementTuple(etup))
continue;
/* Skip deleted tuples */
if (etup->deleted)
continue;
/* Skip if being deleted */
if (!ItemPointerIsValid(&etup->heaptids[0]))
continue;
/* Get neighbor page */
neighborPage = ItemPointerGetBlockNumber(&etup->neighbortid);
neighborOffno = ItemPointerGetOffsetNumber(&etup->neighbortid);
if (neighborPage == blkno)
{
nbuf = buf;
npage = page;
}
else
{
nbuf = ReadBufferExtended(index, MAIN_FORKNUM, neighborPage, RBM_NORMAL, bas);
LockBuffer(nbuf, BUFFER_LOCK_SHARE);
npage = BufferGetPage(nbuf);
}
ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno));
/* Check neighbors */
for (int i = 0; i < ntup->count; i++)
{
ItemPointer indextid = &ntup->indextids[i];
if (!ItemPointerIsValid(indextid))
continue;
/* Check if in deletion list */
if (DeletingElement(vacuumstate->deleting, indextid))
elog(ERROR, "hnsw graph not repaired");
}
if (nbuf != buf)
UnlockReleaseBuffer(nbuf);
}
blkno = HnswPageGetOpaque(page)->nextblkno;
UnlockReleaseBuffer(buf);
} }
} }
@@ -622,15 +456,10 @@ MarkDeleted(HnswVacuumState * vacuumstate)
BufferAccessStrategy bas = vacuumstate->bas; BufferAccessStrategy bas = vacuumstate->bas;
/* /*
* Wait for inserts and index scans to complete. Inserts and scans before * Wait for index scans to complete. Scans before this point may contain
* this point may visit tuples about to be deleted. Inserts and scans * tuples about to be deleted. Scans after this point will not, since the
* after this point will not, since the graph has been repaired. * graph has been repaired.
*/ */
LockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_UPDATE_LOCK, ExclusiveLock);
ConfirmRepaired(vacuumstate);
LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock); LockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock); UnlockPage(index, HNSW_SCAN_LOCK, ExclusiveLock);
@@ -778,11 +607,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
HnswGetMetaPageInfo(index, &vacuumstate->m, NULL); HnswGetMetaPageInfo(index, &vacuumstate->m, NULL);
/* Create hash table */ /* Create hash table */
#if PG_VERSION_NUM >= 170000 vacuumstate->deleted = tidhash_create(CurrentMemoryContext, 256, NULL);
vacuumstate->deleting = TidStoreCreateLocal((AmAutoVacuumWorkerProcess() && autovacuum_work_mem != -1) ? autovacuum_work_mem : maintenance_work_mem, true);
#else
vacuumstate->deleting = tidhash_create(CurrentMemoryContext, 256, NULL);
#endif
} }
/* /*
@@ -791,11 +616,7 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
static void static void
FreeVacuumState(HnswVacuumState * vacuumstate) FreeVacuumState(HnswVacuumState * vacuumstate)
{ {
#if PG_VERSION_NUM >= 170000 tidhash_destroy(vacuumstate->deleted);
TidStoreDestroy(vacuumstate->deleting);
#else
tidhash_destroy(vacuumstate->deleting);
#endif
FreeAccessStrategy(vacuumstate->bas); FreeAccessStrategy(vacuumstate->bas);
pfree(vacuumstate->ntup); pfree(vacuumstate->ntup);
MemoryContextDelete(vacuumstate->tmpCtx); MemoryContextDelete(vacuumstate->tmpCtx);
@@ -813,13 +634,13 @@ hnswbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
InitVacuumState(&vacuumstate, info, stats, callback, callback_state); InitVacuumState(&vacuumstate, info, stats, callback, callback_state);
/* Pass 1: Remove heap TIDs */ /* Pass 1: Remove heap TIDs */
HnswBench("RemoveHeapTids", RemoveHeapTids(&vacuumstate)); RemoveHeapTids(&vacuumstate);
/* Pass 2: Repair graph */ /* Pass 2: Repair graph */
HnswBench("RepairGraph", RepairGraph(&vacuumstate)); RepairGraph(&vacuumstate);
/* Passes 3 and 4: Confirm repaired and mark as deleted */ /* Pass 3: Mark as deleted */
HnswBench("MarkDeleted", MarkDeleted(&vacuumstate)); MarkDeleted(&vacuumstate);
FreeVacuumState(&vacuumstate); FreeVacuumState(&vacuumstate);

View File

@@ -152,7 +152,18 @@ SampleRows(IvfflatBuildState * buildstate)
/* Normalize if needed */ /* Normalize if needed */
if (buildstate->kmeansnormprocinfo != NULL) if (buildstate->kmeansnormprocinfo != NULL)
IvfflatNormVectors(buildstate->typeInfo, buildstate->collation, buildstate->samples, buildstate->tmpCtx); {
VectorArray samples = buildstate->samples;
for (int i = 0; i < samples->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(samples, i));
Datum normValue = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
VectorArraySet(samples, i, DatumGetPointer(normValue));
pfree(DatumGetPointer(normValue));
}
}
} }
/* /*
@@ -388,14 +399,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual); buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
buildstate->memoryUsed = 0; buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->typeInfo->itemSize(buildstate->dimensions));
buildstate->itemsize = buildstate->typeInfo->itemSize(buildstate->dimensions);
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(buildstate->lists, buildstate->itemsize);
IvfflatCheckMemoryUsage(buildstate->memoryUsed);
buildstate->centers = VectorArrayInit(buildstate->lists, buildstate->dimensions, buildstate->itemsize);
/* TODO Move allocation to page creation */
buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists); buildstate->listInfo = palloc(sizeof(ListInfo) * buildstate->lists);
buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext, buildstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
@@ -438,27 +442,19 @@ ComputeCenters(IvfflatBuildState * buildstate)
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS); pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_KMEANS);
/* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50;
if (numSamples < 10000)
numSamples = 10000;
/* Skip samples for unlogged table */ /* Skip samples for unlogged table */
if (buildstate->heap == NULL) if (buildstate->heap == NULL)
numSamples = 1; numSamples = 1;
else
{
int64 maxTuples = (int64) RelationGetNumberOfBlocks(buildstate->heap) * MaxHeapTuplesPerPage;
/* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50;
if (numSamples < 10000)
numSamples = 10000;
/* Save memory since will not have more than max tuples */
numSamples = Max(Min(numSamples, maxTuples), 1);
}
/* Sample rows */ /* Sample rows */
buildstate->memoryUsed += VECTOR_ARRAY_SIZE(numSamples, buildstate->itemsize); /* TODO Ensure within maintenance_work_mem */
IvfflatCheckMemoryUsage(buildstate->memoryUsed); buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->itemsize);
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
{ {
IvfflatBench("sample rows", SampleRows(buildstate)); IvfflatBench("sample rows", SampleRows(buildstate));
@@ -473,7 +469,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
} }
/* Calculate centers */ /* Calculate centers */
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo, buildstate->memoryUsed)); IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers, buildstate->typeInfo));
/* Free samples before we allocate more memory */ /* Free samples before we allocate more memory */
VectorArrayFree(buildstate->samples); VectorArrayFree(buildstate->samples);

View File

@@ -175,6 +175,11 @@ ivfflatvalidate(Oid opclassoid)
return true; return true;
} }
static void
ivfflatunguardbatch(IndexScanDesc scan, IndexScanBatch batch)
{
}
/* /*
* Define index handler * Define index handler
* *
@@ -196,6 +201,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
.amconsistentequality = false, .amconsistentequality = false,
.amconsistentordering = false, .amconsistentordering = false,
.amcanbackward = false, .amcanbackward = false,
.amcanmarkpos = false,
.amcanunique = false, .amcanunique = false,
.amcanmulticol = false, .amcanmulticol = false,
.amoptionalkey = true, .amoptionalkey = true,
@@ -228,11 +234,14 @@ ivfflathandler(PG_FUNCTION_ARGS)
.amadjustmembers = NULL, .amadjustmembers = NULL,
.ambeginscan = ivfflatbeginscan, .ambeginscan = ivfflatbeginscan,
.amrescan = ivfflatrescan, .amrescan = ivfflatrescan,
.amgettuple = ivfflatgettuple, .amgettuple = NULL,
.amgetbatch = ivfflatgetbatch,
.amunguardbatch = ivfflatunguardbatch,
.amkillitemsbatch = NULL,
.amgettransform = NULL,
.amgetbitmap = NULL, .amgetbitmap = NULL,
.amendscan = ivfflatendscan, .amendscan = ivfflatendscan,
.ammarkpos = NULL, .amposreset = NULL,
.amrestrpos = NULL,
.amestimateparallelscan = NULL, .amestimateparallelscan = NULL,
.aminitparallelscan = NULL, .aminitparallelscan = NULL,
.amparallelrescan = NULL, .amparallelrescan = NULL,

View File

@@ -204,7 +204,6 @@ typedef struct IvfflatBuildState
VectorArray samples; VectorArray samples;
VectorArray centers; VectorArray centers;
ListInfo *listInfo; ListInfo *listInfo;
Size itemsize;
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
double inertia; double inertia;
@@ -224,7 +223,6 @@ typedef struct IvfflatBuildState
TupleTableSlot *slot; TupleTableSlot *slot;
/* Memory */ /* Memory */
Size memoryUsed;
MemoryContext tmpCtx; MemoryContext tmpCtx;
/* Parallel builds */ /* Parallel builds */
@@ -305,32 +303,22 @@ typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
static inline Pointer static inline Pointer
VectorArrayGet(VectorArray arr, int offset) VectorArrayGet(VectorArray arr, int offset)
{ {
if (offset >= arr->maxlen)
elog(ERROR, "safety check failed");
return ((char *) arr->items) + (offset * arr->itemsize); return ((char *) arr->items) + (offset * arr->itemsize);
} }
static inline void static inline void
VectorArraySet(VectorArray arr, int offset, Pointer val) VectorArraySet(VectorArray arr, int offset, Pointer val)
{ {
Size size = VARSIZE_ANY(val); memcpy(VectorArrayGet(arr, offset), val, VARSIZE_ANY(val));
if (size > arr->itemsize)
elog(ERROR, "safety check failed");
memcpy(VectorArrayGet(arr, offset), val, size);
} }
/* Methods */ /* Methods */
VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize); VectorArray VectorArrayInit(int maxlen, int dimensions, Size itemsize);
void VectorArrayFree(VectorArray arr); void VectorArrayFree(VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed); void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo);
FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum); FmgrInfo *IvfflatOptionalProcInfo(Relation index, uint16 procnum);
Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value); Datum IvfflatNormValue(const IvfflatTypeInfo * typeInfo, Oid collation, Datum value);
bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value); bool IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value);
void IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx);
void IvfflatCheckMemoryUsage(Size totalSize);
int IvfflatGetLists(Relation index); int IvfflatGetLists(Relation index);
void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions); void IvfflatGetMetaPageInfo(Relation index, int *lists, int *dimensions);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum); void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
@@ -356,7 +344,7 @@ IndexBulkDeleteResult *ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteR
IndexBulkDeleteResult *ivfflatvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats); IndexBulkDeleteResult *ivfflatvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats);
IndexScanDesc ivfflatbeginscan(Relation index, int nkeys, int norderbys); IndexScanDesc ivfflatbeginscan(Relation index, int nkeys, int norderbys);
void ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys); void ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys);
bool ivfflatgettuple(IndexScanDesc scan, ScanDirection dir); IndexScanBatch ivfflatgetbatch(IndexScanDesc scan, IndexScanBatch priorbatch, ScanDirection dir);
void ivfflatendscan(IndexScanDesc scan); void ivfflatendscan(IndexScanDesc scan);
#endif #endif

View File

@@ -99,8 +99,22 @@ NormCenters(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray centers
MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext, MemoryContext normCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat norm temporary context", "Ivfflat norm temporary context",
ALLOCSET_DEFAULT_SIZES); ALLOCSET_DEFAULT_SIZES);
MemoryContext oldCtx = MemoryContextSwitchTo(normCtx);
IvfflatNormVectors(typeInfo, collation, centers, normCtx); for (int j = 0; j < centers->length; j++)
{
Datum center = PointerGetDatum(VectorArrayGet(centers, j));
Datum newCenter = IvfflatNormValue(typeInfo, collation, center);
Size size = VARSIZE_ANY(DatumGetPointer(newCenter));
if (size > centers->itemsize)
elog(ERROR, "safety check failed");
memcpy(DatumGetPointer(center), DatumGetPointer(newCenter), size);
MemoryContextReset(normCtx);
}
MemoryContextSwitchTo(oldCtx);
MemoryContextDelete(normCtx); MemoryContextDelete(normCtx);
} }
@@ -244,7 +258,7 @@ ComputeNewCenters(VectorArray samples, float *agg, VectorArray newCenters, int *
* https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf * https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf
*/ */
static void static void
ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed) ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{ {
FmgrInfo *procinfo; FmgrInfo *procinfo;
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
@@ -263,6 +277,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
float *newcdist; float *newcdist;
/* Calculate allocation sizes */ /* Calculate allocation sizes */
Size samplesSize = VECTOR_ARRAY_SIZE(samples->maxlen, samples->itemsize);
Size centersSize = VECTOR_ARRAY_SIZE(centers->maxlen, centers->itemsize);
Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize); Size newCentersSize = VECTOR_ARRAY_SIZE(numCenters, centers->itemsize);
Size aggSize = sizeof(float) * (int64) numCenters * dimensions; Size aggSize = sizeof(float) * (int64) numCenters * dimensions;
Size centerCountsSize = sizeof(int) * numCenters; Size centerCountsSize = sizeof(int) * numCenters;
@@ -274,13 +290,18 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers, const Ivff
Size newcdistSize = sizeof(float) * numCenters; Size newcdistSize = sizeof(float) * numCenters;
/* Calculate total size */ /* Calculate total size */
Size totalSize = memoryUsed + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize; Size totalSize = samplesSize + centersSize + newCentersSize + aggSize + centerCountsSize + closestCentersSize + lowerBoundSize + upperBoundSize + sSize + halfcdistSize + newcdistSize;
/* Check memory requirements */ /* Check memory requirements */
IvfflatCheckMemoryUsage(totalSize); /* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
/* Ensure indexing does not overflow */ /* Ensure indexing does not overflow */
if (numCenters > INT_MAX / numCenters) if (numCenters * numCenters > INT_MAX)
elog(ERROR, "Indexing overflow detected. Please report a bug."); elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */ /* Set support functions */
@@ -541,7 +562,7 @@ CheckCenters(Relation index, VectorArray centers, const IvfflatTypeInfo * typeIn
* We use spherical k-means for inner product and cosine * We use spherical k-means for inner product and cosine
*/ */
void void
IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo, Size memoryUsed) IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const IvfflatTypeInfo * typeInfo)
{ {
MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext, MemoryContext kmeansCtx = AllocSetContextCreate(CurrentMemoryContext,
"Ivfflat kmeans temporary context", "Ivfflat kmeans temporary context",
@@ -551,7 +572,7 @@ IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers, const Iv
if (samples->length == 0) if (samples->length == 0)
RandomCenters(index, centers, typeInfo); RandomCenters(index, centers, typeInfo);
else else
ElkanKmeans(index, samples, centers, typeInfo, memoryUsed); ElkanKmeans(index, samples, centers, typeInfo);
CheckCenters(index, centers, typeInfo); CheckCenters(index, centers, typeInfo);

View File

@@ -3,6 +3,7 @@
#include <float.h> #include <float.h>
#include "access/genam.h" #include "access/genam.h"
#include "access/indexbatch.h"
#include "access/itup.h" #include "access/itup.h"
#include "access/relscan.h" #include "access/relscan.h"
#include "access/tupdesc.h" #include "access/tupdesc.h"
@@ -261,6 +262,11 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
MemoryContext oldCtx; MemoryContext oldCtx;
scan = RelationGetIndexScan(index, nkeys, norderbys); scan = RelationGetIndexScan(index, nkeys, norderbys);
scan->maxitemsbatch = 1000;
/* unused but must be > 0 */
scan->batch_index_opaque_static = MAXALIGN(1);
scan->batch_index_opaque_dyn = 0;
scan->batch_tuples_workspace = 0;
/* Get lists and dimensions from metapage */ /* Get lists and dimensions from metapage */
IvfflatGetMetaPageInfo(index, &lists, &dimensions); IvfflatGetMetaPageInfo(index, &lists, &dimensions);
@@ -348,14 +354,16 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
} }
/* /*
* Fetch the next tuple in the given scan * Fetch the next batch in the given scan
*/ */
bool IndexScanBatch
ivfflatgettuple(IndexScanDesc scan, ScanDirection dir) ivfflatgetbatch(IndexScanDesc scan, IndexScanBatch priorbatch, ScanDirection dir)
{ {
IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque; IvfflatScanOpaque so = (IvfflatScanOpaque) scan->opaque;
IndexScanBatch batch = indexam_util_alloc_batch(scan);
ItemPointer heaptid; ItemPointer heaptid;
bool isnull; bool isnull;
int nitems = 0;
/* /*
* Index can be used to scan backward, but Postgres doesn't support * Index can be used to scan backward, but Postgres doesn't support
@@ -393,17 +401,37 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
while (!tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL)) while (!tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
{ {
if (so->listIndex == so->maxProbes) if (so->listIndex == so->maxProbes)
return false; {
indexam_util_release_batch(scan, batch);
return NULL;
}
IvfflatBench("GetScanItems", GetScanItems(scan, so->value)); IvfflatBench("GetScanItems", GetScanItems(scan, so->value));
} }
heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull)); for (;;)
{
heaptid = (ItemPointer) DatumGetPointer(slot_getattr(so->mslot, 2, &isnull));
batch->items[nitems].tableTid = *heaptid;
batch->items[nitems].indexOffset = -1;
batch->items[nitems].tupleOffset = 0;
nitems++;
if (nitems == scan->maxitemsbatch)
break;
if (!tuplesort_gettupleslot(so->sortstate, true, false, so->mslot, NULL))
break;
}
scan->xs_heaptid = *heaptid;
scan->xs_recheck = false; scan->xs_recheck = false;
scan->xs_recheckorderby = false; scan->xs_recheckorderby = false;
return true;
batch->firstItem = 0;
batch->lastItem = nitems - 1;
batch->dir = ForwardScanDirection;
return batch;
} }
/* /*

View File

@@ -6,9 +6,7 @@
#include "halfutils.h" #include "halfutils.h"
#include "halfvec.h" #include "halfvec.h"
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "utils/memutils.h"
#include "utils/relcache.h" #include "utils/relcache.h"
#include "utils/varbit.h" #include "utils/varbit.h"
#include "vector.h" #include "vector.h"
@@ -90,40 +88,6 @@ IvfflatCheckNorm(FmgrInfo *procinfo, Oid collation, Datum value)
return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0; return DatumGetFloat8(FunctionCall1Coll(procinfo, collation, value)) > 0;
} }
/*
* Normalize vectors
*/
void
IvfflatNormVectors(const IvfflatTypeInfo * typeInfo, Oid collation, VectorArray arr, MemoryContext tmpCtx)
{
MemoryContext oldCtx = MemoryContextSwitchTo(tmpCtx);
for (int i = 0; i < arr->length; i++)
{
Datum value = PointerGetDatum(VectorArrayGet(arr, i));
Datum newValue = IvfflatNormValue(typeInfo, collation, value);
VectorArraySet(arr, i, DatumGetPointer(newValue));
MemoryContextReset(tmpCtx);
}
MemoryContextSwitchTo(oldCtx);
}
/*
* Check memory usage
*/
void
IvfflatCheckMemoryUsage(Size totalSize)
{
/* Add one to error message to ceil */
if (totalSize > maintenance_work_mem * (Size) 1024)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
}
/* /*
* New buffer * New buffer
*/ */

View File

@@ -40,7 +40,7 @@
#endif #endif
#if PG_VERSION_NUM >= 180000 #if PG_VERSION_NUM >= 180000
PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.4"); PG_MODULE_MAGIC_EXT(.name = "vector", .version = "0.8.2");
#else #else
PG_MODULE_MAGIC; PG_MODULE_MAGIC;
#endif #endif

View File

@@ -49,11 +49,3 @@ CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -100,11 +100,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t
(1 row) (1 row)
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for hnsw index
DROP TABLE t;

View File

@@ -161,7 +161,6 @@ ERROR: value 1001 out of bounds for option "ef_construction"
DETAIL: Valid values are between "4" and "1000". DETAIL: Valid values are between "4" and "1000".
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
ERROR: ef_construction must be greater than or equal to 2 * m ERROR: ef_construction must be greater than or equal to 2 * m
DROP TABLE t;
SHOW hnsw.ef_search; SHOW hnsw.ef_search;
hnsw.ef_search hnsw.ef_search
---------------- ----------------
@@ -199,11 +198,4 @@ SET hnsw.scan_mem_multiplier = 0;
ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000) ERROR: 0 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
SET hnsw.scan_mem_multiplier = 1001; SET hnsw.scan_mem_multiplier = 1001;
ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000) ERROR: 1001 is outside the valid range for parameter "hnsw.scan_mem_multiplier" (1 .. 1000)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for hnsw index
DROP TABLE t; DROP TABLE t;

View File

@@ -35,32 +35,3 @@ NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall. DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data. HINT: Drop the index until the table has more data.
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
ERROR: column cannot have more than 64000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -82,32 +82,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t
(1 row) (1 row)
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
ERROR: column cannot have more than 4000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -143,7 +143,6 @@ DETAIL: Valid values are between "1" and "32768".
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
ERROR: value 32769 out of bounds for option "lists" ERROR: value 32769 out of bounds for option "lists"
DETAIL: Valid values are between "1" and "32768". DETAIL: Valid values are between "1" and "32768".
DROP TABLE t;
SHOW ivfflat.probes; SHOW ivfflat.probes;
ivfflat.probes ivfflat.probes
---------------- ----------------
@@ -173,32 +172,4 @@ SET ivfflat.max_probes = 0;
ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768) ERROR: 0 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
SET ivfflat.max_probes = 32769; SET ivfflat.max_probes = 32769;
ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768) ERROR: 32769 is outside the valid range for parameter "ivfflat.max_probes" (1 .. 32768)
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t; DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
ERROR: column cannot have more than 2000 dimensions for ivfflat index
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
NOTICE: ivfflat index created with little data
DETAIL: This will cause low recall.
HINT: Drop the index until the table has more data.
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -33,13 +33,3 @@ CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(3)) bit_hamming_ops);
CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops); CREATE INDEX ON t USING hnsw ((val::bit(64001)) bit_hamming_ops);
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING hnsw (val bit_hamming_ops);
DROP TABLE t;

View File

@@ -56,13 +56,3 @@ SELECT * FROM t ORDER BY val <+> '[3,3,3]';
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <+> (SELECT NULL::halfvec)) t2;
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING hnsw (val halfvec_l2_ops);
DROP TABLE t;

View File

@@ -95,28 +95,23 @@ CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 101);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 3);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (ef_construction = 1001);
CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31); CREATE INDEX ON t USING hnsw (val vector_l2_ops) WITH (m = 16, ef_construction = 31);
DROP TABLE t;
SHOW hnsw.ef_search; SHOW hnsw.ef_search;
SET hnsw.ef_search = 0; SET hnsw.ef_search = 0;
SET hnsw.ef_search = 1001; SET hnsw.ef_search = 1001;
SHOW hnsw.iterative_scan; SHOW hnsw.iterative_scan;
SET hnsw.iterative_scan = on; SET hnsw.iterative_scan = on;
SHOW hnsw.max_scan_tuples; SHOW hnsw.max_scan_tuples;
SET hnsw.max_scan_tuples = 0; SET hnsw.max_scan_tuples = 0;
SHOW hnsw.scan_mem_multiplier; SHOW hnsw.scan_mem_multiplier;
SET hnsw.scan_mem_multiplier = 0; SET hnsw.scan_mem_multiplier = 0;
SET hnsw.scan_mem_multiplier = 1001; SET hnsw.scan_mem_multiplier = 1001;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING hnsw (val vector_l2_ops);
DROP TABLE t; DROP TABLE t;

View File

@@ -21,28 +21,3 @@ CREATE INDEX ON t USING ivfflat ((val::bit(3)) bit_hamming_ops) WITH (lists = 1)
CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1); CREATE INDEX ON t USING ivfflat ((val::bit(64001)) bit_hamming_ops) WITH (lists = 1);
CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5); CREATE INDEX ON t USING ivfflat ((val::bit(2)) bit_hamming_ops) WITH (lists = 5);
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
CREATE TABLE t (val bit(64001));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '29MB';
CREATE TABLE t (val bit(64000));
INSERT INTO t (val) VALUES (B'0'::bit(64000));
CREATE INDEX ON t USING ivfflat (val bit_hamming_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -43,28 +43,3 @@ SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2; SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::halfvec)) t2;
DROP TABLE t; DROP TABLE t;
-- dimensions
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
CREATE TABLE t (val halfvec(4001));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val halfvec(4000));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '6MB';
CREATE TABLE t (val halfvec(4000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[4000]));
CREATE INDEX ON t USING ivfflat (val halfvec_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -81,40 +81,19 @@ DROP TABLE t;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 0);
CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769); CREATE INDEX ON t USING ivfflat (val vector_l2_ops) WITH (lists = 32769);
DROP TABLE t;
SHOW ivfflat.probes; SHOW ivfflat.probes;
SET ivfflat.probes = 0; SET ivfflat.probes = 0;
SET ivfflat.probes = 32769; SET ivfflat.probes = 32769;
SHOW ivfflat.iterative_scan; SHOW ivfflat.iterative_scan;
SET ivfflat.iterative_scan = on; SET ivfflat.iterative_scan = on;
SHOW ivfflat.max_probes; SHOW ivfflat.max_probes;
SET ivfflat.max_probes = 0; SET ivfflat.max_probes = 0;
SET ivfflat.max_probes = 32769; SET ivfflat.max_probes = 32769;
-- dimensions
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t; DROP TABLE t;
CREATE TABLE t (val vector(2001));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
-- memory
SET maintenance_work_mem = '1MB';
CREATE TABLE t (val vector(2000));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;
SET maintenance_work_mem = '5MB';
CREATE TABLE t (val vector(2000));
INSERT INTO t (val) VALUES (array_fill(0, ARRAY[2000]));
CREATE INDEX ON t USING ivfflat (val vector_l2_ops);
DROP TABLE t;
RESET maintenance_work_mem;

View File

@@ -1,38 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
$node->safe_psql("postgres", "DELETE FROM tst");
# Test HNSW_SCAN_LOCK at the beginning of MarkDeleted is effective
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent SELECTs and VACUUM",
{
"046_hnsw_vacuum_scan_select\@1000" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"046_hnsw_vacuum_scan_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,39 +0,0 @@
use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $dim = 3;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i serial, v vector($dim));");
$node->safe_psql("postgres", "ALTER TABLE tst SET (autovacuum_enabled = false);");
$node->safe_psql("postgres",
"INSERT INTO tst (v) SELECT ARRAY[$array_sql] FROM generate_series(1, 1000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v vector_l2_ops);");
# Test no "hnsw graph not repaired" errors
$node->pgbench(
"--no-vacuum --client=5 --transactions=1000",
0,
[qr{actually processed}],
[qr{^$}],
"concurrent INSERTs, DELETEs, SELECTs, and VACUUM",
{
"047_hnsw_vacuum_insert_insert\@500" => "INSERT INTO tst (v) VALUES (ARRAY[$array_sql]);",
"047_hnsw_vacuum_insert_delete\@500" => "DELETE FROM tst WHERE i = (SELECT i FROM tst LIMIT 1);",
"047_hnsw_vacuum_insert_select\@20" => "SELECT i FROM tst ORDER BY v <-> '[0,0,0]' LIMIT 10;",
"047_hnsw_vacuum_insert_vacuum\@1" => "VACUUM tst;"
}
);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods' comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.8.4' default_version = '0.8.2'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true