Compare commits

..

34 Commits

Author SHA1 Message Date
Andrew Kane
c207a2d50e Reduced lock contention with parallel HNSW index builds 2024-03-11 19:50:48 -07:00
Andrew Kane
569fd36396 Improved performance of parallel HNSW index builds 2024-03-11 18:32:39 -07:00
Andrew Kane
62350b1589 Added note about IVFFlat results [skip ci] 2024-03-06 00:29:55 -08:00
Andrew Kane
dd57309281 Added section about HNSW results - #480 [skip ci] 2024-03-06 00:27:50 -08:00
Andrew Kane
c6ddf62a29 Version bump to 0.6.1 [skip ci] 2024-03-04 10:27:59 -08:00
Andrew Kane
801be04d8b Updated CI [skip ci] 2024-03-02 17:06:11 -08:00
Andrew Kane
587e9ba97c Added link to bulk loading example [skip ci] 2024-02-29 21:25:11 -08:00
Andrew Kane
d57047a935 Moved COPY example and added FORMAT [skip ci] 2024-02-29 20:35:01 -08:00
oneturkmen
f1db1f17e3 Update README.md (#472) 2024-02-29 20:27:53 -08:00
Andrew Kane
2f48c0fac4 Updated changelog [skip ci] 2024-02-29 18:10:19 -08:00
Andrew Kane
84a8aa8176 Added note about HNSW build time [skip ci] 2024-02-29 16:25:58 -08:00
Andrew Kane
f64ebbef50 Added OPTFLAGS to readme [skip ci] 2024-02-29 16:15:22 -08:00
Andrew Kane
ac8156509b Updated vector operators for <= and >= 2024-02-29 15:43:29 -08:00
Andrew Kane
82bf69b479 Fixed vector subtraction being marked as commutative - fixes #475 2024-02-29 14:36:18 -08:00
Andrew Kane
4be2f57916 Updated license year [skip ci] 2024-02-29 13:44:58 -08:00
Andrew Kane
91e3d2905f Fixed sort function for Postgres 12 2024-02-28 16:26:41 -08:00
Andrew Kane
fe2406564f Replaced pairing heap with array in SelectNeighbors - closes #447
Co-authored-by: Heikki Linnakangas <heikki.linnakangas@iki.fi>
2024-02-28 15:47:26 -08:00
Andrew Kane
fa52511eaa Fixed closer caching for Postgres 12 2024-02-28 15:44:38 -08:00
Andrew Kane
447ef4d27a Specify Postgres version for Valgrind [skip ci] 2024-02-28 14:16:48 -08:00
Andrew Kane
b36cd22ccc Enable assertions on CI 2024-02-28 14:15:34 -08:00
Andrew Kane
b447ae4989 Avoid base address for relptr for Postgres < 14.5 2024-02-28 14:10:14 -08:00
Andrew Kane
efed873a3e Revert "Replaced pairing heap with array in SelectNeighbors - closes #447"
This reverts commit 14b278dec9.
2024-02-28 11:33:14 -08:00
Andrew Kane
14b278dec9 Replaced pairing heap with array in SelectNeighbors - closes #447
Co-authored-by: Heikki Linnakangas <heikki.linnakangas@iki.fi>
2024-02-28 11:25:12 -08:00
Andrew Kane
133a728e48 Updated changelog [skip ci] 2024-02-20 16:38:05 -08:00
Andrew Kane
ca10cbaa7d Revert "Remove offsethash"
This reverts commit 1cbd204f52.
2024-02-20 16:07:32 -08:00
Andrew Kane
eb29019a14 Revert "Eliminate a few HnswPtrAccess invocations"
This reverts commit 334c386a45.
2024-02-20 16:07:21 -08:00
Heikki Linnakangas
334c386a45 Eliminate a few HnswPtrAccess invocations
HnswPtrAccess() is pretty cheap, but it stills seems worthwhile to
avoid repeated calls in the hot paths when it can be easily avoided.
2024-02-19 14:13:18 +02:00
Heikki Linnakangas
1cbd204f52 Remove offsethash
The original motivation was to eliminate the superfluous HnswPtrAccess
call from AddToVisited. The caller has to call HnswPtrAccess() anyway,
so it makes sense to pass the HnswElement rather than HnswElementPtr
to AddToVisited(). But then I realized that we can use the
pointer-variant even with shared memory, because the visited-hash is
backend-private, and the addresses where the elements are mapped to in
shared memory are stable within the backend.
2024-02-19 14:12:26 +02:00
Andrew Kane
5ba62fca84 Fixed crash with shared_preload_libraries - fixes #460 2024-02-14 17:13:30 -08:00
Andrew Kane
22cb2a3fe7 Simplified Docker tasks [skip ci] 2024-02-07 12:03:33 -08:00
Andrew Kane
72b144906a Added Valgrind to CI 2024-02-07 09:43:01 -08:00
Andrew Kane
a618c1bc78 Updated port name [skip ci] 2024-02-06 12:43:20 -08:00
Andrew Kane
f43cd0ed98 Improved type [skip ci] 2024-02-05 19:38:52 -08:00
Andrew Kane
51df640961 Added instructions for pkg [skip ci] 2024-02-05 17:01:36 -08:00
17 changed files with 209 additions and 122 deletions

View File

@@ -28,7 +28,7 @@ jobs:
dev-files: true
- run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
@@ -49,7 +49,7 @@ jobs:
postgres-version: 14
- run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install
- run: make installcheck
- if: ${{ failure() }}
@@ -60,7 +60,9 @@ jobs:
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
tar xf REL_14_10.tar.gz
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make PG_CFLAGS="-DUSE_ASSERT_CHECKING"
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
env:
PG_CFLAGS: -DUSE_ASSERT_CHECKING
windows:
runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }}
@@ -97,4 +99,15 @@ jobs:
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
valgrind:
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ankane/setup-postgres-valgrind@v1
with:
postgres-version: 16
- run: make
- run: sudo --preserve-env=PG_CONFIG make install
- run: make installcheck

View File

@@ -1,6 +1,12 @@
## 0.6.1 (unreleased)
## 0.6.2 (unreleased)
- Reduced lock contention with parallel HNSW index builds
## 0.6.1 (2024-03-04)
- Fixed error with `ANALYZE` and vectors with different dimensions
- Fixed error with `shared_preload_libraries`
- Fixed vector subtraction being marked as commutative
## 0.6.0 (2024-01-29)

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.6.0",
"version": "0.6.1",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.6.0",
"version": "0.6.1",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.6.0
EXTVERSION = 0.6.1
MODULE_big = vector
DATA = $(wildcard sql/*--*.sql)
@@ -71,11 +71,9 @@ PG_MAJOR ?= 16
.PHONY: docker
docker:
docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
docker build --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
docker buildx build --push --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.6.0
EXTVERSION = 0.6.1
OBJS = src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
HEADERS = src\vector.h

View File

@@ -20,7 +20,7 @@ Compile and install the extension (supports Postgres 12+)
```sh
cd /tmp
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -28,7 +28,7 @@ make install # may need sudo
See the [installation notes](#installation-notes) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
### Windows
@@ -44,7 +44,7 @@ Then use `nmake` to build:
```cmd
set "PGROOT=C:\Program Files\PostgreSQL\16"
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
cd pgvector
nmake /F Makefile.win
nmake /F Makefile.win install
@@ -102,6 +102,12 @@ Insert vectors
INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]');
```
Or load vectors in bulk using `COPY` ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/bulk_loading.py))
```sql
COPY items (embedding) FROM STDIN WITH (FORMAT BINARY);
```
Upsert vectors
```sql
@@ -264,6 +270,8 @@ HINT: Increase maintenance_work_mem to speed up builds.
Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on the server
Like other index types, its faster to create an index after loading your initial data
Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default)
```sql
@@ -573,6 +581,10 @@ or choose to store vectors inline:
ALTER TABLE items ALTER COLUMN embedding SET STORAGE PLAIN;
```
#### Why are there less results for a query after adding an HNSW index?
Results are limited by the size of the dynamic candidate list (`hnsw.ef_search`). There may be even less results due to dead tuples or filtering conditions in the query. We recommend setting `hnsw.ef_search` to at least twice the `LIMIT` of the query. If you need more than 500 results, use an IVFFlat index instead.
#### Why are there less results for a query after adding an IVFFlat index?
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
@@ -581,6 +593,8 @@ The index was likely created with too little data for the number of lists. Drop
DROP INDEX index_name;
```
Results can also be limited by the number of probes (`ivfflat.probes`).
## Reference
### Vector Type
@@ -656,6 +670,14 @@ Note: Replace `16` with your Postgres server version
If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools.
### Portability
To compile for portability, use:
```sh
make OPTFLAGS=""
```
## Additional Installation Methods
### Docker
@@ -671,7 +693,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually:
```sh
git clone --branch v0.6.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.6.1 https://github.com/pgvector/pgvector.git
cd pgvector
docker build --build-arg PG_MAJOR=16 -t myuser/pgvector .
```
@@ -716,6 +738,21 @@ sudo dnf install pgvector_16
Note: Replace `16` with your Postgres server version
### pkg
Install the FreeBSD package with:
```sh
pkg install postgresql15-pg_vector
```
or the port with:
```sh
cd /usr/ports/databases/pgvector
make install
```
### conda-forge
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:

View File

@@ -0,0 +1,16 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.6.1'" to load this file. \quit
DROP OPERATOR - (vector, vector);
CREATE OPERATOR - (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub
);
ALTER OPERATOR <= (vector, vector) SET (
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
ALTER OPERATOR >= (vector, vector) SET (
RESTRICT = scalargesel, JOIN = scalargejoinsel
);

View File

@@ -180,8 +180,7 @@ CREATE OPERATOR + (
);
CREATE OPERATOR - (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub,
COMMUTATOR = -
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_sub
);
CREATE OPERATOR * (
@@ -195,11 +194,10 @@ CREATE OPERATOR < (
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
);
-- should use scalarlesel and scalarlejoinsel, but not supported in Postgres < 11
CREATE OPERATOR <= (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_le,
COMMUTATOR = >= , NEGATOR = > ,
RESTRICT = scalarltsel, JOIN = scalarltjoinsel
RESTRICT = scalarlesel, JOIN = scalarlejoinsel
);
CREATE OPERATOR = (
@@ -214,11 +212,10 @@ CREATE OPERATOR <> (
RESTRICT = eqsel, JOIN = eqjoinsel
);
-- should use scalargesel and scalargejoinsel, but not supported in Postgres < 11
CREATE OPERATOR >= (
LEFTARG = vector, RIGHTARG = vector, PROCEDURE = vector_ge,
COMMUTATOR = <= , NEGATOR = < ,
RESTRICT = scalargtsel, JOIN = scalargtjoinsel
RESTRICT = scalargesel, JOIN = scalargejoinsel
);
CREATE OPERATOR > (

View File

@@ -8,6 +8,7 @@
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "hnsw.h"
#include "miscadmin.h"
#include "utils/guc.h"
#include "utils/selfuncs.h"
@@ -28,7 +29,7 @@ static relopt_kind hnsw_relopt_kind;
* this grows bigger, we should use a shmem_request_hook and
* RequestAddinShmemSpace() to pre-reserve space for this.
*/
static void
void
HnswInitLockTranche(void)
{
int *tranche_ids;
@@ -53,7 +54,8 @@ HnswInitLockTranche(void)
void
HnswInit(void)
{
HnswInitLockTranche();
if (!process_shared_preload_libraries_in_progress)
HnswInitLockTranche();
hnsw_relopt_kind = add_reloption_kind();
add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",

View File

@@ -80,7 +80,7 @@
#if PG_VERSION_NUM < 130000
#define list_delete_last(list) list_truncate(list, list_length(list) - 1)
#define list_sort(list, cmp) list_qsort(list, cmp)
#define list_sort(list, cmp) ((list) = list_qsort(list, cmp))
#endif
#define HnswIsElementTuple(tup) ((tup)->type == HNSW_ELEMENT_TUPLE_TYPE)
@@ -185,6 +185,7 @@ typedef struct HnswGraph
/* Entry state */
LWLock entryLock;
LWLock entryWaitLock;
HnswElementPtr entryPoint;
/* Allocations state */
@@ -234,12 +235,6 @@ typedef struct HnswAllocator
void *state;
} HnswAllocator;
typedef struct HnswSupport
{
FmgrInfo *procinfo;
Oid collation;
} HnswSupport;
typedef struct HnswBuildState
{
/* Info */
@@ -258,7 +253,7 @@ typedef struct HnswBuildState
double reltuples;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
@@ -335,7 +330,7 @@ typedef struct HnswScanOpaqueData
MemoryContext tmpCtx;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
Oid collation;
} HnswScanOpaqueData;
@@ -355,7 +350,8 @@ typedef struct HnswVacuumState
int efConstruction;
/* Support functions */
HnswSupport support;
FmgrInfo *procinfo;
Oid collation;
/* Variables */
struct tidhash_hash *deleted;
@@ -371,30 +367,30 @@ typedef struct HnswVacuumState
int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
void HnswInitSupport(HnswSupport * support, Relation index);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);
void HnswInit(void);
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement);
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
HnswElement HnswGetEntryPoint(Relation index);
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
void *HnswAlloc(HnswAllocator * allocator, Size size);
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing);
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, HnswSupport * support, bool loadVec);
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator * alloc);
bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building);
void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, HnswSupport * support, bool loadVec);
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, HnswSupport * support);
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
void HnswInitLockTranche(void);
PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
/* Index access methods */

View File

@@ -176,7 +176,7 @@ CreateGraphPages(HnswBuildState * buildstate)
Size etupSize;
Size ntupSize;
Size combinedSize;
void *valuePtr = HnswPtrAccess(base, element->value);
Pointer valuePtr = HnswPtrAccess(base, element->value);
/* Update iterator */
iter = element->next;
@@ -373,7 +373,7 @@ AddElementInMemory(char *base, HnswGraph * graph, HnswElement element)
* Update neighbors
*/
static void
UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
UpdateNeighborsInMemory(char *base, FmgrInfo *procinfo, Oid collation, HnswElement e, int m)
{
for (int lc = e->level; lc >= 0; lc--)
{
@@ -390,7 +390,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
/* Use element for lock instead of hc since hc can be replaced */
LWLockAcquire(&neighborElement->lock, LW_EXCLUSIVE);
HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, support);
HnswUpdateConnection(base, e, hc, lm, lc, NULL, NULL, procinfo, collation);
LWLockRelease(&neighborElement->lock);
}
}
@@ -400,7 +400,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m)
* Update graph in memory
*/
static void
UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
UpdateGraphInMemory(FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate)
{
HnswGraph *graph = buildstate->graph;
char *base = buildstate->hnswarea;
@@ -413,7 +413,7 @@ UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efCon
AddElementInMemory(base, graph, element);
/* Update neighbors */
UpdateNeighborsInMemory(base, support, element, m);
UpdateNeighborsInMemory(base, procinfo, collation, element, m);
/* Update entry point if needed (already have lock) */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -426,13 +426,20 @@ UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efCon
static void
InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
{
FmgrInfo *procinfo = buildstate->procinfo;
Oid collation = buildstate->collation;
HnswGraph *graph = buildstate->graph;
HnswElement entryPoint;
LWLock *entryLock = &graph->entryLock;
LWLock *entryWaitLock = &graph->entryWaitLock;
int efConstruction = buildstate->efConstruction;
int m = buildstate->m;
char *base = buildstate->hnswarea;
/* Wait if another process needs exclusive lock */
if (LWLockAcquireOrWait(entryWaitLock, LW_EXCLUSIVE))
LWLockRelease(entryWaitLock);
/* Get entry point */
LWLockAcquire(entryLock, LW_SHARED);
entryPoint = HnswPtrAccess(base, graph->entryPoint);
@@ -444,17 +451,19 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element)
LWLockRelease(entryLock);
/* Get exclusive lock */
LWLockAcquire(entryWaitLock, LW_EXCLUSIVE);
LWLockAcquire(entryLock, LW_EXCLUSIVE);
LWLockRelease(entryWaitLock);
/* Get latest entry point after lock is acquired */
entryPoint = HnswPtrAccess(base, graph->entryPoint);
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, NULL, &buildstate->support, m, efConstruction, false);
HnswFindElementNeighbors(base, element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
/* Update graph in memory */
UpdateGraphInMemory(&buildstate->support, element, m, efConstruction, entryPoint, buildstate);
UpdateGraphInMemory(procinfo, collation, element, m, efConstruction, entryPoint, buildstate);
/* Release entry lock */
LWLockRelease(entryLock);
@@ -599,6 +608,9 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
static void
InitGraph(HnswGraph * graph, char *base, long memoryTotal)
{
/* Initialize the lock tranche if needed */
HnswInitLockTranche();
HnswPtrStore(base, graph->head, (HnswElement) NULL);
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
graph->memoryUsed = 0;
@@ -607,6 +619,7 @@ InitGraph(HnswGraph * graph, char *base, long memoryTotal)
graph->indtuples = 0;
SpinLockInit(&graph->lock);
LWLockInitialize(&graph->entryLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->entryWaitLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->allocatorLock, hnsw_lock_tranche_id);
LWLockInitialize(&graph->flushLock, hnsw_lock_tranche_id);
}
@@ -681,7 +694,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->indtuples = 0;
/* Get support functions */
HnswInitSupport(&buildstate->support, index);
buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
buildstate->collation = index->rd_indcollation[0];
@@ -972,6 +985,14 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
/* Report less than allocated so never fails */
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
/*
* Avoid base address for relptr for Postgres < 14.5
* https://github.com/postgres/postgres/commit/7201cd18627afc64850537806da7f22150d1a83b
*/
#if PG_VERSION_NUM < 140005
hnswshared->graphData.memoryUsed += MAXALIGN(1);
#endif
shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_SHARED, hnswshared);
shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_AREA, hnswarea);

View File

@@ -340,7 +340,7 @@ ConnectionExists(HnswElement e, HnswNeighborTuple ntup, int startIdx, int lm)
* Update neighbors
*/
void
HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e, int m, bool checkExisting, bool building)
HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building)
{
char *base = NULL;
@@ -373,7 +373,7 @@ HnswUpdateNeighborsOnDisk(Relation index, HnswSupport * support, HnswElement e,
*/
/* Select neighbors */
HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, support);
HnswUpdateConnection(NULL, e, hc, lm, lc, &idx, index, procinfo, collation);
/* New element was not selected as a neighbor */
if (idx == -1)
@@ -527,7 +527,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building)
* Update graph on disk
*/
static void
UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
UpdateGraphOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building)
{
BlockNumber newInsertPage = InvalidBlockNumber;
@@ -543,7 +543,7 @@ UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, in
HnswUpdateMetaPage(index, 0, NULL, newInsertPage, MAIN_FORKNUM, building);
/* Update neighbors */
HnswUpdateNeighborsOnDisk(index, support, element, m, false, building);
HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, false, building);
/* Update entry point if needed */
if (entryPoint == NULL || element->level > entryPoint->level)
@@ -560,11 +560,10 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
HnswElement element;
int m;
int efConstruction = HnswGetEfConstruction(index);
FmgrInfo *procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
Oid collation = index->rd_indcollation[0];
LOCKMODE lockmode = ShareLock;
char *base = NULL;
HnswSupport support;
HnswInitSupport(&support, index);
/*
* Get a shared lock. This allows vacuum to ensure no in-flight inserts
@@ -595,10 +594,10 @@ HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull,
}
/* Find neighbors for element */
HnswFindElementNeighbors(base, element, entryPoint, index, &support, m, efConstruction, false);
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, false);
/* Update graph on disk */
UpdateGraphOnDisk(index, &support, element, m, efConstruction, entryPoint, building);
UpdateGraphOnDisk(index, procinfo, collation, element, m, efConstruction, entryPoint, building);
/* Release lock */
UnlockPage(index, HNSW_UPDATE_LOCK, lockmode);

View File

@@ -15,6 +15,8 @@ GetScanItems(IndexScanDesc scan, Datum q)
{
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
Relation index = scan->indexRelation;
FmgrInfo *procinfo = so->procinfo;
Oid collation = so->collation;
List *ep;
List *w;
int m;
@@ -27,15 +29,15 @@ GetScanItems(IndexScanDesc scan, Datum q)
if (entryPoint == NULL)
return NIL;
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, &so->support, false));
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
for (int lc = entryPoint->level; lc >= 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, &so->support, m, false, NULL);
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
ep = w;
}
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, &so->support, m, false, NULL);
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
}
/*
@@ -106,7 +108,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
ALLOCSET_DEFAULT_SIZES);
/* Set support functions */
HnswInitSupport(&so->support, index);
so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
so->collation = index->rd_indcollation[0];

View File

@@ -149,16 +149,6 @@ HnswOptionalProcInfo(Relation index, uint16 procnum)
return index_getprocinfo(index, 1, procnum);
}
/*
* Init support function
*/
void
HnswInitSupport(HnswSupport * support, Relation index)
{
support->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
support->collation = index->rd_indcollation[0];
}
/*
* Divide by the norm
*
@@ -565,7 +555,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
* Load an element and optionally get its distance from q
*/
void
HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, HnswSupport * support, bool loadVec)
HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
{
Buffer buf;
Page page;
@@ -585,7 +575,7 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
/* Calculate distance */
if (distance != NULL)
*distance = (float) DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, *q, PointerGetDatum(&etup->data)));
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
UnlockReleaseBuffer(buf);
}
@@ -594,27 +584,27 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
* Get the distance for a candidate
*/
static float
GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, HnswSupport * support)
GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
{
HnswElement hce = HnswPtrAccess(base, hc->element);
Datum value = HnswGetValue(base, hce);
return DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, q, value));
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
}
/*
* Create a candidate for the entry point
*/
HnswCandidate *
HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, HnswSupport * support, bool loadVec)
HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
{
HnswCandidate *hc = palloc(sizeof(HnswCandidate));
HnswPtrStore(base, hc->element, entryPoint);
if (index == NULL)
hc->distance = GetCandidateDistance(base, hc, q, support);
hc->distance = GetCandidateDistance(base, hc, q, procinfo, collation);
else
HnswLoadElement(entryPoint, &hc->distance, &q, index, support, loadVec);
HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec);
return hc;
}
@@ -732,7 +722,7 @@ CountElement(char *base, HnswElement skipElement, HnswCandidate * hc)
* Algorithm 2 from paper
*/
List *
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, HnswSupport * support, int m, bool inserting, HnswElement skipElement)
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
{
List *w = NIL;
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
@@ -814,9 +804,9 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, H
f = ((HnswPairingHeapNode *) pairingheap_first(W))->inner;
if (index == NULL)
eDistance = GetCandidateDistance(base, e, q, support);
eDistance = GetCandidateDistance(base, e, q, procinfo, collation);
else
HnswLoadElement(eElement, &eDistance, &q, index, support, inserting);
HnswLoadElement(eElement, &eDistance, &q, index, procinfo, collation, inserting);
Assert(!eElement->deleted);
@@ -870,12 +860,15 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, H
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistances(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistances(const void *a, const void *b)
#endif
{
HnswCandidate *hca = lfirst((ListCell *) a);
HnswCandidate *hcb = lfirst((ListCell *) b);
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
if (hca->distance < hcb->distance)
return 1;
@@ -898,12 +891,15 @@ CompareCandidateDistances(const void *a, const void *b)
static int
#if PG_VERSION_NUM >= 130000
CompareCandidateDistancesOffset(const ListCell *a, const ListCell *b)
{
HnswCandidate *hca = lfirst(a);
HnswCandidate *hcb = lfirst(b);
#else
CompareCandidateDistancesOffset(const void *a, const void *b)
#endif
{
HnswCandidate *hca = lfirst((ListCell *) a);
HnswCandidate *hcb = lfirst((ListCell *) b);
HnswCandidate *hca = lfirst(*(ListCell **) a);
HnswCandidate *hcb = lfirst(*(ListCell **) b);
#endif
if (hca->distance < hcb->distance)
return 1;
@@ -924,19 +920,19 @@ CompareCandidateDistancesOffset(const void *a, const void *b)
* Calculate the distance between elements
*/
static float
HnswGetDistance(char *base, HnswElement a, HnswElement b, HnswSupport * support)
HnswGetDistance(char *base, HnswElement a, HnswElement b, FmgrInfo *procinfo, Oid collation)
{
Datum aValue = HnswGetValue(base, a);
Datum bValue = HnswGetValue(base, b);
return DatumGetFloat8(FunctionCall2Coll(support->procinfo, support->collation, aValue, bValue));
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, aValue, bValue));
}
/*
* Check if an element is closer to q than any element from R
*/
static bool
CheckElementCloser(char *base, HnswCandidate * e, List *r, HnswSupport * support)
CheckElementCloser(char *base, HnswCandidate * e, List *r, FmgrInfo *procinfo, Oid collation)
{
HnswElement eElement = HnswPtrAccess(base, e->element);
ListCell *lc2;
@@ -945,7 +941,7 @@ CheckElementCloser(char *base, HnswCandidate * e, List *r, HnswSupport * support
{
HnswCandidate *ri = lfirst(lc2);
HnswElement riElement = HnswPtrAccess(base, ri->element);
float distance = HnswGetDistance(base, eElement, riElement, support);
float distance = HnswGetDistance(base, eElement, riElement, procinfo, collation);
if (distance <= e->distance)
return false;
@@ -958,11 +954,13 @@ CheckElementCloser(char *base, HnswCandidate * e, List *r, HnswSupport * support
* Algorithm 4 from paper
*/
static List *
SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, HnswElement e2, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
SelectNeighbors(char *base, List *c, int lm, int lc, FmgrInfo *procinfo, Oid collation, HnswElement e2, HnswCandidate * newCandidate, HnswCandidate * *pruned, bool sortCandidates)
{
List *r = NIL;
List *w = list_copy(c);
pairingheap *wd;
HnswCandidate **wd;
int wdlen = 0;
int wdoff = 0;
HnswNeighborArray *neighbors = HnswGetNeighbors(base, e2, lc);
bool mustCalculate = !neighbors->closerSet;
List *added = NIL;
@@ -971,7 +969,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
if (list_length(w) <= lm)
return w;
wd = pairingheap_allocate(CompareNearestCandidates, NULL);
wd = palloc(sizeof(HnswCandidate *) * list_length(w));
/* Ensure order of candidates is deterministic for closer caching */
if (sortCandidates)
@@ -991,7 +989,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
/* Use previous state of r and wd to skip work when possible */
if (mustCalculate)
e->closer = CheckElementCloser(base, e, r, support);
e->closer = CheckElementCloser(base, e, r, procinfo, collation);
else if (list_length(added) > 0)
{
/* Keep Valgrind happy for in-memory, parallel builds */
@@ -1004,7 +1002,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
*/
if (e->closer)
{
e->closer = CheckElementCloser(base, e, added, support);
e->closer = CheckElementCloser(base, e, added, procinfo, collation);
if (!e->closer)
removedAny = true;
@@ -1017,7 +1015,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
*/
if (removedAny)
{
e->closer = CheckElementCloser(base, e, r, support);
e->closer = CheckElementCloser(base, e, r, procinfo, collation);
if (e->closer)
added = lappend(added, e);
}
@@ -1025,7 +1023,7 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
}
else if (e == newCandidate)
{
e->closer = CheckElementCloser(base, e, r, support);
e->closer = CheckElementCloser(base, e, r, procinfo, collation);
if (e->closer)
added = lappend(added, e);
}
@@ -1037,21 +1035,21 @@ SelectNeighbors(char *base, List *c, int lm, int lc, HnswSupport * support, Hnsw
if (e->closer)
r = lappend(r, e);
else
pairingheap_add(wd, &(CreatePairingHeapNode(e)->ph_node));
wd[wdlen++] = e;
}
/* Cached value can only be used in future if sorted deterministically */
neighbors->closerSet = sortCandidates;
/* Keep pruned connections */
while (!pairingheap_is_empty(wd) && list_length(r) < lm)
r = lappend(r, ((HnswPairingHeapNode *) pairingheap_remove_first(wd))->inner);
while (wdoff < wdlen && list_length(r) < lm)
r = lappend(r, wd[wdoff++]);
/* Return pruned for update connections */
if (pruned != NULL)
{
if (!pairingheap_is_empty(wd))
*pruned = ((HnswPairingHeapNode *) pairingheap_first(wd))->inner;
if (wdoff < wdlen)
*pruned = wd[wdoff];
else
*pruned = linitial(w);
}
@@ -1076,7 +1074,7 @@ AddConnections(char *base, HnswElement element, List *neighbors, int lc)
* Update connections
*/
void
HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, HnswSupport * support)
HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation)
{
HnswElement hce = HnswPtrAccess(base, hc->element);
HnswNeighborArray *currentNeighbors = HnswGetNeighbors(base, hce, lc);
@@ -1109,9 +1107,9 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
HnswElement hc3Element = HnswPtrAccess(base, hc3->element);
if (HnswPtrIsNull(base, hc3Element->value))
HnswLoadElement(hc3Element, &hc3->distance, &q, index, support, true);
HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true);
else
hc3->distance = GetCandidateDistance(base, hc3, q, support);
hc3->distance = GetCandidateDistance(base, hc3, q, procinfo, collation);
/* Prune element if being deleted */
if (hc3Element->heaptidsLength == 0)
@@ -1131,7 +1129,7 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
c = lappend(c, &currentNeighbors->items[i]);
c = lappend(c, &hc2);
SelectNeighbors(base, c, lm, lc, support, hce, &hc2, &pruned, true);
SelectNeighbors(base, c, lm, lc, procinfo, collation, hce, &hc2, &pruned, true);
/* Should not happen */
if (pruned == NULL)
@@ -1205,7 +1203,7 @@ PrecomputeHash(char *base, HnswElement element)
* Algorithm 1 from paper
*/
void
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing)
HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing)
{
List *ep;
List *w;
@@ -1225,13 +1223,13 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
return;
/* Get entry point and level */
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, support, true));
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, true));
entryLevel = entryPoint->level;
/* 1st phase: greedy search to insert level */
for (int lc = entryLevel; lc >= level + 1; lc--)
{
w = HnswSearchLayer(base, q, ep, 1, lc, index, support, m, true, skipElement);
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
ep = w;
}
@@ -1249,7 +1247,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
List *neighbors;
List *lw;
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, support, m, true, skipElement);
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
/* Elements being deleted or skipped can help with search */
/* but should be removed before selecting neighbors */
@@ -1263,7 +1261,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
* sortCandidates to true for in-memory builds to enable closer
* caching, but there does not seem to be a difference in performance.
*/
neighbors = SelectNeighbors(base, lw, lm, lc, support, element, NULL, NULL, false);
neighbors = SelectNeighbors(base, lw, lm, lc, procinfo, collation, element, NULL, NULL, false);
AddConnections(base, element, neighbors, lc);

View File

@@ -189,6 +189,8 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
GenericXLogState *state;
int m = vacuumstate->m;
int efConstruction = vacuumstate->efConstruction;
FmgrInfo *procinfo = vacuumstate->procinfo;
Oid collation = vacuumstate->collation;
BufferAccessStrategy bas = vacuumstate->bas;
HnswNeighborTuple ntup = vacuumstate->ntup;
Size ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, m);
@@ -203,7 +205,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
element->heaptidsLength = 0;
/* Find neighbors for element, skipping itself */
HnswFindElementNeighbors(base, element, entryPoint, index, &vacuumstate->support, m, efConstruction, true);
HnswFindElementNeighbors(base, element, entryPoint, index, procinfo, collation, m, efConstruction, true);
/* Zero memory for each element */
MemSet(ntup, 0, HNSW_TUPLE_ALLOC_SIZE);
@@ -227,7 +229,7 @@ RepairGraphElement(HnswVacuumState * vacuumstate, HnswElement element, HnswEleme
UnlockReleaseBuffer(buf);
/* Update neighbors */
HnswUpdateNeighborsOnDisk(index, &vacuumstate->support, element, m, true, false);
HnswUpdateNeighborsOnDisk(index, procinfo, collation, element, m, true, false);
}
/*
@@ -254,7 +256,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
LockPage(index, HNSW_UPDATE_LOCK, ShareLock);
/* Load element */
HnswLoadElement(highestPoint, NULL, NULL, index, &vacuumstate->support, true);
HnswLoadElement(highestPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
/* Repair if needed */
if (NeedsUpdated(vacuumstate, highestPoint))
@@ -292,7 +294,7 @@ RepairGraphEntryPoint(HnswVacuumState * vacuumstate)
* is outdated, this can remove connections at higher levels in
* the graph until they are repaired, but this should be fine.
*/
HnswLoadElement(entryPoint, NULL, NULL, index, &vacuumstate->support, true);
HnswLoadElement(entryPoint, NULL, NULL, index, vacuumstate->procinfo, vacuumstate->collation, true);
if (NeedsUpdated(vacuumstate, entryPoint))
{
@@ -571,8 +573,8 @@ InitVacuumState(HnswVacuumState * vacuumstate, IndexVacuumInfo *info, IndexBulkD
vacuumstate->callback_state = callback_state;
vacuumstate->efConstruction = HnswGetEfConstruction(index);
vacuumstate->bas = GetAccessStrategy(BAS_BULKREAD);
vacuumstate->support.procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
vacuumstate->support.collation = index->rd_indcollation[0];
vacuumstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
vacuumstate->collation = index->rd_indcollation[0];
vacuumstate->ntup = palloc0(HNSW_TUPLE_ALLOC_SIZE);
vacuumstate->tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
"Hnsw vacuum temporary context",

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat and hnsw access methods'
default_version = '0.6.0'
default_version = '0.6.1'
module_pathname = '$libdir/vector'
relocatable = true