mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
3 Commits
hnsw-query
...
ci6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
491804d99f | ||
|
|
f43cd0ed98 | ||
|
|
51df640961 |
63
.github/workflows/build.yml
vendored
63
.github/workflows/build.yml
vendored
@@ -29,9 +29,7 @@ jobs:
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
- run: |
|
||||
export PG_CONFIG=`which pg_config`
|
||||
sudo --preserve-env=PG_CONFIG make install
|
||||
- run: sudo make install
|
||||
- run: make installcheck
|
||||
- if: ${{ failure() }}
|
||||
run: cat regression.diffs
|
||||
@@ -39,62 +37,3 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install libipc-run-perl
|
||||
- run: make prove_installcheck
|
||||
mac:
|
||||
runs-on: macos-latest
|
||||
if: ${{ !startsWith(github.ref_name, 'windows') }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: 14
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
|
||||
- run: make install
|
||||
- run: make installcheck
|
||||
- if: ${{ failure() }}
|
||||
run: cat regression.diffs
|
||||
- run: |
|
||||
brew install cpanm
|
||||
cpanm --notest IPC::Run
|
||||
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
|
||||
tar xf REL_14_10.tar.gz
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
|
||||
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make PG_CFLAGS="-DUSE_ASSERT_CHECKING"
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
if: ${{ !startsWith(github.ref_name, 'mac') }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ankane/setup-postgres@v1
|
||||
with:
|
||||
postgres-version: 14
|
||||
- run: |
|
||||
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
|
||||
nmake /NOLOGO /F Makefile.win && ^
|
||||
nmake /NOLOGO /F Makefile.win install && ^
|
||||
nmake /NOLOGO /F Makefile.win installcheck && ^
|
||||
nmake /NOLOGO /F Makefile.win clean && ^
|
||||
nmake /NOLOGO /F Makefile.win uninstall
|
||||
shell: cmd
|
||||
i386:
|
||||
if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: debian:12
|
||||
options: --platform linux/386
|
||||
steps:
|
||||
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
|
||||
- run: service postgresql start
|
||||
- run: |
|
||||
git clone https://github.com/${{ github.repository }}.git pgvector
|
||||
cd pgvector
|
||||
git fetch origin ${{ github.ref }}
|
||||
git reset --hard FETCH_HEAD
|
||||
make
|
||||
make install
|
||||
chown -R postgres .
|
||||
sudo -u postgres make installcheck
|
||||
sudo -u postgres make prove_installcheck
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
|
||||
17
README.md
17
README.md
@@ -28,7 +28,7 @@ make install # may need sudo
|
||||
|
||||
See the [installation notes](#installation-notes) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector).
|
||||
|
||||
### Windows
|
||||
|
||||
@@ -716,6 +716,21 @@ sudo dnf install pgvector_16
|
||||
|
||||
Note: Replace `16` with your Postgres server version
|
||||
|
||||
### pkg
|
||||
|
||||
Install the FreeBSD package with:
|
||||
|
||||
```sh
|
||||
pkg install postgresql15-pg_vector
|
||||
```
|
||||
|
||||
or the port with:
|
||||
|
||||
```sh
|
||||
cd /usr/ports/databases/pg_vector
|
||||
make install
|
||||
```
|
||||
|
||||
### conda-forge
|
||||
|
||||
With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with:
|
||||
|
||||
11
src/hnsw.h
11
src/hnsw.h
@@ -362,11 +362,6 @@ typedef struct HnswVacuumState
|
||||
MemoryContext tmpCtx;
|
||||
} HnswVacuumState;
|
||||
|
||||
typedef struct HnswQuery
|
||||
{
|
||||
Datum value;
|
||||
} HnswQuery;
|
||||
|
||||
/* Methods */
|
||||
int HnswGetM(Relation index);
|
||||
int HnswGetEfConstruction(Relation index);
|
||||
@@ -375,14 +370,14 @@ bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * re
|
||||
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
|
||||
void HnswInitPage(Buffer buf, Page page);
|
||||
void HnswInit(void);
|
||||
List *HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
|
||||
List *HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement);
|
||||
HnswElement HnswGetEntryPoint(Relation index);
|
||||
void HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
|
||||
void *HnswAlloc(HnswAllocator * allocator, Size size);
|
||||
HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc);
|
||||
HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
|
||||
void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
|
||||
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
|
||||
HnswCandidate *HnswEntryCandidate(char *base, HnswElement em, Datum q, Relation rel, FmgrInfo *procinfo, Oid collation, bool loadVec);
|
||||
void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building);
|
||||
void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m);
|
||||
void HnswAddHeapTid(HnswElement element, ItemPointer heaptid);
|
||||
@@ -390,7 +385,7 @@ void HnswInitNeighbors(char *base, HnswElement element, int m, HnswAllocator *
|
||||
bool HnswInsertTupleOnDisk(Relation index, Datum value, Datum *values, bool *isnull, ItemPointer heap_tid, bool building);
|
||||
void HnswUpdateNeighborsOnDisk(Relation index, FmgrInfo *procinfo, Oid collation, HnswElement e, int m, bool checkExisting, bool building);
|
||||
void HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHeaptids, bool loadVec);
|
||||
void HnswLoadElement(HnswElement element, float *distance, HnswQuery * q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
|
||||
void HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec);
|
||||
void HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
|
||||
void HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
|
||||
void HnswLoadNeighbors(HnswElement element, Relation index, int m);
|
||||
|
||||
@@ -176,7 +176,7 @@ CreateGraphPages(HnswBuildState * buildstate)
|
||||
Size etupSize;
|
||||
Size ntupSize;
|
||||
Size combinedSize;
|
||||
void *valuePtr = HnswPtrAccess(base, element->value);
|
||||
Pointer valuePtr = HnswPtrAccess(base, element->value);
|
||||
|
||||
/* Update iterator */
|
||||
iter = element->next;
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* Algorithm 5 from paper
|
||||
*/
|
||||
static List *
|
||||
GetScanItems(IndexScanDesc scan, Datum value)
|
||||
GetScanItems(IndexScanDesc scan, Datum q)
|
||||
{
|
||||
HnswScanOpaque so = (HnswScanOpaque) scan->opaque;
|
||||
Relation index = scan->indexRelation;
|
||||
@@ -22,9 +22,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
int m;
|
||||
HnswElement entryPoint;
|
||||
char *base = NULL;
|
||||
HnswQuery q;
|
||||
|
||||
q.value = value;
|
||||
|
||||
/* Get m and entry point */
|
||||
HnswGetMetaPageInfo(index, &m, &entryPoint);
|
||||
@@ -32,15 +29,15 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
if (entryPoint == NULL)
|
||||
return NIL;
|
||||
|
||||
ep = list_make1(HnswEntryCandidate(base, entryPoint, &q, index, procinfo, collation, false));
|
||||
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, false));
|
||||
|
||||
for (int lc = entryPoint->level; lc >= 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, &q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
return HnswSearchLayer(base, &q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
||||
return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -555,7 +555,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
|
||||
* Load an element and optionally get its distance from q
|
||||
*/
|
||||
void
|
||||
HnswLoadElement(HnswElement element, float *distance, HnswQuery * q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
|
||||
HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
|
||||
{
|
||||
Buffer buf;
|
||||
Page page;
|
||||
@@ -575,7 +575,7 @@ HnswLoadElement(HnswElement element, float *distance, HnswQuery * q, Relation in
|
||||
|
||||
/* Calculate distance */
|
||||
if (distance != NULL)
|
||||
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q->value, PointerGetDatum(&etup->data)));
|
||||
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->data)));
|
||||
|
||||
UnlockReleaseBuffer(buf);
|
||||
}
|
||||
@@ -584,19 +584,19 @@ HnswLoadElement(HnswElement element, float *distance, HnswQuery * q, Relation in
|
||||
* Get the distance for a candidate
|
||||
*/
|
||||
static float
|
||||
GetCandidateDistance(char *base, HnswCandidate * hc, HnswQuery * q, FmgrInfo *procinfo, Oid collation)
|
||||
GetCandidateDistance(char *base, HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
|
||||
{
|
||||
HnswElement hce = HnswPtrAccess(base, hc->element);
|
||||
Datum value = HnswGetValue(base, hce);
|
||||
|
||||
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q->value, value));
|
||||
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, value));
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a candidate for the entry point
|
||||
*/
|
||||
HnswCandidate *
|
||||
HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
|
||||
HnswEntryCandidate(char *base, HnswElement entryPoint, Datum q, Relation index, FmgrInfo *procinfo, Oid collation, bool loadVec)
|
||||
{
|
||||
HnswCandidate *hc = palloc(sizeof(HnswCandidate));
|
||||
|
||||
@@ -604,7 +604,7 @@ HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation i
|
||||
if (index == NULL)
|
||||
hc->distance = GetCandidateDistance(base, hc, q, procinfo, collation);
|
||||
else
|
||||
HnswLoadElement(entryPoint, &hc->distance, q, index, procinfo, collation, loadVec);
|
||||
HnswLoadElement(entryPoint, &hc->distance, &q, index, procinfo, collation, loadVec);
|
||||
return hc;
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ CountElement(char *base, HnswElement skipElement, HnswCandidate * hc)
|
||||
* Algorithm 2 from paper
|
||||
*/
|
||||
List *
|
||||
HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
|
||||
HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, FmgrInfo *procinfo, Oid collation, int m, bool inserting, HnswElement skipElement)
|
||||
{
|
||||
List *w = NIL;
|
||||
pairingheap *C = pairingheap_allocate(CompareNearestCandidates, NULL);
|
||||
@@ -806,7 +806,7 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in
|
||||
if (index == NULL)
|
||||
eDistance = GetCandidateDistance(base, e, q, procinfo, collation);
|
||||
else
|
||||
HnswLoadElement(eElement, &eDistance, q, index, procinfo, collation, inserting);
|
||||
HnswLoadElement(eElement, &eDistance, &q, index, procinfo, collation, inserting);
|
||||
|
||||
Assert(!eElement->deleted);
|
||||
|
||||
@@ -1091,9 +1091,7 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
|
||||
/* Load elements on insert */
|
||||
if (index != NULL)
|
||||
{
|
||||
HnswQuery q;
|
||||
|
||||
q.value = HnswGetValue(base, hce);
|
||||
Datum q = HnswGetValue(base, hce);
|
||||
|
||||
for (int i = 0; i < currentNeighbors->length; i++)
|
||||
{
|
||||
@@ -1103,7 +1101,7 @@ HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm
|
||||
if (HnswPtrIsNull(base, hc3Element->value))
|
||||
HnswLoadElement(hc3Element, &hc3->distance, &q, index, procinfo, collation, true);
|
||||
else
|
||||
hc3->distance = GetCandidateDistance(base, hc3, &q, procinfo, collation);
|
||||
hc3->distance = GetCandidateDistance(base, hc3, q, procinfo, collation);
|
||||
|
||||
/* Prune element if being deleted */
|
||||
if (hc3Element->heaptidsLength == 0)
|
||||
@@ -1203,11 +1201,9 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
List *w;
|
||||
int level = element->level;
|
||||
int entryLevel;
|
||||
HnswQuery q;
|
||||
Datum q = HnswGetValue(base, element);
|
||||
HnswElement skipElement = existing ? element : NULL;
|
||||
|
||||
q.value = HnswGetValue(base, element);
|
||||
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
/* Precompute hash */
|
||||
if (index == NULL)
|
||||
@@ -1219,13 +1215,13 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
return;
|
||||
|
||||
/* Get entry point and level */
|
||||
ep = list_make1(HnswEntryCandidate(base, entryPoint, &q, index, procinfo, collation, true));
|
||||
ep = list_make1(HnswEntryCandidate(base, entryPoint, q, index, procinfo, collation, true));
|
||||
entryLevel = entryPoint->level;
|
||||
|
||||
/* 1st phase: greedy search to insert level */
|
||||
for (int lc = entryLevel; lc >= level + 1; lc--)
|
||||
{
|
||||
w = HnswSearchLayer(base, &q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, true, skipElement);
|
||||
ep = w;
|
||||
}
|
||||
|
||||
@@ -1243,7 +1239,7 @@ HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint
|
||||
List *neighbors;
|
||||
List *lw;
|
||||
|
||||
w = HnswSearchLayer(base, &q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
|
||||
w = HnswSearchLayer(base, q, ep, efConstruction, lc, index, procinfo, collation, m, true, skipElement);
|
||||
|
||||
/* Elements being deleted or skipped can help with search */
|
||||
/* but should be removed before selecting neighbors */
|
||||
|
||||
Reference in New Issue
Block a user