Compare commits

..

6 Commits

Author SHA1 Message Date
Andrew Kane
fc5cce1cb8 Use FreeTupleDesc 2023-10-22 19:39:33 -07:00
Andrew Kane
1749ecb6e7 Copy the tuple descriptor 2023-10-22 19:36:45 -07:00
Andrew Kane
592a711a94 Fixed CI 2023-10-22 19:31:04 -07:00
Andrew Kane
139007ea68 Disabled compression for index tuples with IVFFlat 2023-10-22 19:26:25 -07:00
Andrew Kane
3f49b95f01 Added Postgres 17 to CI [skip ci] 2023-10-19 00:37:24 -07:00
Andrew Kane
ef1bea7163 Updated checkout action [skip ci] 2023-10-19 00:36:53 -07:00
12 changed files with 63 additions and 193 deletions

View File

@@ -8,6 +8,8 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- postgres: 17
os: ubuntu-22.04
- postgres: 16 - postgres: 16
os: ubuntu-22.04 os: ubuntu-22.04
- postgres: 15 - postgres: 15
@@ -21,7 +23,7 @@ jobs:
- postgres: 11 - postgres: 11
os: ubuntu-20.04 os: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
@@ -43,7 +45,7 @@ jobs:
runs-on: macos-latest runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }} if: ${{ !startsWith(github.ref_name, 'windows') }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: 14 postgres-version: 14
@@ -65,7 +67,7 @@ jobs:
runs-on: windows-latest runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }} if: ${{ !startsWith(github.ref_name, 'mac') }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: 14 postgres-version: 14

View File

@@ -33,12 +33,6 @@ HnswInit(void)
HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION HNSW_DEFAULT_EF_CONSTRUCTION, HNSW_MIN_EF_CONSTRUCTION, HNSW_MAX_EF_CONSTRUCTION
#if PG_VERSION_NUM >= 130000 #if PG_VERSION_NUM >= 130000
,AccessExclusiveLock ,AccessExclusiveLock
#endif
);
add_int_reloption(hnsw_relopt_kind, "dimensions", "Number of dimensions",
HNSW_DEFAULT_DIMENSIONS, HNSW_MIN_DIMENSIONS, HNSW_MAX_DIMENSIONS
#if PG_VERSION_NUM >= 130000
,AccessExclusiveLock
#endif #endif
); );
@@ -131,7 +125,6 @@ hnswoptions(Datum reloptions, bool validate)
static const relopt_parse_elt tab[] = { static const relopt_parse_elt tab[] = {
{"m", RELOPT_TYPE_INT, offsetof(HnswOptions, m)}, {"m", RELOPT_TYPE_INT, offsetof(HnswOptions, m)},
{"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)}, {"ef_construction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)},
{"dimensions", RELOPT_TYPE_INT, offsetof(HnswOptions, dimensions)},
}; };
#if PG_VERSION_NUM >= 130000 #if PG_VERSION_NUM >= 130000

View File

@@ -42,9 +42,6 @@
#define HNSW_DEFAULT_EF_SEARCH 40 #define HNSW_DEFAULT_EF_SEARCH 40
#define HNSW_MIN_EF_SEARCH 1 #define HNSW_MIN_EF_SEARCH 1
#define HNSW_MAX_EF_SEARCH 1000 #define HNSW_MAX_EF_SEARCH 1000
#define HNSW_DEFAULT_DIMENSIONS -1
#define HNSW_MIN_DIMENSIONS 1
#define HNSW_MAX_DIMENSIONS HNSW_MAX_DIM
/* Tuple types */ /* Tuple types */
#define HNSW_ELEMENT_TUPLE_TYPE 1 #define HNSW_ELEMENT_TUPLE_TYPE 1
@@ -62,7 +59,7 @@
#define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData)) #define HNSW_MAX_SIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - MAXALIGN(sizeof(HnswPageOpaqueData)) - sizeof(ItemIdData))
#define HNSW_ELEMENT_TUPLE_SIZE(_datum) MAXALIGN(offsetof(HnswElementTupleData, value) + VARSIZE_ANY(_datum)) #define HNSW_ELEMENT_TUPLE_SIZE(_dim) MAXALIGN(offsetof(HnswElementTupleData, vec) + VECTOR_SIZE(_dim))
#define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData)) #define HNSW_NEIGHBOR_TUPLE_SIZE(level, m) MAXALIGN(offsetof(HnswNeighborTupleData, indextids) + ((level) + 2) * (m) * sizeof(ItemPointerData))
#define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page)) #define HnswPageGetOpaque(page) ((HnswPageOpaque) PageGetSpecialPointer(page))
@@ -106,7 +103,7 @@ typedef struct HnswElementData
OffsetNumber offno; OffsetNumber offno;
OffsetNumber neighborOffno; OffsetNumber neighborOffno;
BlockNumber neighborPage; BlockNumber neighborPage;
Datum value; Vector *vec;
} HnswElementData; } HnswElementData;
typedef HnswElementData * HnswElement; typedef HnswElementData * HnswElement;
@@ -137,7 +134,6 @@ typedef struct HnswOptions
int32 vl_len_; /* varlena header (do not touch directly!) */ int32 vl_len_; /* varlena header (do not touch directly!) */
int m; /* number of connections */ int m; /* number of connections */
int efConstruction; /* size of dynamic candidate list */ int efConstruction; /* size of dynamic candidate list */
int dimensions;
} HnswOptions; } HnswOptions;
typedef struct HnswBuildState typedef struct HnswBuildState
@@ -208,7 +204,7 @@ typedef struct HnswElementTupleData
ItemPointerData heaptids[HNSW_HEAPTIDS]; ItemPointerData heaptids[HNSW_HEAPTIDS];
ItemPointerData neighbortid; ItemPointerData neighbortid;
uint16 unused2; uint16 unused2;
char value[FLEXIBLE_ARRAY_MEMBER]; Vector vec;
} HnswElementTupleData; } HnswElementTupleData;
typedef HnswElementTupleData * HnswElementTuple; typedef HnswElementTupleData * HnswElementTuple;
@@ -266,7 +262,6 @@ typedef struct HnswVacuumState
/* Methods */ /* Methods */
int HnswGetM(Relation index); int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index); int HnswGetEfConstruction(Relation index);
int HnswGetDimensions(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum); FmgrInfo *HnswOptionalProcInfo(Relation index, uint16 procnum);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result); bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
void HnswCommitBuffer(Buffer buf, GenericXLogState *state); void HnswCommitBuffer(Buffer buf, GenericXLogState *state);

View File

@@ -8,7 +8,6 @@
#include "lib/pairingheap.h" #include "lib/pairingheap.h"
#include "nodes/pg_list.h" #include "nodes/pg_list.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
#if PG_VERSION_NUM >= 140000 #if PG_VERSION_NUM >= 140000
@@ -106,6 +105,8 @@ CreateElementPages(HnswBuildState * buildstate)
{ {
Relation index = buildstate->index; Relation index = buildstate->index;
ForkNumber forkNum = buildstate->forkNum; ForkNumber forkNum = buildstate->forkNum;
int dimensions = buildstate->dimensions;
Size etupSize;
Size maxSize; Size maxSize;
HnswElementTuple etup; HnswElementTuple etup;
HnswNeighborTuple ntup; HnswNeighborTuple ntup;
@@ -117,9 +118,10 @@ CreateElementPages(HnswBuildState * buildstate)
/* Calculate sizes */ /* Calculate sizes */
maxSize = HNSW_MAX_SIZE; maxSize = HNSW_MAX_SIZE;
etupSize = HNSW_ELEMENT_TUPLE_SIZE(dimensions);
/* Allocate once */ /* Allocate once */
etup = palloc0(BLCKSZ); etup = palloc0(etupSize);
ntup = palloc0(BLCKSZ); ntup = palloc0(BLCKSZ);
/* Prepare first page */ /* Prepare first page */
@@ -131,14 +133,12 @@ CreateElementPages(HnswBuildState * buildstate)
foreach(lc, buildstate->elements) foreach(lc, buildstate->elements)
{ {
HnswElement element = lfirst(lc); HnswElement element = lfirst(lc);
Size etupSize;
Size ntupSize; Size ntupSize;
Size combinedSize; Size combinedSize;
HnswSetElementTuple(etup, element); HnswSetElementTuple(etup, element);
/* Calculate sizes */ /* Calculate sizes */
etupSize = HNSW_ELEMENT_TUPLE_SIZE(element->value);
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, buildstate->m); ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(element->level, buildstate->m);
combinedSize = etupSize + ntupSize + sizeof(ItemIdData); combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
@@ -273,15 +273,18 @@ InsertTuple(Relation index, Datum *values, HnswElement element, HnswBuildState *
int m = buildstate->m; int m = buildstate->m;
/* Detoast once for all calls */ /* Detoast once for all calls */
element->value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Normalize if needed */ /* Normalize if needed */
if (buildstate->normprocinfo != NULL) if (buildstate->normprocinfo != NULL)
{ {
if (!HnswNormValue(buildstate->normprocinfo, collation, &element->value, buildstate->normvec)) if (!HnswNormValue(buildstate->normprocinfo, collation, &value, buildstate->normvec))
return false; return false;
} }
/* Copy value to element so accessible outside of memory context */
memcpy(element->vec, DatumGetVector(value), VECTOR_SIZE(buildstate->dimensions));
/* Insert element in graph */ /* Insert element in graph */
HnswInsertElement(element, entryPoint, NULL, procinfo, collation, m, efConstruction, false); HnswInsertElement(element, entryPoint, NULL, procinfo, collation, m, efConstruction, false);
@@ -357,6 +360,7 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
/* Allocate necessary memory outside of memory context */ /* Allocate necessary memory outside of memory context */
element = HnswInitElement(tid, buildstate->m, buildstate->ml, buildstate->maxLevel); element = HnswInitElement(tid, buildstate->m, buildstate->ml, buildstate->maxLevel);
element->vec = palloc(VECTOR_SIZE(buildstate->dimensions));
/* Use memory context since detoast can allocate */ /* Use memory context since detoast can allocate */
oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx); oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
@@ -364,8 +368,9 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
/* Insert tuple */ /* Insert tuple */
inserted = InsertTuple(index, values, element, buildstate, &dup); inserted = InsertTuple(index, values, element, buildstate, &dup);
/* Switch memory context */ /* Reset memory context */
MemoryContextSwitchTo(oldCtx); MemoryContextSwitchTo(oldCtx);
MemoryContextReset(buildstate->tmpCtx);
/* Add outside memory context */ /* Add outside memory context */
if (dup != NULL) if (dup != NULL)
@@ -373,18 +378,9 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
/* Add to buildstate or free */ /* Add to buildstate or free */
if (inserted) if (inserted)
{
element->value = datumCopy(element->value, false, -1);
buildstate->elements = lappend(buildstate->elements, element); buildstate->elements = lappend(buildstate->elements, element);
}
else else
{
element->value = PointerGetDatum(NULL);
HnswFreeElement(element); HnswFreeElement(element);
}
/* Reset memory context */
MemoryContextReset(buildstate->tmpCtx);
} }
/* /*
@@ -399,7 +395,6 @@ HnswGetMaxInMemoryElements(int m, double ml, int dimensions)
elementSize += sizeof(HnswNeighborArray) * (avgLevel + 1); elementSize += sizeof(HnswNeighborArray) * (avgLevel + 1);
elementSize += sizeof(HnswCandidate) * (m * (avgLevel + 2)); elementSize += sizeof(HnswCandidate) * (m * (avgLevel + 2));
elementSize += sizeof(ItemPointerData); elementSize += sizeof(ItemPointerData);
/* TODO Handle non-vector types */
elementSize += VECTOR_SIZE(dimensions); elementSize += VECTOR_SIZE(dimensions);
return (maintenance_work_mem * 1024L) / elementSize; return (maintenance_work_mem * 1024L) / elementSize;
} }
@@ -417,10 +412,7 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
buildstate->m = HnswGetM(index); buildstate->m = HnswGetM(index);
buildstate->efConstruction = HnswGetEfConstruction(index); buildstate->efConstruction = HnswGetEfConstruction(index);
buildstate->dimensions = HnswGetDimensions(index); buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
if (buildstate->dimensions < 0)
buildstate->dimensions = TupleDescAttr(index->rd_att, 0)->atttypmod;
/* Require column to have dimensions to be indexed */ /* Require column to have dimensions to be indexed */
if (buildstate->dimensions < 0) if (buildstate->dimensions < 0)

View File

@@ -123,6 +123,7 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
Size minCombinedSize; Size minCombinedSize;
HnswElementTuple etup; HnswElementTuple etup;
BlockNumber currentPage = insertPage; BlockNumber currentPage = insertPage;
int dimensions = e->vec->dim;
HnswNeighborTuple ntup; HnswNeighborTuple ntup;
Buffer nbuf; Buffer nbuf;
Page npage; Page npage;
@@ -131,7 +132,7 @@ WriteNewElementPages(Relation index, HnswElement e, int m, BlockNumber insertPag
BlockNumber newInsertPage = InvalidBlockNumber; BlockNumber newInsertPage = InvalidBlockNumber;
/* Calculate sizes */ /* Calculate sizes */
etupSize = HNSW_ELEMENT_TUPLE_SIZE(e->value); etupSize = HNSW_ELEMENT_TUPLE_SIZE(dimensions);
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m); ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(e->level, m);
combinedSize = etupSize + ntupSize + sizeof(ItemIdData); combinedSize = etupSize + ntupSize + sizeof(ItemIdData);
maxSize = HNSW_MAX_SIZE; maxSize = HNSW_MAX_SIZE;
@@ -404,7 +405,7 @@ HnswAddDuplicate(Relation index, HnswElement element, HnswElement dup)
Buffer buf; Buffer buf;
Page page; Page page;
GenericXLogState *state; GenericXLogState *state;
Size etupSize = HNSW_ELEMENT_TUPLE_SIZE(dup->value); Size etupSize = HNSW_ELEMENT_TUPLE_SIZE(dup->vec->dim);
HnswElementTuple etup; HnswElementTuple etup;
int i; int i;
@@ -514,7 +515,7 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
/* Create an element */ /* Create an element */
element = HnswInitElement(heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m)); element = HnswInitElement(heap_tid, m, HnswGetMl(m), HnswGetMaxLevel(m));
element->value = value; element->vec = DatumGetVector(value);
/* Prevent concurrent inserts when likely updating entry point */ /* Prevent concurrent inserts when likely updating entry point */
if (entryPoint == NULL || element->level > entryPoint->level) if (entryPoint == NULL || element->level > entryPoint->level)

View File

@@ -4,7 +4,6 @@
#include "hnsw.h" #include "hnsw.h"
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "utils/datum.h"
#include "vector.h" #include "vector.h"
/* /*
@@ -35,20 +34,6 @@ HnswGetEfConstruction(Relation index)
return HNSW_DEFAULT_EF_CONSTRUCTION; return HNSW_DEFAULT_EF_CONSTRUCTION;
} }
/*
* Get the number of dimensions in the index
*/
int
HnswGetDimensions(Relation index)
{
HnswOptions *opts = (HnswOptions *) index->rd_options;
if (opts)
return opts->dimensions;
return HNSW_DEFAULT_DIMENSIONS;
}
/* /*
* Get proc * Get proc
*/ */
@@ -202,8 +187,7 @@ HnswFreeElement(HnswElement element)
{ {
HnswFreeNeighbors(element); HnswFreeNeighbors(element);
list_free_deep(element->heaptids); list_free_deep(element->heaptids);
if (DatumGetPointer(element->value)) pfree(element->vec);
pfree(DatumGetPointer(element->value));
pfree(element); pfree(element);
} }
@@ -230,7 +214,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno)
element->blkno = blkno; element->blkno = blkno;
element->offno = offno; element->offno = offno;
element->neighbors = NULL; element->neighbors = NULL;
element->value = PointerGetDatum(NULL); element->vec = NULL;
return element; return element;
} }
@@ -340,7 +324,7 @@ HnswSetElementTuple(HnswElementTuple etup, HnswElement element)
else else
ItemPointerSetInvalid(&etup->heaptids[i]); ItemPointerSetInvalid(&etup->heaptids[i]);
} }
memcpy(&etup->value, DatumGetPointer(element->value), VARSIZE_ANY(element->value)); memcpy(&etup->vec, element->vec, VECTOR_SIZE(element->vec->dim));
} }
/* /*
@@ -463,9 +447,8 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe
if (loadVec) if (loadVec)
{ {
Datum value = PointerGetDatum(&etup->value); element->vec = palloc(VECTOR_SIZE(etup->vec.dim));
memcpy(element->vec, &etup->vec, VECTOR_SIZE(etup->vec.dim));
element->value = datumCopy(value, false, -1);
} }
} }
@@ -493,7 +476,7 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
/* Calculate distance */ /* Calculate distance */
if (distance != NULL) if (distance != NULL)
*distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->value))); *distance = (float) DatumGetFloat8(FunctionCall2Coll(procinfo, collation, *q, PointerGetDatum(&etup->vec)));
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
@@ -504,7 +487,7 @@ HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation index,
static float static float
GetCandidateDistance(HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation) GetCandidateDistance(HnswCandidate * hc, Datum q, FmgrInfo *procinfo, Oid collation)
{ {
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, hc->element->value)); return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, q, PointerGetDatum(hc->element->vec)));
} }
/* /*
@@ -767,7 +750,7 @@ HnswGetDistance(HnswElement a, HnswElement b, int lc, FmgrInfo *procinfo, Oid co
} }
} }
return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, a->value, b->value)); return DatumGetFloat8(FunctionCall2Coll(procinfo, collation, PointerGetDatum(a->vec), PointerGetDatum(b->vec)));
} }
/* /*
@@ -894,7 +877,7 @@ HnswFindDuplicate(HnswElement e)
HnswCandidate *neighbor = &neighbors->items[i]; HnswCandidate *neighbor = &neighbors->items[i];
/* Exit early since ordered by distance */ /* Exit early since ordered by distance */
if (!datumIsEqual(e->value, neighbor->element->value, false, -1)) if (vector_cmp_internal(e->vec, neighbor->element->vec) != 0)
break; break;
/* Check for space */ /* Check for space */
@@ -947,13 +930,13 @@ HnswUpdateConnection(HnswElement element, HnswCandidate * hc, int m, int lc, int
/* Load elements on insert */ /* Load elements on insert */
if (index != NULL) if (index != NULL)
{ {
Datum q = hc->element->value; Datum q = PointerGetDatum(hc->element->vec);
for (int i = 0; i < currentNeighbors->length; i++) for (int i = 0; i < currentNeighbors->length; i++)
{ {
HnswCandidate *hc3 = &currentNeighbors->items[i]; HnswCandidate *hc3 = &currentNeighbors->items[i];
if (!DatumGetPointer(hc3->element->value)) if (hc3->element->vec == NULL)
HnswLoadElement(hc3->element, &hc3->distance, &q, index, procinfo, collation, true); HnswLoadElement(hc3->element, &hc3->distance, &q, index, procinfo, collation, true);
else else
hc3->distance = GetCandidateDistance(hc3, q, procinfo, collation); hc3->distance = GetCandidateDistance(hc3, q, procinfo, collation);
@@ -1034,7 +1017,7 @@ HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, F
List *w; List *w;
int level = element->level; int level = element->level;
int entryLevel; int entryLevel;
Datum q = element->value; Datum q = PointerGetDatum(element->vec);
HnswElement skipElement = existing ? element : NULL; HnswElement skipElement = existing ? element : NULL;
/* No neighbors if no entry point */ /* No neighbors if no entry point */

View File

@@ -93,7 +93,7 @@ RemoveHeapTids(HnswVacuumState * vacuumstate)
if (itemUpdated) if (itemUpdated)
{ {
Size etupSize = HNSW_ELEMENT_TUPLE_SIZE(PointerGetDatum(&etup->value)); Size etupSize = HNSW_ELEMENT_TUPLE_SIZE(etup->vec.dim);
/* Mark rest as invalid */ /* Mark rest as invalid */
for (int i = idx; i < HNSW_HEAPTIDS; i++) for (int i = idx; i < HNSW_HEAPTIDS; i++)
@@ -481,7 +481,6 @@ MarkDeleted(HnswVacuumState * vacuumstate)
HnswNeighborTuple ntup; HnswNeighborTuple ntup;
Size etupSize; Size etupSize;
Size ntupSize; Size ntupSize;
Datum value;
Buffer nbuf; Buffer nbuf;
Page npage; Page npage;
BlockNumber neighborPage; BlockNumber neighborPage;
@@ -505,11 +504,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
if (ItemPointerIsValid(&etup->heaptids[0])) if (ItemPointerIsValid(&etup->heaptids[0]))
continue; continue;
/* Get datum */
value = PointerGetDatum(&etup->value);
/* Calculate sizes */ /* Calculate sizes */
etupSize = HNSW_ELEMENT_TUPLE_SIZE(value); etupSize = HNSW_ELEMENT_TUPLE_SIZE(etup->vec.dim);
ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m); ntupSize = HNSW_NEIGHBOR_TUPLE_SIZE(etup->level, vacuumstate->m);
/* Get neighbor page */ /* Get neighbor page */
@@ -532,7 +528,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
/* Overwrite element */ /* Overwrite element */
etup->deleted = 1; etup->deleted = 1;
MemSet(&etup->value, 0, VARSIZE_ANY(value)); MemSet(&etup->vec.x, 0, etup->vec.dim * sizeof(float));
/* Overwrite neighbors */ /* Overwrite neighbors */
for (int i = 0; i < ntup->count; i++) for (int i = 0; i < ntup->count; i++)

View File

@@ -287,7 +287,7 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
#else #else
TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc); TupleTableSlot *slot = MakeSingleTupleTableSlot(buildstate->tupdesc);
#endif #endif
TupleDesc tupdesc = RelationGetDescr(index); TupleDesc tupdesc = IvfflatTupleDesc(index);
UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD); UpdateProgress(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_IVFFLAT_PHASE_LOAD);
@@ -339,6 +339,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
/* Set the start and insert pages */ /* Set the start and insert pages */
IvfflatUpdateList(index, buildstate->listInfo[i], insertPage, InvalidBlockNumber, startPage, forkNum); IvfflatUpdateList(index, buildstate->listInfo[i], insertPage, InvalidBlockNumber, startPage, forkNum);
} }
FreeTupleDesc(tupdesc);
} }
/* /*

View File

@@ -287,6 +287,7 @@ void IvfflatInitPage(Buffer buf, Page page);
void IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state); void IvfflatInitRegisterPage(Relation index, Buffer *buf, Page *page, GenericXLogState **state);
void IvfflatInit(void); void IvfflatInit(void);
PGDLLEXPORT void IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc); PGDLLEXPORT void IvfflatParallelBuildMain(dsm_segment *seg, shm_toc *toc);
TupleDesc IvfflatTupleDesc(Relation index);
/* Index access methods */ /* Index access methods */
IndexBuildResult *ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo); IndexBuildResult *ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo);

View File

@@ -94,7 +94,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, R
originalInsertPage = insertPage; originalInsertPage = insertPage;
/* Form tuple */ /* Form tuple */
itup = index_form_tuple(RelationGetDescr(index), &value, isnull); itup = index_form_tuple(IvfflatTupleDesc(index), &value, isnull);
itup->t_tid = *heap_tid; itup->t_tid = *heap_tid;
/* Get tuple size */ /* Get tuple size */

View File

@@ -4,6 +4,10 @@
#include "storage/bufmgr.h" #include "storage/bufmgr.h"
#include "vector.h" #include "vector.h"
#if PG_VERSION_NUM < 130000
#define TYPSTORAGE_PLAIN 'p'
#endif
/* /*
* Allocate a vector array * Allocate a vector array
*/ */
@@ -238,3 +242,17 @@ IvfflatUpdateList(Relation index, ListInfo listInfo,
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(buf);
} }
} }
/*
* Get the tuple descriptor
*/
TupleDesc
IvfflatTupleDesc(Relation index)
{
TupleDesc tupdesc = CreateTupleDescCopyConstr(RelationGetDescr(index));
/* Prevent compression */
TupleDescAttr(tupdesc, 0)->attstorage = TYPSTORAGE_PLAIN;
return tupdesc;
}

View File

@@ -1,113 +0,0 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More;
my $node;
my @queries = ();
my @expected;
my $limit = 20;
sub test_recall
{
my ($min, $operator) = @_;
my $correct = 0;
my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries)
{
my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SELECT i FROM tst ORDER BY v $operator '$queries[$i]' LIMIT $limit;
));
my @actual_ids = split("\n", $actual);
my %actual_set = map { $_ => 1 } @actual_ids;
my @expected_ids = split("\n", $expected[$i]);
foreach (@expected_ids)
{
if (exists($actual_set{$_}))
{
$correct++;
}
$total++;
}
}
cmp_ok($correct / $total, ">=", $min, $operator);
}
# Initialize node
$node = get_new_node('node');
$node->init;
$node->start;
# Create table
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (i int4, v float4[3]);");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT i, ARRAY[random(), random(), random()] FROM generate_series(1, 10000) i;"
);
$node->safe_psql("postgres", qq(
CREATE FUNCTION float4_l2_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN l2_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION float4_l2_squared_distance(float4[], float4[]) RETURNS float8
AS 'BEGIN RETURN vector_l2_squared_distance(\$1::vector, \$2::vector); END;'
LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
CREATE OPERATOR <-> (
LEFTARG = float4[], RIGHTARG = float4[], PROCEDURE = float4_l2_distance,
COMMUTATOR = '<->'
);
CREATE OPERATOR CLASS float4_l2_ops
FOR TYPE float4[] USING hnsw AS
OPERATOR 1 <-> (float4[], float4[]) FOR ORDER BY float_ops,
FUNCTION 1 float4_l2_squared_distance(float4[], float4[]);
));
# Generate queries
for (1 .. 20)
{
my $r1 = rand();
my $r2 = rand();
my $r3 = rand();
push(@queries, "{$r1,$r2,$r3}");
}
# Check each index type
my @operators = ("<->");
my @opclasses = ("float4_l2_ops");
for my $i (0 .. $#operators)
{
my $operator = $operators[$i];
my $opclass = $opclasses[$i];
# Get exact results
@expected = ();
foreach (@queries)
{
my $res = $node->safe_psql("postgres", "SELECT i FROM tst ORDER BY v $operator '$_' LIMIT $limit;");
push(@expected, $res);
}
# Add index
$node->safe_psql("postgres", "CREATE INDEX ON tst USING hnsw (v $opclass) WITH (dimensions = 3);");
my $min = $operator eq "<#>" ? 0.80 : 0.99;
test_recall($min, $operator);
}
done_testing();