Compare commits

..

2 Commits

Author SHA1 Message Date
Andrew Kane
1184ad9fd3 Try double 2023-10-16 17:00:48 -07:00
Andrew Kane
1ed278c82c Debug failure 2023-10-16 16:57:48 -07:00
17 changed files with 67 additions and 99 deletions

View File

@@ -77,23 +77,5 @@ jobs:
nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall
shell: cmd
i386:
runs-on: ubuntu-latest
container:
image: debian:11
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-13 postgresql-server-dev-13 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- if: ${{ failure() }}
run: cat regression.diffs

View File

@@ -268,15 +268,15 @@ CREATE OPERATOR CLASS vector_ip_ops
OPERATOR 1 <#> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 3 vector_spherical_distance(vector, vector),
FUNCTION 6 normalize_l2(vector);
FUNCTION 4 vector_norm(vector);
CREATE OPERATOR CLASS vector_cosine_ops
FOR TYPE vector USING ivfflat AS
OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 2 vector_norm(vector),
FUNCTION 3 vector_spherical_distance(vector, vector),
FUNCTION 5 normalize_l2(vector),
FUNCTION 6 normalize_l2(vector);
FUNCTION 4 vector_norm(vector);
CREATE OPERATOR CLASS vector_l2_ops
FOR TYPE vector USING hnsw AS
@@ -292,4 +292,4 @@ CREATE OPERATOR CLASS vector_cosine_ops
FOR TYPE vector USING hnsw AS
OPERATOR 1 <=> (vector, vector) FOR ORDER BY float_ops,
FUNCTION 1 vector_negative_inner_product(vector, vector),
FUNCTION 3 normalize_l2(vector);
FUNCTION 2 vector_norm(vector);

View File

@@ -167,7 +167,7 @@ hnswhandler(PG_FUNCTION_ARGS)
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 3;
amroutine->amsupport = 2;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif

View File

@@ -19,7 +19,6 @@
/* Support functions */
#define HNSW_DISTANCE_PROC 1
#define HNSW_NORM_PROC 2
#define HNSW_NORMALIZE_PROC 3
#define HNSW_VERSION 1
#define HNSW_MAGIC_NUMBER 0xA953A953
@@ -148,7 +147,6 @@ typedef struct HnswBuildState
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
/* Variables */
@@ -222,7 +220,6 @@ typedef struct HnswScanOpaqueData
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
} HnswScanOpaqueData;
@@ -258,7 +255,7 @@ typedef struct HnswVacuumState
int HnswGetM(Relation index);
int HnswGetEfConstruction(Relation index);
FmgrInfo *HnswOptionalProcInfo(Relation rel, uint16 procnum);
void HnswNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation, Datum *value, Vector * result);
bool HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
void HnswCommitBuffer(Buffer buf, GenericXLogState *state);
Buffer HnswNewBuffer(Relation index, ForkNumber forkNum);
void HnswInitPage(Buffer buf, Page page);

View File

@@ -278,7 +278,11 @@ InsertTuple(Relation index, Datum *values, HnswElement element, HnswBuildState *
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Normalize if needed */
HnswNormValue(buildstate->normprocinfo, buildstate->normalizeprocinfo, collation, &value, buildstate->normvec);
if (buildstate->normprocinfo != NULL)
{
if (!HnswNormValue(buildstate->normprocinfo, collation, &value, buildstate->normvec))
return false;
}
/* Copy value to element so accessible outside of memory context */
memcpy(element->vec, DatumGetVector(value), VECTOR_SIZE(buildstate->dimensions));
@@ -409,7 +413,6 @@ InitBuildState(HnswBuildState * buildstate, Relation heap, Relation index, Index
/* Get support functions */
buildstate->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
buildstate->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
buildstate->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
buildstate->collation = index->rd_indcollation[0];
buildstate->elements = NIL;

View File

@@ -417,7 +417,6 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
{
Datum value;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
HnswElement entryPoint;
HnswElement element;
int m = HnswGetM(index);
@@ -433,8 +432,11 @@ HnswInsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_ti
/* Normalize if needed */
normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
HnswNormValue(normprocinfo, normalizeprocinfo, collation, &value, NULL);
if (normprocinfo != NULL)
{
if (!HnswNormValue(normprocinfo, collation, &value, NULL))
return false;
}
/* Create an element */
element = HnswInitElement(heap_tid, m, ml, HnswGetMaxLevel(m));

View File

@@ -78,7 +78,6 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
/* Set support functions */
so->procinfo = index_getprocinfo(index, 1, HNSW_DISTANCE_PROC);
so->normprocinfo = HnswOptionalProcInfo(index, HNSW_NORM_PROC);
so->normalizeprocinfo = HnswOptionalProcInfo(index, HNSW_NORMALIZE_PROC);
so->collation = index->rd_indcollation[0];
scan->opaque = so;
@@ -141,7 +140,8 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Fine if normalization fails */
HnswNormValue(so->normprocinfo, so->normalizeprocinfo, so->collation, &value, NULL);
if (so->normprocinfo != NULL)
HnswNormValue(so->normprocinfo, so->collation, &value, NULL);
}
GetScanItems(scan, value);

View File

@@ -47,26 +47,17 @@ HnswOptionalProcInfo(Relation rel, uint16 procnum)
}
/*
* Normalize a vector
* Divide by the norm
*
* Returns false if value should not be indexed
*
* The caller needs to free the pointer stored in value
* if it's different than the original value
*/
void
HnswNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation, Datum *value, Vector * result)
bool
HnswNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
{
double norm;
if (normalizeprocinfo != NULL)
{
*value = FunctionCall1Coll(normalizeprocinfo, collation, *value);
return;
}
if (procinfo == NULL)
return;
norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
if (norm > 0)
{
@@ -79,7 +70,11 @@ HnswNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation, Da
result->x[i] = v->x[i] / norm;
*value = PointerGetDatum(result);
return true;
}
return false;
}
/*

View File

@@ -75,7 +75,11 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
* Normalize with KMEANS_NORM_PROC since spherical distance function
* expects unit vectors
*/
IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->kmeansnormalizeprocinfo, buildstate->collation, &value, buildstate->normvec);
if (buildstate->kmeansnormprocinfo != NULL)
{
if (!IvfflatNormValue(buildstate->kmeansnormprocinfo, buildstate->collation, &value, buildstate->normvec))
return;
}
if (samples->length < targsamples)
{
@@ -172,7 +176,11 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
/* Normalize if needed */
IvfflatNormValue(buildstate->normprocinfo, buildstate->normalizeprocinfo, buildstate->collation, &value, buildstate->normvec);
if (buildstate->normprocinfo != NULL)
{
if (!IvfflatNormValue(buildstate->normprocinfo, buildstate->collation, &value, buildstate->normvec))
return;
}
/* Find the list that minimizes the distance */
for (int i = 0; i < centers->length; i++)
@@ -360,8 +368,6 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
buildstate->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
buildstate->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
buildstate->kmeansnormprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
buildstate->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
buildstate->kmeansnormalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORMALIZE_PROC);
buildstate->collation = index->rd_indcollation[0];
/* Require more than one dimension for spherical k-means */

View File

@@ -194,7 +194,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);
amroutine->amstrategies = 0;
amroutine->amsupport = 6;
amroutine->amsupport = 4;
#if PG_VERSION_NUM >= 130000
amroutine->amoptsprocnum = 0;
#endif

View File

@@ -31,8 +31,6 @@
#define IVFFLAT_NORM_PROC 2
#define IVFFLAT_KMEANS_DISTANCE_PROC 3
#define IVFFLAT_KMEANS_NORM_PROC 4
#define IVFFLAT_NORMALIZE_PROC 5
#define IVFFLAT_KMEANS_NORMALIZE_PROC 6
#define IVFFLAT_VERSION 1
#define IVFFLAT_MAGIC_NUMBER 0x14FF1A7
@@ -174,8 +172,6 @@ typedef struct IvfflatBuildState
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *kmeansnormprocinfo;
FmgrInfo *normalizeprocinfo;
FmgrInfo *kmeansnormalizeprocinfo;
Oid collation;
/* Variables */
@@ -257,7 +253,6 @@ typedef struct IvfflatScanOpaqueData
/* Support functions */
FmgrInfo *procinfo;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Oid collation;
/* Lists */
@@ -278,7 +273,7 @@ void VectorArrayFree(VectorArray arr);
void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);
void IvfflatNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation, Datum *value, Vector * result);
bool IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result);
int IvfflatGetLists(Relation index);
void IvfflatUpdateList(Relation index, ListInfo listInfo, BlockNumber insertPage, BlockNumber originalInsertPage, BlockNumber startPage, ForkNumber forkNum);
void IvfflatCommitBuffer(Buffer buf, GenericXLogState *state);

View File

@@ -68,7 +68,6 @@ InsertTuple(Relation rel, Datum *values, bool *isnull, ItemPointer heap_tid, Rel
IndexTuple itup;
Datum value;
FmgrInfo *normprocinfo;
FmgrInfo *normalizeprocinfo;
Buffer buf;
Page page;
GenericXLogState *state;
@@ -82,8 +81,11 @@ InsertTuple(Relation rel, Datum *values, bool *isnull, ItemPointer heap_tid, Rel
/* Normalize if needed */
normprocinfo = IvfflatOptionalProcInfo(rel, IVFFLAT_NORM_PROC);
normalizeprocinfo = IvfflatOptionalProcInfo(rel, IVFFLAT_NORMALIZE_PROC);
IvfflatNormValue(normprocinfo, normalizeprocinfo, rel->rd_indcollation[0], &value, NULL);
if (normprocinfo != NULL)
{
if (!IvfflatNormValue(normprocinfo, rel->rd_indcollation[0], &value, NULL))
return;
}
/* Find the insert page - sets the page and list info */
FindInsertPage(rel, values, &insertPage, &listInfo);

View File

@@ -232,7 +232,6 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
/* Set support functions */
so->procinfo = index_getprocinfo(index, 1, IVFFLAT_DISTANCE_PROC);
so->normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORM_PROC);
so->normalizeprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_NORMALIZE_PROC);
so->collation = index->rd_indcollation[0];
/* Create tuple description for sorting */
@@ -320,7 +319,8 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir)
Assert(!VARATT_IS_EXTENDED(DatumGetPointer(value)));
/* Fine if normalization fails */
IvfflatNormValue(so->normprocinfo, so->normalizeprocinfo, so->collation, &value, NULL);
if (so->normprocinfo != NULL)
IvfflatNormValue(so->normprocinfo, so->collation, &value, NULL);
}
IvfflatBench("GetScanLists", GetScanLists(scan, value));

View File

@@ -66,26 +66,17 @@ IvfflatOptionalProcInfo(Relation rel, uint16 procnum)
}
/*
* Normalize a vector
* Divide by the norm
*
* Returns false if value should not be indexed
*
* The caller needs to free the pointer stored in value
* if it's different than the original value
*/
void
IvfflatNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation, Datum *value, Vector * result)
bool
IvfflatNormValue(FmgrInfo *procinfo, Oid collation, Datum *value, Vector * result)
{
double norm;
if (normalizeprocinfo != NULL)
{
*value = FunctionCall1Coll(normalizeprocinfo, collation, *value);
return;
}
if (procinfo == NULL)
return;
norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
double norm = DatumGetFloat8(FunctionCall1Coll(procinfo, collation, *value));
if (norm > 0)
{
@@ -98,7 +89,11 @@ IvfflatNormValue(FmgrInfo *procinfo, FmgrInfo *normalizeprocinfo, Oid collation,
result->x[i] = v->x[i] / norm;
*value = PointerGetDatum(result);
return true;
}
return false;
}
/*

View File

@@ -755,12 +755,6 @@ normalize_l2(PG_FUNCTION_ARGS)
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
double norm = 0.0;
#ifdef _MSC_VER
/* Fix precision on Windows */
double normf;
#else
float normf;
#endif
Vector *result;
float *rx;
@@ -772,13 +766,12 @@ normalize_l2(PG_FUNCTION_ARGS)
norm += (double) ax[i] * (double) ax[i];
norm = sqrt(norm);
normf = norm;
if (normf > 0)
if (norm > 0)
{
/* Auto-vectorized */
for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] / normf;
rx[i] = ax[i] / norm;
/* Check for overflow */
for (int i = 0, imax = a->dim; i < imax; i++)

View File

@@ -9,19 +9,18 @@ SELECT * FROM t ORDER BY val <=> '[3,3,3]';
[1,1,1]
[1,2,3]
[1,2,4]
[0,0,0]
(4 rows)
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
4
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
4
3
(1 row)
DROP TABLE t;

View File

@@ -9,19 +9,18 @@ SELECT * FROM t ORDER BY val <=> '[3,3,3]';
[1,1,1]
[1,2,3]
[1,2,4]
[0,0,0]
(4 rows)
(3 rows)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> '[0,0,0]') t2;
count
-------
4
3
(1 row)
SELECT COUNT(*) FROM (SELECT * FROM t ORDER BY val <=> (SELECT NULL::vector)) t2;
count
-------
4
3
(1 row)
DROP TABLE t;