Test sparsevec vacuum recall

Co-authored-by: Heikki Linnakangas <heikki.linnakangas@iki.fi>

.github/workflows/build.yml

@@ -1,15 +1,31 @@
 name: build
 on: [push, pull_request]
 jobs:
-  valgrind:
+  ubuntu:
+    runs-on: ${{ matrix.os }}
     if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - postgres: 14
+            os: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: ankane/setup-postgres-valgrind@v1
+      - uses: ankane/setup-postgres@v1
         with:
-          postgres-version: 16
-          check-ub: yes
-      - run: make OPTFLAGS=""
-      - run: sudo --preserve-env=PG_CONFIG make install
-      - run: make prove_installcheck PROVE_TESTS=test/t/031_hnsw_sparsevec_vacuum_recall.pl
+          postgres-version: ${{ matrix.postgres }}
+          dev-files: true
+      - run: make
+        env:
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
+      - run: |
+          export PG_CONFIG=`which pg_config`
+          sudo --preserve-env=PG_CONFIG make install
+      - run: make installcheck
+      - if: ${{ failure() }}
+        run: cat regression.diffs
+      - run: |
+          sudo apt-get update
+          sudo apt-get install libipc-run-perl
+      - run: for run in {1..100}; do make prove_installcheck PROVE_TESTS=test/t/031_hnsw_sparsevec_vacuum_recall.pl || break; done

src/sparsevec.c

@@ -683,7 +683,7 @@ SparsevecL2SquaredDistance(SparseVector * a, SparseVector * b)
 
 			if (ai == bi)
 			{
-				double		diff = ax[i] - bx[j];
+				float		diff = ax[i] - bx[j];
 
 				distance += diff * diff;
 			}
@@ -721,7 +721,7 @@ sparsevec_l2_distance(PG_FUNCTION_ARGS)
 
 	CheckDims(a, b);
 
-	PG_RETURN_FLOAT8(sqrt(SparsevecL2SquaredDistance(a, b)));
+	PG_RETURN_FLOAT8(sqrt((double) SparsevecL2SquaredDistance(a, b)));
 }
 
 /*
@@ -737,7 +737,7 @@ sparsevec_l2_squared_distance(PG_FUNCTION_ARGS)
 
 	CheckDims(a, b);
 
-	PG_RETURN_FLOAT8(SparsevecL2SquaredDistance(a, b));
+	PG_RETURN_FLOAT8((double) SparsevecL2SquaredDistance(a, b));
 }
 
 /*
@@ -788,7 +788,7 @@ sparsevec_inner_product(PG_FUNCTION_ARGS)
 
 	CheckDims(a, b);
 
-	PG_RETURN_FLOAT8(SparsevecInnerProduct(a, b));
+	PG_RETURN_FLOAT8((double) SparsevecInnerProduct(a, b));
 }
 
 /*
@@ -803,7 +803,7 @@ sparsevec_negative_inner_product(PG_FUNCTION_ARGS)
 
 	CheckDims(a, b);
 
-	PG_RETURN_FLOAT8(-SparsevecInnerProduct(a, b));
+	PG_RETURN_FLOAT8((double) -SparsevecInnerProduct(a, b));
 }
 
 /*
@@ -862,7 +862,7 @@ sparsevec_l1_distance(PG_FUNCTION_ARGS)
 	SparseVector *b = PG_GETARG_SPARSEVEC_P(1);
 	float	   *ax = SPARSEVEC_VALUES(a);
 	float	   *bx = SPARSEVEC_VALUES(b);
-	double		distance = 0.0;
+	float		distance = 0.0;
 	int			bpos = 0;
 
 	CheckDims(a, b);
@@ -897,7 +897,7 @@ sparsevec_l1_distance(PG_FUNCTION_ARGS)
 	for (int j = bpos; j < b->nnz; j++)
 		distance += fabsf(bx[j]);
 
-	PG_RETURN_FLOAT8(distance);
+	PG_RETURN_FLOAT8((double) distance);
 }
 
 /*

src/sparsevec.h

@@ -3,25 +3,38 @@
 
 #include "fmgr.h"
 
-#define SPARSEVEC_MAX_DIM 100000
+#define SPARSEVEC_MAX_DIM 1000000
 #define SPARSEVEC_MAX_NNZ 16000
 
-/* Ensure values are aligned */
-#define SPARSEVEC_SIZE(_nnz)		(offsetof(SparseVector, indices) + MAXALIGN((_nnz) * sizeof(int32)) + (_nnz * sizeof(float)))
-#define SPARSEVEC_VALUES(x)		((float *) (((char *) (x)) + offsetof(SparseVector, indices) + MAXALIGN((x)->nnz * sizeof(int32))))
 #define DatumGetSparseVector(x)		((SparseVector *) PG_DETOAST_DATUM(x))
 #define PG_GETARG_SPARSEVEC_P(x)	DatumGetSparseVector(PG_GETARG_DATUM(x))
 #define PG_RETURN_SPARSEVEC_P(x)	PG_RETURN_POINTER(x)
 
+/* Indices are always sorted */
+/* Values come after indices */
 typedef struct SparseVector
 {
 	int32		vl_len_;		/* varlena header (do not touch directly!) */
 	int32		dim;			/* number of dimensions */
-	int32		nnz;
+	int32		nnz;			/* number of non-zero elements */
 	int32		unused;
 	int32		indices[FLEXIBLE_ARRAY_MEMBER];
 }			SparseVector;
 
+/* Use functions instead of macros to avoid double evaluation */
+
+static inline Size
+SPARSEVEC_SIZE(int nnz)
+{
+	return offsetof(SparseVector, indices) + (nnz * sizeof(int32)) + (nnz * sizeof(float));
+}
+
+static inline float *
+SPARSEVEC_VALUES(SparseVector * x)
+{
+	return (float *) (((char *) x) + offsetof(SparseVector, indices) + (x->nnz * sizeof(int32)));
+}
+
 SparseVector *InitSparseVector(int dim, int nnz);
 
 #endif

test/expected/halfvec_functions.out

@@ -176,6 +176,18 @@ SELECT l2_norm('[0,1]'::halfvec);
        1
 (1 row)
 
+SELECT l2_norm('[0,0]'::halfvec);
+ l2_norm 
+---------
+       0
+(1 row)
+
+SELECT l2_norm('[2]'::halfvec);
+ l2_norm 
+---------
+       2
+(1 row)
+
 SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
  l2_distance 
 -------------

test/expected/sparsevec_functions.out

@@ -136,6 +136,18 @@ SELECT l2_norm('{1:3e37,2:4e37}/2'::sparsevec)::real;
    5e+37
 (1 row)
 
+SELECT l2_norm('{}/2'::sparsevec);
+ l2_norm 
+---------
+       0
+(1 row)
+
+SELECT l2_norm('{1:2}/1'::sparsevec);
+ l2_norm 
+---------
+       2
+(1 row)
+
 SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
  l2_distance 
 -------------
@@ -154,6 +166,12 @@ SELECT l2_distance('{2:4}/2'::sparsevec, '{1:3}/2');
            5
 (1 row)
 
+SELECT l2_distance('{1:3,2:4}/2'::sparsevec, '{}/2');
+ l2_distance 
+-------------
+           5
+(1 row)
+
 SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
  l2_distance 
 -------------
@@ -174,6 +192,36 @@ SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
 
 SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
 ERROR:  different sparsevec dimensions 2 and 1
+SELECT inner_product('{1:1,3:3}/4'::sparsevec, '{2:2,4:4}/4');
+ inner_product 
+---------------
+             0
+(1 row)
+
+SELECT inner_product('{2:2,4:4}/4'::sparsevec, '{1:1,3:3}/4');
+ inner_product 
+---------------
+             0
+(1 row)
+
+SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
+ inner_product 
+---------------
+            18
+(1 row)
+
+SELECT inner_product('{1:1}/2'::sparsevec, '{}/2');
+ inner_product 
+---------------
+             0
+(1 row)
+
+SELECT inner_product('{}/2'::sparsevec, '{1:1}/2');
+ inner_product 
+---------------
+             0
+(1 row)
+
 SELECT inner_product('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
  inner_product 
 ---------------
@@ -228,6 +276,12 @@ SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
                1
 (1 row)
 
+SELECT cosine_distance('{2:2}/2'::sparsevec, '{1:2}/2');
+ cosine_distance 
+-----------------
+               1
+(1 row)
+
 SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
 ERROR:  different sparsevec dimensions 2 and 1
 SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1.1,2:1.1}/2');
@@ -292,6 +346,12 @@ SELECT l1_distance('{1:1,3:3,5:5,7:7,9:9}/9'::sparsevec, '{2:2,4:4,6:6,8:8}/9');
           45
 (1 row)
 
+SELECT '{}/2'::sparsevec <+> '{1:3,2:4}/2';
+ ?column? 
+----------
+        7
+(1 row)
+
 SELECT l2_normalize('{1:3,2:4}/2'::sparsevec);
   l2_normalize   
 -----------------

test/expected/sparsevec_input.out

@@ -187,9 +187,9 @@ SELECT '{}/-1'::sparsevec;
 ERROR:  sparsevec must have at least 1 dimension
 LINE 1: SELECT '{}/-1'::sparsevec;
                ^
-SELECT '{}/100001'::sparsevec;
-ERROR:  sparsevec cannot have more than 100000 dimensions
-LINE 1: SELECT '{}/100001'::sparsevec;
+SELECT '{}/1000001'::sparsevec;
+ERROR:  sparsevec cannot have more than 1000000 dimensions
+LINE 1: SELECT '{}/1000001'::sparsevec;
                ^
 SELECT '{0:1}/1'::sparsevec;
 ERROR:  index "0" is out of range for type sparsevec
@@ -219,7 +219,7 @@ SELECT '{}/3'::sparsevec(0);
 ERROR:  dimensions for type sparsevec must be at least 1
 LINE 1: SELECT '{}/3'::sparsevec(0);
                        ^
-SELECT '{}/3'::sparsevec(100001);
-ERROR:  dimensions for type sparsevec cannot exceed 100000
-LINE 1: SELECT '{}/3'::sparsevec(100001);
+SELECT '{}/3'::sparsevec(1000001);
+ERROR:  dimensions for type sparsevec cannot exceed 1000000
+LINE 1: SELECT '{}/3'::sparsevec(1000001);
                        ^

test/expected/vector_functions.out

@@ -182,6 +182,18 @@ SELECT vector_norm('[3e37,4e37]')::real;
        5e+37
 (1 row)
 
+SELECT vector_norm('[0,0]');
+ vector_norm 
+-------------
+           0
+(1 row)
+
+SELECT vector_norm('[2]');
+ vector_norm 
+-------------
+           2
+(1 row)
+
 SELECT l2_distance('[0,0]'::vector, '[3,4]');
  l2_distance 
 -------------

test/sql/halfvec_functions.sql

@@ -40,6 +40,8 @@ SELECT vector_dims('[1,2,3]'::halfvec);
 SELECT round(l2_norm('[1,1]'::halfvec)::numeric, 5);
 SELECT l2_norm('[3,4]'::halfvec);
 SELECT l2_norm('[0,1]'::halfvec);
+SELECT l2_norm('[0,0]'::halfvec);
+SELECT l2_norm('[2]'::halfvec);
 
 SELECT l2_distance('[0,0]'::halfvec, '[3,4]');
 SELECT l2_distance('[0,0]'::halfvec, '[0,1]');

test/sql/sparsevec_functions.sql

@@ -23,15 +23,23 @@ SELECT round(l2_norm('{1:1,2:1}/2'::sparsevec)::numeric, 5);
 SELECT l2_norm('{1:3,2:4}/2'::sparsevec);
 SELECT l2_norm('{2:1}/2'::sparsevec);
 SELECT l2_norm('{1:3e37,2:4e37}/2'::sparsevec)::real;
+SELECT l2_norm('{}/2'::sparsevec);
+SELECT l2_norm('{1:2}/1'::sparsevec);
 
 SELECT l2_distance('{}/2'::sparsevec, '{1:3,2:4}/2');
 SELECT l2_distance('{1:3}/2'::sparsevec, '{2:4}/2');
 SELECT l2_distance('{2:4}/2'::sparsevec, '{1:3}/2');
+SELECT l2_distance('{1:3,2:4}/2'::sparsevec, '{}/2');
 SELECT l2_distance('{}/2'::sparsevec, '{2:1}/2');
 SELECT '{}/2'::sparsevec <-> '{1:3,2:4}/2';
 
 SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:2,2:4}/2');
 SELECT inner_product('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
+SELECT inner_product('{1:1,3:3}/4'::sparsevec, '{2:2,4:4}/4');
+SELECT inner_product('{2:2,4:4}/4'::sparsevec, '{1:1,3:3}/4');
+SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
+SELECT inner_product('{1:1}/2'::sparsevec, '{}/2');
+SELECT inner_product('{}/2'::sparsevec, '{1:1}/2');
 SELECT inner_product('{1:3e38}/1'::sparsevec, '{1:3e38}/1');
 SELECT inner_product('{1:1,3:3,5:5}/5'::sparsevec, '{2:4,3:6,4:8}/5');
 SELECT '{1:1,2:2}/2'::sparsevec <#> '{1:3,2:4}/2';
@@ -42,6 +50,7 @@ SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1,2:1}/2');
 SELECT cosine_distance('{1:1}/2'::sparsevec, '{2:2}/2');
 SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1,2:-1}/2');
 SELECT cosine_distance('{1:2}/2'::sparsevec, '{2:2}/2');
+SELECT cosine_distance('{2:2}/2'::sparsevec, '{1:2}/2');
 SELECT cosine_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
 SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:1.1,2:1.1}/2');
 SELECT cosine_distance('{1:1,2:1}/2'::sparsevec, '{1:-1.1,2:-1.1}/2');
@@ -55,6 +64,7 @@ SELECT l1_distance('{1:1,2:2}/2'::sparsevec, '{1:3}/1');
 SELECT l1_distance('{1:3e38}/1'::sparsevec, '{1:-3e38}/1');
 SELECT l1_distance('{1:1,3:3,5:5,7:7}/8'::sparsevec, '{2:2,4:4,6:6,8:8}/8');
 SELECT l1_distance('{1:1,3:3,5:5,7:7,9:9}/9'::sparsevec, '{2:2,4:4,6:6,8:8}/9');
+SELECT '{}/2'::sparsevec <+> '{1:3,2:4}/2';
 
 SELECT l2_normalize('{1:3,2:4}/2'::sparsevec);
 SELECT l2_normalize('{1:3}/2'::sparsevec);

test/sql/sparsevec_input.sql

@@ -38,7 +38,7 @@ SELECT '{1:1,1:1}/2'::sparsevec;
 SELECT '{1:1,2:1,1:1}/2'::sparsevec;
 SELECT '{}/5'::sparsevec;
 SELECT '{}/-1'::sparsevec;
-SELECT '{}/100001'::sparsevec;
+SELECT '{}/1000001'::sparsevec;
 SELECT '{0:1}/1'::sparsevec;
 SELECT '{2:1}/1'::sparsevec;
 
@@ -47,4 +47,4 @@ SELECT '{}/3'::sparsevec(2);
 SELECT '{}/3'::sparsevec(3, 2);
 SELECT '{}/3'::sparsevec('a');
 SELECT '{}/3'::sparsevec(0);
-SELECT '{}/3'::sparsevec(100001);
+SELECT '{}/3'::sparsevec(1000001);

test/sql/vector_functions.sql

@@ -41,6 +41,8 @@ SELECT round(vector_norm('[1,1]')::numeric, 5);
 SELECT vector_norm('[3,4]');
 SELECT vector_norm('[0,1]');
 SELECT vector_norm('[3e37,4e37]')::real;
+SELECT vector_norm('[0,0]');
+SELECT vector_norm('[2]');
 
 SELECT l2_distance('[0,0]'::vector, '[3,4]');
 SELECT l2_distance('[0,0]'::vector, '[0,1]');