Updated number of parallel workers for IVFFlat index builds

This reverts commit 14b278dec9.

.github/workflows/build.yml

@@ -28,8 +28,10 @@ jobs:
           dev-files: true
       - run: make
         env:
-          PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
-      - run: sudo make install
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
+      - run: |
+          export PG_CONFIG=`which pg_config`
+          sudo --preserve-env=PG_CONFIG make install
       - run: make installcheck
       - if: ${{ failure() }}
         run: cat regression.diffs
@@ -37,3 +39,73 @@ jobs:
           sudo apt-get update
           sudo apt-get install libipc-run-perl
       - run: make prove_installcheck
+  mac:
+    runs-on: macos-latest
+    if: ${{ !startsWith(github.ref_name, 'windows') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ankane/setup-postgres@v1
+        with:
+          postgres-version: 14
+      - run: make
+        env:
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter
+      - run: make install
+      - run: make installcheck
+      - if: ${{ failure() }}
+        run: cat regression.diffs
+      - run: |
+          brew install cpanm
+          cpanm --notest IPC::Run
+          wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_10.tar.gz
+          tar xf REL_14_10.tar.gz
+      - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_10/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
+      - run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make PG_CFLAGS="-DUSE_ASSERT_CHECKING"
+  windows:
+    runs-on: windows-latest
+    if: ${{ !startsWith(github.ref_name, 'mac') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ankane/setup-postgres@v1
+        with:
+          postgres-version: 14
+      - run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
+          nmake /NOLOGO /F Makefile.win && ^
+          nmake /NOLOGO /F Makefile.win install && ^
+          nmake /NOLOGO /F Makefile.win installcheck && ^
+          nmake /NOLOGO /F Makefile.win clean && ^
+          nmake /NOLOGO /F Makefile.win uninstall
+        shell: cmd
+  i386:
+    if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
+    runs-on: ubuntu-latest
+    container:
+      image: debian:12
+      options: --platform linux/386
+    steps:
+      - run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-15 postgresql-server-dev-15 sudo
+      - run: service postgresql start
+      - run: |
+          git clone https://github.com/${{ github.repository }}.git pgvector
+          cd pgvector
+          git fetch origin ${{ github.ref }}
+          git reset --hard FETCH_HEAD
+          make
+          make install
+          chown -R postgres .
+          sudo -u postgres make installcheck
+          sudo -u postgres make prove_installcheck
+        env:
+          PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
+  valgrind:
+    if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ankane/setup-postgres-valgrind@v1
+        with:
+          postgres-version: 16
+      - run: make
+      - run: sudo --preserve-env=PG_CONFIG make install
+      - run: make installcheck

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## 0.6.1 (unreleased)
 
 - Fixed error with `ANALYZE` and vectors with different dimensions
+- Fixed error with `shared_preload_libraries`
 
 ## 0.6.0 (2024-01-29)
 

Makefile

@@ -71,11 +71,9 @@ PG_MAJOR ?= 16
 .PHONY: docker
 
 docker:
-	docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
-	docker build --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
+	docker build --pull --no-cache --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
 
 .PHONY: docker-release
 
 docker-release:
-	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) .
-	docker buildx build --push --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .
+	docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) .

README.md

@@ -727,7 +727,7 @@ pkg install postgresql15-pg_vector
 or the port with:
 
 ```sh
-cd /usr/ports/databases/pg_vector
+cd /usr/ports/databases/pgvector
 make install
 ```
 

src/hnsw.c

@@ -8,6 +8,7 @@
 #include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "hnsw.h"
+#include "miscadmin.h"
 #include "utils/guc.h"
 #include "utils/selfuncs.h"
 
@@ -28,7 +29,7 @@ static relopt_kind hnsw_relopt_kind;
  * this grows bigger, we should use a shmem_request_hook and
  * RequestAddinShmemSpace() to pre-reserve space for this.
  */
-static void
+void
 HnswInitLockTranche(void)
 {
 	int		   *tranche_ids;
@@ -53,7 +54,8 @@ HnswInitLockTranche(void)
 void
 HnswInit(void)
 {
-	HnswInitLockTranche();
+	if (!process_shared_preload_libraries_in_progress)
+		HnswInitLockTranche();
 
 	hnsw_relopt_kind = add_reloption_kind();
 	add_int_reloption(hnsw_relopt_kind, "m", "Max number of connections",

src/hnsw.h

@@ -389,6 +389,7 @@ void		HnswLoadElement(HnswElement element, float *distance, Datum *q, Relation i
 void		HnswSetElementTuple(char *base, HnswElementTuple etup, HnswElement element);
 void		HnswUpdateConnection(char *base, HnswElement element, HnswCandidate * hc, int lm, int lc, int *updateIdx, Relation index, FmgrInfo *procinfo, Oid collation);
 void		HnswLoadNeighbors(HnswElement element, Relation index, int m);
+void		HnswInitLockTranche(void);
 PGDLLEXPORT void HnswParallelBuildMain(dsm_segment *seg, shm_toc *toc);
 
 /* Index access methods */

src/hnswbuild.c

@@ -601,6 +601,9 @@ BuildCallback(Relation index, CALLBACK_ITEM_POINTER, Datum *values,
 static void
 InitGraph(HnswGraph * graph, char *base, long memoryTotal)
 {
+	/* Initialize the lock tranche if needed */
+	HnswInitLockTranche();
+
 	HnswPtrStore(base, graph->head, (HnswElement) NULL);
 	HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
 	graph->memoryUsed = 0;
@@ -974,6 +977,14 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
 	/* Report less than allocated so never fails */
 	InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
 
+	/*
+	 * Avoid base address for relptr for Postgres < 14.5
+	 * https://github.com/postgres/postgres/commit/7201cd18627afc64850537806da7f22150d1a83b
+	 */
+#if PG_VERSION_NUM < 140005
+	hnswshared->graphData.memoryUsed += MAXALIGN(1);
+#endif
+
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_SHARED, hnswshared);
 	shm_toc_insert(pcxt->toc, PARALLEL_KEY_HNSW_AREA, hnswarea);
 

src/ivfbuild.c

@@ -904,6 +904,27 @@ IvfflatBeginParallel(IvfflatBuildState * buildstate, bool isconcurrent, int requ
 	WaitForParallelWorkersToAttach(pcxt);
 }
 
+/*
+ * Compute parallel workers
+ */
+static int
+ComputeParallelWorkers(Relation heap, Relation index)
+{
+	int			parallel_workers;
+
+	/* Make sure it's safe to use parallel workers */
+	parallel_workers = plan_create_index_workers(RelationGetRelid(heap), RelationGetRelid(index));
+	if (parallel_workers == 0)
+		return 0;
+
+	/* Use parallel_workers storage parameter on table if set */
+	parallel_workers = RelationGetParallelWorkers(heap, -1);
+	if (parallel_workers != -1)
+		return Min(parallel_workers, max_parallel_maintenance_workers);
+
+	return max_parallel_maintenance_workers;
+}
+
 /*
  * Scan table for tuples to index
  */
@@ -923,7 +944,7 @@ AssignTuples(IvfflatBuildState * buildstate)
 
 	/* Calculate parallel workers */
 	if (buildstate->heap != NULL)
-		parallel_workers = plan_create_index_workers(RelationGetRelid(buildstate->heap), RelationGetRelid(buildstate->index));
+		parallel_workers = ComputeParallelWorkers(buildstate->heap, buildstate->index);
 
 	/* Attempt to launch parallel worker scan when required */
 	if (parallel_workers > 0)