Compare commits

...

55 Commits

Author SHA1 Message Date
Andrew Kane
bb127fce8d Try again 2022-12-09 02:01:39 -08:00
Andrew Kane
7ce4e1f53e Skip IPC::Run 2022-12-09 01:57:10 -08:00
Andrew Kane
6e41c81412 Skip tests 2022-12-09 01:06:03 -08:00
Andrew Kane
98ec05145b Install IPC::Run 2022-12-09 00:57:28 -08:00
Andrew Kane
e302c99d4a Extract tar 2022-12-09 00:52:34 -08:00
Andrew Kane
d6ab29e772 Try again 2022-12-09 00:47:45 -08:00
Andrew Kane
6ad96fe8ca Use set 2022-12-08 23:05:11 -08:00
Andrew Kane
e52ef4f2b5 Added prove_installcheck for Windows 2022-12-08 22:57:46 -08:00
Andrew Kane
8c81213592 Improved installcheck on Windows 2022-12-08 22:33:19 -08:00
Andrew Kane
f77b044303 Simplified tests 2022-12-08 22:11:49 -08:00
Andrew Kane
fab7b50c3a Updated readme [skip ci] 2022-12-08 16:03:22 -08:00
Andrew Kane
9e2e726010 Improved macros [skip ci] 2022-12-08 15:42:22 -08:00
Andrew Kane
2621f9f947 Added experimental support for Windows (including auto-vectorization) - closes #49 2022-12-08 13:27:26 -08:00
Andrew Kane
4fbe55f97e Split CI into separate groups of steps 2022-12-08 00:26:48 -08:00
Andrew Kane
9c9489e888 Removed nested FLEXIBLE_ARRAY_MEMBER for Windows 2022-12-07 19:53:48 -08:00
Andrew Kane
d376011087 Added VectorArrayFree 2022-12-07 19:45:44 -08:00
Andrew Kane
5107e057ad Improved CI [skip ci] 2022-12-07 16:35:03 -08:00
Andrew Kane
1c1812aeac Updated indentation [skip ci] 2022-12-07 16:34:01 -08:00
Andrew Kane
da12d9d8b0 Fixed copy test on Windows [skip ci] 2022-12-07 16:10:35 -08:00
Andrew Kane
de26e1e78a Perform version check earlier [skip ci] 2022-12-07 15:52:35 -08:00
Andrew Kane
8691d2ad53 Include port.h to fix buggy strtof on some platforms 2022-12-07 15:47:39 -08:00
Andrew Kane
573200c188 Use pg_prng_uint32 for Postgres 15 2022-12-07 15:28:11 -08:00
Andrew Kane
cd7cab804f Added port.h for random() function on Windows 2022-12-07 15:17:32 -08:00
Andrew Kane
84a8bdd661 Fixed FLOAT_SHORTEST_DECIMAL_LEN for Postgres < 12 [skip ci] 2022-12-02 10:54:14 -08:00
Andrew Kane
7a13a6cd3c Updated changelog [skip ci] 2022-12-01 21:23:22 -08:00
Andrew Kane
35d5cb9f06 Fixed CI (behavior of extra_float_digits changed between Postgres 11 and 12) 2022-12-01 21:07:57 -08:00
Andrew Kane
bc6b9e4bba Added more tests for text output 2022-12-01 20:45:27 -08:00
Andrew Kane
d605948be4 Updated changelog [skip ci] 2022-12-01 20:28:17 -08:00
Andrew Kane
37a784d3f6 Fixed improved vector text representation for Postgres < 12 2022-12-01 15:41:12 -08:00
Andrew Kane
c74bb5495b Improved vector text representation - fixes #46 2022-12-01 14:57:47 -08:00
Andrew Kane
cc35af160e Use strtof like float4in to avoid a double-rounding problem 2022-12-01 14:41:09 -08:00
Andrew Kane
a7f712b5a4 Version bump to 0.3.2 [skip ci] 2022-11-22 13:46:33 -08:00
Andrew Kane
a532d82bda Use palloc_extended for halfcdist 2022-11-17 19:52:46 -08:00
Andrew Kane
ccec96ba54 Fixed invalid memory alloc request size error - fixes #43 2022-11-17 14:12:49 -08:00
Andrew Kane
b7c5849ba7 Switched to dev-files option on CI 2022-11-03 15:24:12 -07:00
Andrew Kane
135147893a Updated comment [skip ci] 2022-11-02 16:55:20 -07:00
Andrew Kane
00391953dd Updated comment [skip ci] 2022-11-02 16:54:27 -07:00
Andrew Kane
81bba1f37e Improved order of insert operations 2022-11-02 16:41:56 -07:00
Andrew Kane
2d8b7e5f81 Version bump to 0.3.1 [skip ci] 2022-11-02 15:24:29 -07:00
Andrew Kane
cebae5e5ff Fixed link [skip ci] 2022-11-02 15:21:44 -07:00
Andrew Kane
4c3332fc4d Added upgrade notes [skip ci] 2022-11-02 15:20:53 -07:00
Andrew Kane
386a3b5dd5 Improved test for inserts 2022-11-02 14:29:41 -07:00
Andrew Kane
fb819eb8b2 Added comment [skip ci] 2022-11-02 12:56:32 -07:00
Andrew Kane
3f8407460e Improved test 2022-11-02 12:50:50 -07:00
Andrew Kane
059e88dda5 Fixed randomness in WAL test 2022-11-02 12:41:54 -07:00
Andrew Kane
12f158d4bb Fixed warning 2022-11-02 12:27:09 -07:00
Andrew Kane
2a3c0a769a Ensure only one backend tries to extend relation at the same time - fixes #41 2022-11-02 12:23:49 -07:00
Andrew Kane
349d844fa1 Fixed issue with uninitialized pages - fixes #41 2022-11-02 11:01:47 -07:00
Andrew Kane
a57ca81d8f Switched to runtime check 2022-10-30 12:07:03 -07:00
Andrew Kane
dc4a79a7e7 Added check for indexing overflow 2022-10-30 12:01:24 -07:00
Andrew Kane
1652a130c5 Removed unused assignments 2022-10-30 11:55:17 -07:00
Andrew Kane
a6743bd206 Added .bc to .gitignore 2022-10-30 11:54:58 -07:00
Andrew Kane
e25627074f Removed unused code [skip ci] 2022-10-30 11:49:06 -07:00
Andrew Kane
b3cad93f3a Fixed segmentation fault with index creation when lists > 6500 - fixes #42 2022-10-30 11:14:58 -07:00
Andrew Kane
379a76098e Fixed version in META.json [skip ci] 2022-10-15 14:44:44 -07:00
39 changed files with 384 additions and 188 deletions

View File

@@ -1,38 +1,65 @@
name: build
on: [push, pull_request]
jobs:
build:
runs-on: ${{ matrix.os }}
ubuntu:
runs-on: ubuntu-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
postgres: [15, 14, 13, 12, 11, 10]
include:
- os: macos-latest
postgres: 14
steps:
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: sudo apt-get update && sudo apt-get install postgresql-server-dev-${{ matrix.postgres }} libipc-run-perl
- run: make
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
- if: ${{ startsWith(matrix.os, 'macos') }}
run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: make prove_installcheck
- if: ${{ startsWith(matrix.os, 'macos') }}
run: |
brew install cpanm && cpanm IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
tar xf REL_14_5.tar.gz
make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: ${{ matrix.postgres }}
dev-files: true
- run: make
- run: |
export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
sudo apt-get update
sudo apt-get install libipc-run-perl
make prove_installcheck
mac:
runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
steps:
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: make
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
brew install cpanm
cpanm IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
tar xf REL_14_5.tar.gz
make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
nmake /NOLOGO /F Makefile.win
nmake /NOLOGO /F Makefile.win install
curl -Ls -o REL_14_5.tar.gz https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
7z x REL_14_5.tar.gz
7z x REL_14_5.tar
ls ./postgres-REL_14_5/src/test/perl
set PROVE=prove
set PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl"
nmake /NOLOGO /F Makefile.win prove_installcheck
shell: cmd

1
.gitignore vendored
View File

@@ -5,3 +5,4 @@
regression.*
*.o
*.so
*.bc

View File

@@ -1,3 +1,20 @@
## 0.4.0 (unreleased)
- Changed text representation for vector elements to match `real`
- Improved accuracy of text parsing for certain inputs
- Added experimental support for Windows
## 0.3.2 (2022-11-22)
- Fixed `invalid memory alloc request size` error
## 0.3.1 (2022-11-02)
If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
- Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7)
- Fixed segmentation fault with index creation when lists > 6500
## 0.3.0 (2022-10-15)
- Added support for Postgres 15

View File

@@ -2,7 +2,7 @@
"name": "vector",
"abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.3.0",
"version": "0.3.2",
"maintainer": [
"Andrew Kane <andrew@ankane.org>"
],
@@ -12,7 +12,7 @@
"prereqs": {
"runtime": {
"requires": {
"PostgreSQL": "10.0"
"PostgreSQL": "10.0.0"
}
}
},
@@ -20,7 +20,7 @@
"vector": {
"file": "sql/vector.sql",
"docfile": "README.md",
"version": "0.3.0",
"version": "0.3.2",
"abstract": "Open-source vector similarity search for Postgres"
}
},

View File

@@ -1,5 +1,5 @@
EXTENSION = vector
EXTVERSION = 0.3.0
EXTVERSION = 0.3.2
MODULE_big = vector
DATA = $(wildcard sql/*--*.sql)
@@ -7,7 +7,7 @@ OBJS = src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.
TESTS = $(wildcard test/sql/*.sql)
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
REGRESS_OPTS = --inputdir=test
REGRESS_OPTS = --inputdir=test --load-extension=vector
OPTFLAGS = -march=native

62
Makefile.win Normal file
View File

@@ -0,0 +1,62 @@
EXTENSION = vector
EXTVERSION = 0.3.2
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
REGRESS_OPTS = --inputdir=test --load-extension=vector
# For /arch flags
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-minimum-cpu-architecture
OPTFLAGS =
# For auto-vectorization:
# - MSVC (needs /O2 /fp:fast) - https://learn.microsoft.com/en-us/cpp/parallel/auto-parallelization-and-auto-vectorization?#auto-vectorizer
PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
# Debug MSVC auto-vectorization
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
all: sql\$(EXTENSION)--$(EXTVERSION).sql
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
# TODO use pg_config
BINDIR = $(PGROOT)\bin
INCLUDEDIR = $(PGROOT)\include
INCLUDEDIR_SERVER = $(PGROOT)\include\server
LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
SHLIB = src\$(EXTENSION).dll
LIBS = "$(LIBDIR)\postgres.lib"
.c.obj:
$(CC) $(CFLAGS) /c $< /Fo$@
$(SHLIB): $(OBJS)
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
all: $(SHLIB)
install:
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
installcheck:
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
prove_installcheck:
rm -rf tmp_check
set PGPORT=65432
set PG_REGRESS="$(BINDIR)\pg_regress"
$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) test/t/*.pl

View File

@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
Compile and install the extension (supports Postgres 10+)
```sh
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
cd pgvector
make
make install # may need sudo
@@ -164,7 +164,7 @@ CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000
### Vector Type
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a float, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions.
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions.
### Vector Operators
@@ -232,7 +232,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
You can also build the image manually
```sh
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
cd pgvector
docker build -t pgvector .
```
@@ -270,6 +270,22 @@ Install the latest version and run:
ALTER EXTENSION vector UPDATE;
```
## Upgrade Notes
### 0.3.1
If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed.
```sql
-- Postgres 12+
REINDEX INDEX CONCURRENTLY index_name;
-- Postgres < 12
CREATE INDEX CONCURRENTLY temp_name ON table USING ivfflat (column opclass);
DROP INDEX CONCURRENTLY index_name;
ALTER INDEX temp_name RENAME TO index_name;
```
## Thanks
Thanks to:

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.1'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.2'" to load this file. \quit

View File

@@ -107,7 +107,7 @@ SampleRows(IvfflatBuildState * buildstate)
buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, random());
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
reservoir_init_selection_state(&buildstate->rstate, targsamples);
while (BlockSampler_HasMore(&buildstate->bs))
@@ -233,8 +233,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
GenericXLogState *state;
int list;
IndexTuple itup = NULL; /* silence compiler warning */
BlockNumber startPage = InvalidBlockNumber;
BlockNumber insertPage = InvalidBlockNumber;
BlockNumber startPage;
BlockNumber insertPage;
Size itemsz;
int i;
int64 inserted = 0;
@@ -362,7 +362,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
static void
FreeBuildState(IvfflatBuildState * buildstate)
{
pfree(buildstate->centers);
VectorArrayFree(buildstate->centers);
pfree(buildstate->listInfo);
pfree(buildstate->normvec);
@@ -401,7 +401,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
/* Free samples before we allocate more memory */
pfree(buildstate->samples);
VectorArrayFree(buildstate->samples);
}
/*

View File

@@ -164,7 +164,7 @@ ivfflatvalidate(Oid opclassoid)
*
* See https://www.postgresql.org/docs/current/index-api.html
*/
PG_FUNCTION_INFO_V1(ivfflathandler);
PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
Datum
ivfflathandler(PG_FUNCTION_ARGS)
{

View File

@@ -3,19 +3,24 @@
#include "postgres.h"
#if PG_VERSION_NUM < 100000
#error "Requires PostgreSQL 10+"
#endif
#include "access/generic_xlog.h"
#include "access/reloptions.h"
#include "nodes/execnodes.h"
#include "port.h" /* for strtof() and random() */
#include "utils/sampling.h"
#include "utils/tuplesort.h"
#include "vector.h"
#ifdef IVFFLAT_BENCH
#include "portability/instr_time.h"
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
#if PG_VERSION_NUM < 100000
#error "Requires PostgreSQL 10+"
#ifdef IVFFLAT_BENCH
#include "portability/instr_time.h"
#endif
/* Support functions */
@@ -62,15 +67,26 @@
#define IvfflatBench(name, code) (code)
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#endif
/* Variables */
extern int ivfflat_probes;
/* Exported functions */
PGDLLEXPORT void _PG_init(void);
typedef struct VectorArrayData
{
int length;
int maxlen;
int dim;
Vector items[FLEXIBLE_ARRAY_MEMBER];
Vector *items;
} VectorArrayData;
typedef VectorArrayData * VectorArray;
@@ -191,14 +207,14 @@ typedef struct IvfflatScanOpaqueData
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
#define VECTOR_ARRAY_SIZE(_length, _dim) (offsetof(VectorArrayData, items) + _length * VECTOR_SIZE(_dim))
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) _arr + offsetof(VectorArrayData, items) + (_offset) * VECTOR_SIZE(_arr->dim))
#define VECTOR_ARRAY_SIZE(_length, _dim) (sizeof(VectorArrayData) + (_length) * VECTOR_SIZE(_dim))
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * VECTOR_SIZE((_arr)->dim))
#define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset))
#define VectorArraySet(_arr, _offset, _val) (memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE(_arr->dim)))
#define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE((_arr)->dim))
/* Methods */
void _PG_init(void);
VectorArray VectorArrayInit(int maxlen, int dimensions);
void VectorArrayFree(VectorArray arr);
void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);

View File

@@ -97,26 +97,42 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
}
else
{
/* Add a new page */
Buffer newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
Page newpage = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE);
Buffer metabuf;
Buffer newbuf;
Page newpage;
/*
* From ReadBufferExtended: Caller is responsible for ensuring
* that only one backend tries to extend a relation at the same
* time!
*/
metabuf = ReadBuffer(rel, IVFFLAT_METAPAGE_BLKNO);
LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE);
/* Add a new page */
newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
newpage = GenericXLogRegisterBuffer(state, newbuf, GENERIC_XLOG_FULL_IMAGE);
/* Init new page */
IvfflatInitPage(newbuf, newpage);
/* Update insert page */
insertPage = BufferGetBlockNumber(newbuf);
/* Update previous buffer */
IvfflatPageGetOpaque(page)->nextblkno = insertPage;
/* Init page */
IvfflatInitPage(newbuf, newpage);
/* Commit */
MarkBufferDirty(buf);
MarkBufferDirty(newbuf);
MarkBufferDirty(buf);
GenericXLogFinish(state);
/* Unlock */
UnlockReleaseBuffer(buf);
/* Unlock extend relation lock as early as possible */
UnlockReleaseBuffer(metabuf);
/* Unlock rest */
UnlockReleaseBuffer(newbuf);
UnlockReleaseBuffer(buf);
}
}

View File

@@ -5,16 +5,6 @@
#include "ivfflat.h"
#include "miscadmin.h"
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#endif
/*
* Initialize with kmeans++
*
@@ -26,7 +16,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
FmgrInfo *procinfo;
Oid collation;
int i;
int j;
int64 j;
double distance;
double sum;
double choice;
@@ -39,7 +29,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
collation = index->rd_indcollation[0];
/* Choose an initial center uniformly at random */
VectorArraySet(centers, 0, VectorArrayGet(samples, random() % samples->length));
VectorArraySet(centers, 0, VectorArrayGet(samples, RandomInt() % samples->length));
centers->length++;
for (j = 0; j < numSamples; j++)
@@ -182,8 +172,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
Vector *vec;
Vector *newCenter;
int iteration;
int j;
int k;
int64 j;
int64 k;
int dimensions = centers->dim;
int numCenters = centers->maxlen;
int numSamples = samples->length;
@@ -227,6 +217,10 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
/* Ensure indexing does not overflow */
if (numCenters * numCenters > INT_MAX)
elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
@@ -239,7 +233,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
upperBound = palloc(upperBoundSize);
s = palloc(sSize);
halfcdist = palloc(halfcdistSize);
halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
newcdist = palloc(newcdistSize);
newCenters = VectorArrayInit(numCenters, dimensions);
@@ -259,8 +253,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
minDistance = DBL_MAX;
closestCenter = -1;
vec = VectorArrayGet(samples, j);
/* Find closest center */
for (k = 0; k < numCenters; k++)
{
@@ -451,7 +443,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
break;
}
pfree(newCenters);
VectorArrayFree(newCenters);
pfree(centerCounts);
pfree(closestCenters);
pfree(lowerBound);

View File

@@ -10,14 +10,25 @@
VectorArray
VectorArrayInit(int maxlen, int dimensions)
{
VectorArray res = palloc0(VECTOR_ARRAY_SIZE(maxlen, dimensions));
VectorArray res = palloc(sizeof(VectorArrayData));
res->length = 0;
res->maxlen = maxlen;
res->dim = dimensions;
res->items = palloc_extended(maxlen * VECTOR_SIZE(dimensions), MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
return res;
}
/*
* Free a vector array
*/
void
VectorArrayFree(VectorArray arr)
{
pfree(arr->items);
pfree(arr);
}
/*
* Print vector array - useful for debugging
*/

View File

@@ -13,7 +13,10 @@
#include "utils/numeric.h"
#if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h"
#else
#include <float.h>
#endif
#if PG_VERSION_NUM < 130000
@@ -106,14 +109,14 @@ PrintVector(char *msg, Vector * vector)
/*
* Convert textual representation to internal representation
*/
PG_FUNCTION_INFO_V1(vector_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
Datum
vector_in(PG_FUNCTION_ARGS)
{
char *str = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2);
int i;
double x[VECTOR_MAX_DIM];
float x[VECTOR_MAX_DIM];
int dim = 0;
char *pt;
char *stringEnd;
@@ -136,7 +139,8 @@ vector_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
x[dim] = strtod(pt, &stringEnd);
/* Use strtof like float4in to avoid a double-rounding problem */
x[dim] = strtof(pt, &stringEnd);
CheckElement(x[dim]);
dim++;
@@ -182,35 +186,68 @@ vector_in(PG_FUNCTION_ARGS)
/*
* Convert internal representation to textual representation
*/
PG_FUNCTION_INFO_V1(vector_out);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
Datum
vector_out(PG_FUNCTION_ARGS)
{
Vector *vector = PG_GETARG_VECTOR_P(0);
StringInfoData buf;
int dim = vector->dim;
char *buf;
char *ptr;
int i;
int n;
initStringInfo(&buf);
#if PG_VERSION_NUM < 120000
int ndig = FLT_DIG + extra_float_digits;
appendStringInfoChar(&buf, '[');
if (ndig < 1)
ndig = 1;
#define FLOAT_SHORTEST_DECIMAL_LEN (ndig + 10)
#endif
/*
* Need:
*
* dim * (FLOAT_SHORTEST_DECIMAL_LEN - 1) bytes for
* float_to_shortest_decimal_bufn
*
* dim - 1 bytes for separator
*
* 3 bytes for [, ], and \0
*/
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
ptr = buf;
*ptr = '[';
ptr++;
for (i = 0; i < dim; i++)
{
if (i > 0)
appendStringInfoString(&buf, ",");
{
*ptr = ',';
ptr++;
}
appendStringInfoString(&buf, float8out_internal(vector->x[i]));
#if PG_VERSION_NUM >= 120000
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
#else
n = sprintf(ptr, "%.*g", ndig, vector->x[i]);
#endif
ptr += n;
}
appendStringInfoChar(&buf, ']');
*ptr = ']';
ptr++;
*ptr = '\0';
PG_FREE_IF_COPY(vector, 0);
PG_RETURN_CSTRING(buf.data);
PG_RETURN_CSTRING(buf);
}
/*
* Convert type modifier
*/
PG_FUNCTION_INFO_V1(vector_typmod_in);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
Datum
vector_typmod_in(PG_FUNCTION_ARGS)
{
@@ -241,7 +278,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
/*
* Convert external binary representation to internal representation
*/
PG_FUNCTION_INFO_V1(vector_recv);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
Datum
vector_recv(PG_FUNCTION_ARGS)
{
@@ -273,7 +310,7 @@ vector_recv(PG_FUNCTION_ARGS)
/*
* Convert internal representation to the external binary representation
*/
PG_FUNCTION_INFO_V1(vector_send);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
Datum
vector_send(PG_FUNCTION_ARGS)
{
@@ -293,7 +330,7 @@ vector_send(PG_FUNCTION_ARGS)
/*
* Convert vector to vector
*/
PG_FUNCTION_INFO_V1(vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
Datum
vector(PG_FUNCTION_ARGS)
{
@@ -308,7 +345,7 @@ vector(PG_FUNCTION_ARGS)
/*
* Convert array to vector
*/
PG_FUNCTION_INFO_V1(array_to_vector);
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
Datum
array_to_vector(PG_FUNCTION_ARGS)
{
@@ -366,7 +403,7 @@ array_to_vector(PG_FUNCTION_ARGS)
/*
* Convert vector to float4[]
*/
PG_FUNCTION_INFO_V1(vector_to_float4);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
Datum
vector_to_float4(PG_FUNCTION_ARGS)
{
@@ -389,12 +426,14 @@ vector_to_float4(PG_FUNCTION_ARGS)
/*
* Get the L2 distance between vectors
*/
PG_FUNCTION_INFO_V1(l2_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
Datum
l2_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0;
double diff;
@@ -402,7 +441,7 @@ l2_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++)
{
diff = a->x[i] - b->x[i];
diff = ax[i] - bx[i];
distance += diff * diff;
}
@@ -413,12 +452,14 @@ l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between vectors
* This saves a sqrt calculation
*/
PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
Datum
vector_l2_squared_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0;
double diff;
@@ -426,7 +467,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++)
{
diff = a->x[i] - b->x[i];
diff = ax[i] - bx[i];
distance += diff * diff;
}
@@ -436,18 +477,20 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
/*
* Get the inner product of two vectors
*/
PG_FUNCTION_INFO_V1(inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
Datum
inner_product(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0;
CheckDims(a, b);
for (int i = 0; i < a->dim; i++)
distance += a->x[i] * b->x[i];
distance += ax[i] * bx[i];
PG_RETURN_FLOAT8(distance);
}
@@ -455,18 +498,20 @@ inner_product(PG_FUNCTION_ARGS)
/*
* Get the negative inner product of two vectors
*/
PG_FUNCTION_INFO_V1(vector_negative_inner_product);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
Datum
vector_negative_inner_product(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0;
CheckDims(a, b);
for (int i = 0; i < a->dim; i++)
distance += a->x[i] * b->x[i];
distance += ax[i] * bx[i];
PG_RETURN_FLOAT8(distance * -1);
}
@@ -474,12 +519,14 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
/*
* Get the cosine distance between two vectors
*/
PG_FUNCTION_INFO_V1(cosine_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
Datum
cosine_distance(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0;
double norma = 0.0;
double normb = 0.0;
@@ -488,9 +535,9 @@ cosine_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++)
{
distance += a->x[i] * b->x[i];
norma += a->x[i] * a->x[i];
normb += b->x[i] * b->x[i];
distance += ax[i] * bx[i];
norma += ax[i] * ax[i];
normb += bx[i] * bx[i];
}
PG_RETURN_FLOAT8(1 - (distance / (sqrt(norma) * sqrt(normb))));
@@ -501,7 +548,7 @@ cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm)
*/
PG_FUNCTION_INFO_V1(vector_spherical_distance);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
Datum
vector_spherical_distance(PG_FUNCTION_ARGS)
{
@@ -526,7 +573,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
/*
* Get the dimensions of a vector
*/
PG_FUNCTION_INFO_V1(vector_dims);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
Datum
vector_dims(PG_FUNCTION_ARGS)
{
@@ -538,15 +585,16 @@ vector_dims(PG_FUNCTION_ARGS)
/*
* Get the L2 norm of a vector
*/
PG_FUNCTION_INFO_V1(vector_norm);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
Datum
vector_norm(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
double norm = 0.0;
for (int i = 0; i < a->dim; i++)
norm += a->x[i] * a->x[i];
norm += ax[i] * ax[i];
PG_RETURN_FLOAT8(sqrt(norm));
}
@@ -554,20 +602,23 @@ vector_norm(PG_FUNCTION_ARGS)
/*
* Add vectors
*/
PG_FUNCTION_INFO_V1(vector_add);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
Datum
vector_add(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
Vector *result;
int i;
float *rx;
CheckDims(a, b);
result = InitVector(a->dim);
for (i = 0; i < a->dim; i++)
result->x[i] = a->x[i] + b->x[i];
rx = result->x;
for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] + bx[i];
PG_RETURN_POINTER(result);
}
@@ -575,20 +626,23 @@ vector_add(PG_FUNCTION_ARGS)
/*
* Subtract vectors
*/
PG_FUNCTION_INFO_V1(vector_sub);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
Datum
vector_sub(PG_FUNCTION_ARGS)
{
Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
Vector *result;
int i;
float *rx;
CheckDims(a, b);
result = InitVector(a->dim);
for (i = 0; i < a->dim; i++)
result->x[i] = a->x[i] - b->x[i];
rx = result->x;
for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] - bx[i];
PG_RETURN_POINTER(result);
}
@@ -617,7 +671,7 @@ vector_cmp_internal(Vector * a, Vector * b)
/*
* Less than
*/
PG_FUNCTION_INFO_V1(vector_lt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
Datum
vector_lt(PG_FUNCTION_ARGS)
{
@@ -630,7 +684,7 @@ vector_lt(PG_FUNCTION_ARGS)
/*
* Less than or equal
*/
PG_FUNCTION_INFO_V1(vector_le);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
Datum
vector_le(PG_FUNCTION_ARGS)
{
@@ -643,7 +697,7 @@ vector_le(PG_FUNCTION_ARGS)
/*
* Equal
*/
PG_FUNCTION_INFO_V1(vector_eq);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
Datum
vector_eq(PG_FUNCTION_ARGS)
{
@@ -656,7 +710,7 @@ vector_eq(PG_FUNCTION_ARGS)
/*
* Not equal
*/
PG_FUNCTION_INFO_V1(vector_ne);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
Datum
vector_ne(PG_FUNCTION_ARGS)
{
@@ -669,7 +723,7 @@ vector_ne(PG_FUNCTION_ARGS)
/*
* Greater than or equal
*/
PG_FUNCTION_INFO_V1(vector_ge);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
Datum
vector_ge(PG_FUNCTION_ARGS)
{
@@ -682,7 +736,7 @@ vector_ge(PG_FUNCTION_ARGS)
/*
* Greater than
*/
PG_FUNCTION_INFO_V1(vector_gt);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
Datum
vector_gt(PG_FUNCTION_ARGS)
{
@@ -695,7 +749,7 @@ vector_gt(PG_FUNCTION_ARGS)
/*
* Compare vectors
*/
PG_FUNCTION_INFO_V1(vector_cmp);
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
Datum
vector_cmp(PG_FUNCTION_ARGS)
{

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT ARRAY[1,2,3]::vector;
array
---------

View File

@@ -1,10 +1,8 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3));
\copy t TO '/tmp/data.bin' WITH (FORMAT binary)
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary)
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;
val
---------

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector + '[4,5,6]';
?column?
----------

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector;
vector
---------
@@ -12,6 +10,12 @@ SELECT '[-1,2,3]'::vector;
[-1,2,3]
(1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "hello"
LINE 1: SELECT '[hello,1]'::vector;

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 0);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));

View File

@@ -1,6 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT ARRAY[1,2,3]::vector;
SELECT ARRAY[1.0,2.0,3.0]::vector;
SELECT ARRAY[1,2,3]::float4[]::vector;

View File

@@ -1,13 +1,10 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3));
\copy t TO '/tmp/data.bin' WITH (FORMAT binary)
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary)
\copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val;

View File

@@ -1,6 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[1,2,3]'::vector - '[4,5,6]';

View File

@@ -1,8 +1,6 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector;
SELECT '[-1,2,3]'::vector;
SELECT '[1.23456]'::vector;
SELECT '[hello,1]'::vector;
SELECT '[NaN,1]'::vector;
SELECT '[Infinity,1]'::vector;

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3));

View File

@@ -51,6 +51,11 @@ sub test_index_replay
return;
}
# Use ARRAY[random(), random(), random(), ...] over
# SELECT array_agg(random()) FROM generate_series(1, $dim)
# to generate different values for each row
my $array_sql = join(",", ('random()') x $dim);
# Initialize primary node
$node_primary = get_new_node('primary');
$node_primary->init(allows_streaming => 1);
@@ -74,7 +79,7 @@ $node_replica->start;
$node_primary->safe_psql("postgres", "CREATE EXTENSION vector;");
$node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
$node_primary->safe_psql("postgres",
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series(1, 100000) i;"
"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
);
$node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
@@ -90,7 +95,7 @@ for my $i (1 .. 10)
test_index_replay("vacuum $i");
my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000);
$node_primary->safe_psql("postgres",
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series($start, $end) i;"
"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series($start, $end) i;"
);
test_index_replay("insert $i");
}

View File

@@ -2,7 +2,11 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 3;
use Test::More tests => 5;
my $dim = 768;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node
my $node = get_new_node('node');
@@ -11,9 +15,9 @@ $node->start;
# Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(768));");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector($dim));");
$node->safe_psql("postgres",
"INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10000) i;"
"INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
);
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
@@ -24,10 +28,18 @@ $node->pgbench(
[qr{^$}],
"concurrent INSERTs",
{
"007_concurrent" => q(
BEGIN;
INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10) i;
COMMIT;
),
"007_inserts" => "INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10) i;"
}
);
my $expected = 10000 + 5 * 100 * 10;
my $count = $node->safe_psql("postgres", "SELECT COUNT(*) FROM tst;");
is($count, $expected);
$count = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET ivfflat.probes = 100;
SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
));
is($count, $expected);

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat access method'
default_version = '0.3.0'
default_version = '0.3.2'
module_pathname = '$libdir/vector'
relocatable = true