Compare commits

...

55 Commits

Author SHA1 Message Date
Andrew Kane
bb127fce8d Try again 2022-12-09 02:01:39 -08:00
Andrew Kane
7ce4e1f53e Skip IPC::Run 2022-12-09 01:57:10 -08:00
Andrew Kane
6e41c81412 Skip tests 2022-12-09 01:06:03 -08:00
Andrew Kane
98ec05145b Install IPC::Run 2022-12-09 00:57:28 -08:00
Andrew Kane
e302c99d4a Extract tar 2022-12-09 00:52:34 -08:00
Andrew Kane
d6ab29e772 Try again 2022-12-09 00:47:45 -08:00
Andrew Kane
6ad96fe8ca Use set 2022-12-08 23:05:11 -08:00
Andrew Kane
e52ef4f2b5 Added prove_installcheck for Windows 2022-12-08 22:57:46 -08:00
Andrew Kane
8c81213592 Improved installcheck on Windows 2022-12-08 22:33:19 -08:00
Andrew Kane
f77b044303 Simplified tests 2022-12-08 22:11:49 -08:00
Andrew Kane
fab7b50c3a Updated readme [skip ci] 2022-12-08 16:03:22 -08:00
Andrew Kane
9e2e726010 Improved macros [skip ci] 2022-12-08 15:42:22 -08:00
Andrew Kane
2621f9f947 Added experimental support for Windows (including auto-vectorization) - closes #49 2022-12-08 13:27:26 -08:00
Andrew Kane
4fbe55f97e Split CI into separate groups of steps 2022-12-08 00:26:48 -08:00
Andrew Kane
9c9489e888 Removed nested FLEXIBLE_ARRAY_MEMBER for Windows 2022-12-07 19:53:48 -08:00
Andrew Kane
d376011087 Added VectorArrayFree 2022-12-07 19:45:44 -08:00
Andrew Kane
5107e057ad Improved CI [skip ci] 2022-12-07 16:35:03 -08:00
Andrew Kane
1c1812aeac Updated indentation [skip ci] 2022-12-07 16:34:01 -08:00
Andrew Kane
da12d9d8b0 Fixed copy test on Windows [skip ci] 2022-12-07 16:10:35 -08:00
Andrew Kane
de26e1e78a Perform version check earlier [skip ci] 2022-12-07 15:52:35 -08:00
Andrew Kane
8691d2ad53 Include port.h to fix buggy strtof on some platforms 2022-12-07 15:47:39 -08:00
Andrew Kane
573200c188 Use pg_prng_uint32 for Postgres 15 2022-12-07 15:28:11 -08:00
Andrew Kane
cd7cab804f Added port.h for random() function on Windows 2022-12-07 15:17:32 -08:00
Andrew Kane
84a8bdd661 Fixed FLOAT_SHORTEST_DECIMAL_LEN for Postgres < 12 [skip ci] 2022-12-02 10:54:14 -08:00
Andrew Kane
7a13a6cd3c Updated changelog [skip ci] 2022-12-01 21:23:22 -08:00
Andrew Kane
35d5cb9f06 Fixed CI (behavior of extra_float_digits changed between Postgres 11 and 12) 2022-12-01 21:07:57 -08:00
Andrew Kane
bc6b9e4bba Added more tests for text output 2022-12-01 20:45:27 -08:00
Andrew Kane
d605948be4 Updated changelog [skip ci] 2022-12-01 20:28:17 -08:00
Andrew Kane
37a784d3f6 Fixed improved vector text representation for Postgres < 12 2022-12-01 15:41:12 -08:00
Andrew Kane
c74bb5495b Improved vector text representation - fixes #46 2022-12-01 14:57:47 -08:00
Andrew Kane
cc35af160e Use strtof like float4in to avoid a double-rounding problem 2022-12-01 14:41:09 -08:00
Andrew Kane
a7f712b5a4 Version bump to 0.3.2 [skip ci] 2022-11-22 13:46:33 -08:00
Andrew Kane
a532d82bda Use palloc_extended for halfcdist 2022-11-17 19:52:46 -08:00
Andrew Kane
ccec96ba54 Fixed invalid memory alloc request size error - fixes #43 2022-11-17 14:12:49 -08:00
Andrew Kane
b7c5849ba7 Switched to dev-files option on CI 2022-11-03 15:24:12 -07:00
Andrew Kane
135147893a Updated comment [skip ci] 2022-11-02 16:55:20 -07:00
Andrew Kane
00391953dd Updated comment [skip ci] 2022-11-02 16:54:27 -07:00
Andrew Kane
81bba1f37e Improved order of insert operations 2022-11-02 16:41:56 -07:00
Andrew Kane
2d8b7e5f81 Version bump to 0.3.1 [skip ci] 2022-11-02 15:24:29 -07:00
Andrew Kane
cebae5e5ff Fixed link [skip ci] 2022-11-02 15:21:44 -07:00
Andrew Kane
4c3332fc4d Added upgrade notes [skip ci] 2022-11-02 15:20:53 -07:00
Andrew Kane
386a3b5dd5 Improved test for inserts 2022-11-02 14:29:41 -07:00
Andrew Kane
fb819eb8b2 Added comment [skip ci] 2022-11-02 12:56:32 -07:00
Andrew Kane
3f8407460e Improved test 2022-11-02 12:50:50 -07:00
Andrew Kane
059e88dda5 Fixed randomness in WAL test 2022-11-02 12:41:54 -07:00
Andrew Kane
12f158d4bb Fixed warning 2022-11-02 12:27:09 -07:00
Andrew Kane
2a3c0a769a Ensure only one backend tries to extend relation at the same time - fixes #41 2022-11-02 12:23:49 -07:00
Andrew Kane
349d844fa1 Fixed issue with uninitialized pages - fixes #41 2022-11-02 11:01:47 -07:00
Andrew Kane
a57ca81d8f Switched to runtime check 2022-10-30 12:07:03 -07:00
Andrew Kane
dc4a79a7e7 Added check for indexing overflow 2022-10-30 12:01:24 -07:00
Andrew Kane
1652a130c5 Removed unused assignments 2022-10-30 11:55:17 -07:00
Andrew Kane
a6743bd206 Added .bc to .gitignore 2022-10-30 11:54:58 -07:00
Andrew Kane
e25627074f Removed unused code [skip ci] 2022-10-30 11:49:06 -07:00
Andrew Kane
b3cad93f3a Fixed segmentation fault with index creation when lists > 6500 - fixes #42 2022-10-30 11:14:58 -07:00
Andrew Kane
379a76098e Fixed version in META.json [skip ci] 2022-10-15 14:44:44 -07:00
39 changed files with 384 additions and 188 deletions

View File

@@ -1,38 +1,65 @@
name: build name: build
on: [push, pull_request] on: [push, pull_request]
jobs: jobs:
build: ubuntu:
runs-on: ${{ matrix.os }} runs-on: ubuntu-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
os: [ubuntu-latest]
postgres: [15, 14, 13, 12, 11, 10] postgres: [15, 14, 13, 12, 11, 10]
include:
- os: macos-latest
postgres: 14
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1 - uses: ankane/setup-postgres@v1
with: with:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
- if: ${{ startsWith(matrix.os, 'ubuntu') }} dev-files: true
run: sudo apt-get update && sudo apt-get install postgresql-server-dev-${{ matrix.postgres }} libipc-run-perl
- run: make - run: make
- if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: |
run: |
export PG_CONFIG=`which pg_config` export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install sudo --preserve-env=PG_CONFIG make install
- if: ${{ startsWith(matrix.os, 'macos') }}
run: make install
- run: make installcheck - run: make installcheck
- if: ${{ failure() }} - if: ${{ failure() }}
run: cat regression.diffs run: cat regression.diffs
- if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: |
run: make prove_installcheck sudo apt-get update
- if: ${{ startsWith(matrix.os, 'macos') }} sudo apt-get install libipc-run-perl
run: | make prove_installcheck
brew install cpanm && cpanm IPC::Run mac:
runs-on: macos-latest
if: ${{ !startsWith(github.ref_name, 'windows') }}
steps:
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: make
- run: make install
- run: make installcheck
- if: ${{ failure() }}
run: cat regression.diffs
- run: |
brew install cpanm
cpanm IPC::Run
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
tar xf REL_14_5.tar.gz tar xf REL_14_5.tar.gz
make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5" make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: ankane/setup-postgres@v1
with:
postgres-version: 14
- run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
nmake /NOLOGO /F Makefile.win
nmake /NOLOGO /F Makefile.win install
curl -Ls -o REL_14_5.tar.gz https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
7z x REL_14_5.tar.gz
7z x REL_14_5.tar
ls ./postgres-REL_14_5/src/test/perl
set PROVE=prove
set PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl"
nmake /NOLOGO /F Makefile.win prove_installcheck
shell: cmd

1
.gitignore vendored
View File

@@ -5,3 +5,4 @@
regression.* regression.*
*.o *.o
*.so *.so
*.bc

View File

@@ -1,3 +1,20 @@
## 0.4.0 (unreleased)
- Changed text representation for vector elements to match `real`
- Improved accuracy of text parsing for certain inputs
- Added experimental support for Windows
## 0.3.2 (2022-11-22)
- Fixed `invalid memory alloc request size` error
## 0.3.1 (2022-11-02)
If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
- Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7)
- Fixed segmentation fault with index creation when lists > 6500
## 0.3.0 (2022-10-15) ## 0.3.0 (2022-10-15)
- Added support for Postgres 15 - Added support for Postgres 15

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.3.0", "version": "0.3.2",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -12,7 +12,7 @@
"prereqs": { "prereqs": {
"runtime": { "runtime": {
"requires": { "requires": {
"PostgreSQL": "10.0" "PostgreSQL": "10.0.0"
} }
} }
}, },
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.3.0", "version": "0.3.2",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.3.0 EXTVERSION = 0.3.2
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*.sql) DATA = $(wildcard sql/*--*.sql)
@@ -7,7 +7,7 @@ OBJS = src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.
TESTS = $(wildcard test/sql/*.sql) TESTS = $(wildcard test/sql/*.sql)
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
REGRESS_OPTS = --inputdir=test REGRESS_OPTS = --inputdir=test --load-extension=vector
OPTFLAGS = -march=native OPTFLAGS = -march=native

62
Makefile.win Normal file
View File

@@ -0,0 +1,62 @@
EXTENSION = vector
EXTVERSION = 0.3.2
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
REGRESS_OPTS = --inputdir=test --load-extension=vector
# For /arch flags
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-minimum-cpu-architecture
OPTFLAGS =
# For auto-vectorization:
# - MSVC (needs /O2 /fp:fast) - https://learn.microsoft.com/en-us/cpp/parallel/auto-parallelization-and-auto-vectorization?#auto-vectorizer
PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
# Debug MSVC auto-vectorization
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
all: sql\$(EXTENSION)--$(EXTVERSION).sql
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
copy sql\$(EXTENSION).sql $@
# TODO use pg_config
BINDIR = $(PGROOT)\bin
INCLUDEDIR = $(PGROOT)\include
INCLUDEDIR_SERVER = $(PGROOT)\include\server
LIBDIR = $(PGROOT)\lib
PKGLIBDIR = $(PGROOT)\lib
SHAREDIR = $(PGROOT)\share
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
SHLIB = src\$(EXTENSION).dll
LIBS = "$(LIBDIR)\postgres.lib"
.c.obj:
$(CC) $(CFLAGS) /c $< /Fo$@
$(SHLIB): $(OBJS)
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
all: $(SHLIB)
install:
copy $(SHLIB) "$(PKGLIBDIR)"
copy $(EXTENSION).control "$(SHAREDIR)\extension"
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
installcheck:
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
prove_installcheck:
rm -rf tmp_check
set PGPORT=65432
set PG_REGRESS="$(BINDIR)\pg_regress"
$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) test/t/*.pl

View File

@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
Compile and install the extension (supports Postgres 10+) Compile and install the extension (supports Postgres 10+)
```sh ```sh
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
@@ -164,7 +164,7 @@ CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000
### Vector Type ### Vector Type
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a float, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions. Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions.
### Vector Operators ### Vector Operators
@@ -232,7 +232,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
You can also build the image manually You can also build the image manually
```sh ```sh
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build -t pgvector . docker build -t pgvector .
``` ```
@@ -270,6 +270,22 @@ Install the latest version and run:
ALTER EXTENSION vector UPDATE; ALTER EXTENSION vector UPDATE;
``` ```
## Upgrade Notes
### 0.3.1
If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed.
```sql
-- Postgres 12+
REINDEX INDEX CONCURRENTLY index_name;
-- Postgres < 12
CREATE INDEX CONCURRENTLY temp_name ON table USING ivfflat (column opclass);
DROP INDEX CONCURRENTLY index_name;
ALTER INDEX temp_name RENAME TO index_name;
```
## Thanks ## Thanks
Thanks to: Thanks to:

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.1'" to load this file. \quit

View File

@@ -0,0 +1,2 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.2'" to load this file. \quit

View File

@@ -107,7 +107,7 @@ SampleRows(IvfflatBuildState * buildstate)
buildstate->rowstoskip = -1; buildstate->rowstoskip = -1;
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, random()); BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
reservoir_init_selection_state(&buildstate->rstate, targsamples); reservoir_init_selection_state(&buildstate->rstate, targsamples);
while (BlockSampler_HasMore(&buildstate->bs)) while (BlockSampler_HasMore(&buildstate->bs))
@@ -233,8 +233,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
GenericXLogState *state; GenericXLogState *state;
int list; int list;
IndexTuple itup = NULL; /* silence compiler warning */ IndexTuple itup = NULL; /* silence compiler warning */
BlockNumber startPage = InvalidBlockNumber; BlockNumber startPage;
BlockNumber insertPage = InvalidBlockNumber; BlockNumber insertPage;
Size itemsz; Size itemsz;
int i; int i;
int64 inserted = 0; int64 inserted = 0;
@@ -362,7 +362,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
static void static void
FreeBuildState(IvfflatBuildState * buildstate) FreeBuildState(IvfflatBuildState * buildstate)
{ {
pfree(buildstate->centers); VectorArrayFree(buildstate->centers);
pfree(buildstate->listInfo); pfree(buildstate->listInfo);
pfree(buildstate->normvec); pfree(buildstate->normvec);
@@ -401,7 +401,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers)); IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
/* Free samples before we allocate more memory */ /* Free samples before we allocate more memory */
pfree(buildstate->samples); VectorArrayFree(buildstate->samples);
} }
/* /*

View File

@@ -164,7 +164,7 @@ ivfflatvalidate(Oid opclassoid)
* *
* See https://www.postgresql.org/docs/current/index-api.html * See https://www.postgresql.org/docs/current/index-api.html
*/ */
PG_FUNCTION_INFO_V1(ivfflathandler); PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
Datum Datum
ivfflathandler(PG_FUNCTION_ARGS) ivfflathandler(PG_FUNCTION_ARGS)
{ {

View File

@@ -3,19 +3,24 @@
#include "postgres.h" #include "postgres.h"
#if PG_VERSION_NUM < 100000
#error "Requires PostgreSQL 10+"
#endif
#include "access/generic_xlog.h" #include "access/generic_xlog.h"
#include "access/reloptions.h" #include "access/reloptions.h"
#include "nodes/execnodes.h" #include "nodes/execnodes.h"
#include "port.h" /* for strtof() and random() */
#include "utils/sampling.h" #include "utils/sampling.h"
#include "utils/tuplesort.h" #include "utils/tuplesort.h"
#include "vector.h" #include "vector.h"
#ifdef IVFFLAT_BENCH #if PG_VERSION_NUM >= 150000
#include "portability/instr_time.h" #include "common/pg_prng.h"
#endif #endif
#if PG_VERSION_NUM < 100000 #ifdef IVFFLAT_BENCH
#error "Requires PostgreSQL 10+" #include "portability/instr_time.h"
#endif #endif
/* Support functions */ /* Support functions */
@@ -62,15 +67,26 @@
#define IvfflatBench(name, code) (code) #define IvfflatBench(name, code) (code)
#endif #endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#define RandomInt() random()
#endif
/* Variables */ /* Variables */
extern int ivfflat_probes; extern int ivfflat_probes;
/* Exported functions */
PGDLLEXPORT void _PG_init(void);
typedef struct VectorArrayData typedef struct VectorArrayData
{ {
int length; int length;
int maxlen; int maxlen;
int dim; int dim;
Vector items[FLEXIBLE_ARRAY_MEMBER]; Vector *items;
} VectorArrayData; } VectorArrayData;
typedef VectorArrayData * VectorArray; typedef VectorArrayData * VectorArray;
@@ -191,14 +207,14 @@ typedef struct IvfflatScanOpaqueData
typedef IvfflatScanOpaqueData * IvfflatScanOpaque; typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
#define VECTOR_ARRAY_SIZE(_length, _dim) (offsetof(VectorArrayData, items) + _length * VECTOR_SIZE(_dim)) #define VECTOR_ARRAY_SIZE(_length, _dim) (sizeof(VectorArrayData) + (_length) * VECTOR_SIZE(_dim))
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) _arr + offsetof(VectorArrayData, items) + (_offset) * VECTOR_SIZE(_arr->dim)) #define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * VECTOR_SIZE((_arr)->dim))
#define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset)) #define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset))
#define VectorArraySet(_arr, _offset, _val) (memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE(_arr->dim))) #define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE((_arr)->dim))
/* Methods */ /* Methods */
void _PG_init(void);
VectorArray VectorArrayInit(int maxlen, int dimensions); VectorArray VectorArrayInit(int maxlen, int dimensions);
void VectorArrayFree(VectorArray arr);
void PrintVectorArray(char *msg, VectorArray arr); void PrintVectorArray(char *msg, VectorArray arr);
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers); void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum); FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);

View File

@@ -97,26 +97,42 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
} }
else else
{ {
/* Add a new page */ Buffer metabuf;
Buffer newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM); Buffer newbuf;
Page newpage = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE); Page newpage;
/*
* From ReadBufferExtended: Caller is responsible for ensuring
* that only one backend tries to extend a relation at the same
* time!
*/
metabuf = ReadBuffer(rel, IVFFLAT_METAPAGE_BLKNO);
LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE);
/* Add a new page */
newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
newpage = GenericXLogRegisterBuffer(state, newbuf, GENERIC_XLOG_FULL_IMAGE);
/* Init new page */
IvfflatInitPage(newbuf, newpage);
/* Update insert page */
insertPage = BufferGetBlockNumber(newbuf); insertPage = BufferGetBlockNumber(newbuf);
/* Update previous buffer */ /* Update previous buffer */
IvfflatPageGetOpaque(page)->nextblkno = insertPage; IvfflatPageGetOpaque(page)->nextblkno = insertPage;
/* Init page */
IvfflatInitPage(newbuf, newpage);
/* Commit */ /* Commit */
MarkBufferDirty(buf);
MarkBufferDirty(newbuf); MarkBufferDirty(newbuf);
MarkBufferDirty(buf);
GenericXLogFinish(state); GenericXLogFinish(state);
/* Unlock */ /* Unlock extend relation lock as early as possible */
UnlockReleaseBuffer(buf); UnlockReleaseBuffer(metabuf);
/* Unlock rest */
UnlockReleaseBuffer(newbuf); UnlockReleaseBuffer(newbuf);
UnlockReleaseBuffer(buf);
} }
} }

View File

@@ -5,16 +5,6 @@
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h" #include "miscadmin.h"
#if PG_VERSION_NUM >= 150000
#include "common/pg_prng.h"
#endif
#if PG_VERSION_NUM >= 150000
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
#else
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
#endif
/* /*
* Initialize with kmeans++ * Initialize with kmeans++
* *
@@ -26,7 +16,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
FmgrInfo *procinfo; FmgrInfo *procinfo;
Oid collation; Oid collation;
int i; int i;
int j; int64 j;
double distance; double distance;
double sum; double sum;
double choice; double choice;
@@ -39,7 +29,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
collation = index->rd_indcollation[0]; collation = index->rd_indcollation[0];
/* Choose an initial center uniformly at random */ /* Choose an initial center uniformly at random */
VectorArraySet(centers, 0, VectorArrayGet(samples, random() % samples->length)); VectorArraySet(centers, 0, VectorArrayGet(samples, RandomInt() % samples->length));
centers->length++; centers->length++;
for (j = 0; j < numSamples; j++) for (j = 0; j < numSamples; j++)
@@ -182,8 +172,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
Vector *vec; Vector *vec;
Vector *newCenter; Vector *newCenter;
int iteration; int iteration;
int j; int64 j;
int k; int64 k;
int dimensions = centers->dim; int dimensions = centers->dim;
int numCenters = centers->maxlen; int numCenters = centers->maxlen;
int numSamples = samples->length; int numSamples = samples->length;
@@ -227,6 +217,10 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB", errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024))); totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
/* Ensure indexing does not overflow */
if (numCenters * numCenters > INT_MAX)
elog(ERROR, "Indexing overflow detected. Please report a bug.");
/* Set support functions */ /* Set support functions */
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC); procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC); normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
@@ -239,7 +233,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE); lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
upperBound = palloc(upperBoundSize); upperBound = palloc(upperBoundSize);
s = palloc(sSize); s = palloc(sSize);
halfcdist = palloc(halfcdistSize); halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
newcdist = palloc(newcdistSize); newcdist = palloc(newcdistSize);
newCenters = VectorArrayInit(numCenters, dimensions); newCenters = VectorArrayInit(numCenters, dimensions);
@@ -259,8 +253,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
minDistance = DBL_MAX; minDistance = DBL_MAX;
closestCenter = -1; closestCenter = -1;
vec = VectorArrayGet(samples, j);
/* Find closest center */ /* Find closest center */
for (k = 0; k < numCenters; k++) for (k = 0; k < numCenters; k++)
{ {
@@ -451,7 +443,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
break; break;
} }
pfree(newCenters); VectorArrayFree(newCenters);
pfree(centerCounts); pfree(centerCounts);
pfree(closestCenters); pfree(closestCenters);
pfree(lowerBound); pfree(lowerBound);

View File

@@ -10,14 +10,25 @@
VectorArray VectorArray
VectorArrayInit(int maxlen, int dimensions) VectorArrayInit(int maxlen, int dimensions)
{ {
VectorArray res = palloc0(VECTOR_ARRAY_SIZE(maxlen, dimensions)); VectorArray res = palloc(sizeof(VectorArrayData));
res->length = 0; res->length = 0;
res->maxlen = maxlen; res->maxlen = maxlen;
res->dim = dimensions; res->dim = dimensions;
res->items = palloc_extended(maxlen * VECTOR_SIZE(dimensions), MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
return res; return res;
} }
/*
* Free a vector array
*/
void
VectorArrayFree(VectorArray arr)
{
pfree(arr->items);
pfree(arr);
}
/* /*
* Print vector array - useful for debugging * Print vector array - useful for debugging
*/ */

View File

@@ -13,7 +13,10 @@
#include "utils/numeric.h" #include "utils/numeric.h"
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
#include "common/shortest_dec.h"
#include "utils/float.h" #include "utils/float.h"
#else
#include <float.h>
#endif #endif
#if PG_VERSION_NUM < 130000 #if PG_VERSION_NUM < 130000
@@ -106,14 +109,14 @@ PrintVector(char *msg, Vector * vector)
/* /*
* Convert textual representation to internal representation * Convert textual representation to internal representation
*/ */
PG_FUNCTION_INFO_V1(vector_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
Datum Datum
vector_in(PG_FUNCTION_ARGS) vector_in(PG_FUNCTION_ARGS)
{ {
char *str = PG_GETARG_CSTRING(0); char *str = PG_GETARG_CSTRING(0);
int32 typmod = PG_GETARG_INT32(2); int32 typmod = PG_GETARG_INT32(2);
int i; int i;
double x[VECTOR_MAX_DIM]; float x[VECTOR_MAX_DIM];
int dim = 0; int dim = 0;
char *pt; char *pt;
char *stringEnd; char *stringEnd;
@@ -136,7 +139,8 @@ vector_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM))); errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
x[dim] = strtod(pt, &stringEnd); /* Use strtof like float4in to avoid a double-rounding problem */
x[dim] = strtof(pt, &stringEnd);
CheckElement(x[dim]); CheckElement(x[dim]);
dim++; dim++;
@@ -182,35 +186,68 @@ vector_in(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to textual representation * Convert internal representation to textual representation
*/ */
PG_FUNCTION_INFO_V1(vector_out); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
Datum Datum
vector_out(PG_FUNCTION_ARGS) vector_out(PG_FUNCTION_ARGS)
{ {
Vector *vector = PG_GETARG_VECTOR_P(0); Vector *vector = PG_GETARG_VECTOR_P(0);
StringInfoData buf;
int dim = vector->dim; int dim = vector->dim;
char *buf;
char *ptr;
int i; int i;
int n;
initStringInfo(&buf); #if PG_VERSION_NUM < 120000
int ndig = FLT_DIG + extra_float_digits;
appendStringInfoChar(&buf, '['); if (ndig < 1)
ndig = 1;
#define FLOAT_SHORTEST_DECIMAL_LEN (ndig + 10)
#endif
/*
* Need:
*
* dim * (FLOAT_SHORTEST_DECIMAL_LEN - 1) bytes for
* float_to_shortest_decimal_bufn
*
* dim - 1 bytes for separator
*
* 3 bytes for [, ], and \0
*/
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
ptr = buf;
*ptr = '[';
ptr++;
for (i = 0; i < dim; i++) for (i = 0; i < dim; i++)
{ {
if (i > 0) if (i > 0)
appendStringInfoString(&buf, ","); {
*ptr = ',';
appendStringInfoString(&buf, float8out_internal(vector->x[i])); ptr++;
} }
appendStringInfoChar(&buf, ']');
#if PG_VERSION_NUM >= 120000
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
#else
n = sprintf(ptr, "%.*g", ndig, vector->x[i]);
#endif
ptr += n;
}
*ptr = ']';
ptr++;
*ptr = '\0';
PG_FREE_IF_COPY(vector, 0); PG_FREE_IF_COPY(vector, 0);
PG_RETURN_CSTRING(buf.data); PG_RETURN_CSTRING(buf);
} }
/* /*
* Convert type modifier * Convert type modifier
*/ */
PG_FUNCTION_INFO_V1(vector_typmod_in); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
Datum Datum
vector_typmod_in(PG_FUNCTION_ARGS) vector_typmod_in(PG_FUNCTION_ARGS)
{ {
@@ -241,7 +278,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
/* /*
* Convert external binary representation to internal representation * Convert external binary representation to internal representation
*/ */
PG_FUNCTION_INFO_V1(vector_recv); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
Datum Datum
vector_recv(PG_FUNCTION_ARGS) vector_recv(PG_FUNCTION_ARGS)
{ {
@@ -273,7 +310,7 @@ vector_recv(PG_FUNCTION_ARGS)
/* /*
* Convert internal representation to the external binary representation * Convert internal representation to the external binary representation
*/ */
PG_FUNCTION_INFO_V1(vector_send); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
Datum Datum
vector_send(PG_FUNCTION_ARGS) vector_send(PG_FUNCTION_ARGS)
{ {
@@ -293,7 +330,7 @@ vector_send(PG_FUNCTION_ARGS)
/* /*
* Convert vector to vector * Convert vector to vector
*/ */
PG_FUNCTION_INFO_V1(vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
Datum Datum
vector(PG_FUNCTION_ARGS) vector(PG_FUNCTION_ARGS)
{ {
@@ -308,7 +345,7 @@ vector(PG_FUNCTION_ARGS)
/* /*
* Convert array to vector * Convert array to vector
*/ */
PG_FUNCTION_INFO_V1(array_to_vector); PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
Datum Datum
array_to_vector(PG_FUNCTION_ARGS) array_to_vector(PG_FUNCTION_ARGS)
{ {
@@ -366,7 +403,7 @@ array_to_vector(PG_FUNCTION_ARGS)
/* /*
* Convert vector to float4[] * Convert vector to float4[]
*/ */
PG_FUNCTION_INFO_V1(vector_to_float4); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
Datum Datum
vector_to_float4(PG_FUNCTION_ARGS) vector_to_float4(PG_FUNCTION_ARGS)
{ {
@@ -389,12 +426,14 @@ vector_to_float4(PG_FUNCTION_ARGS)
/* /*
* Get the L2 distance between vectors * Get the L2 distance between vectors
*/ */
PG_FUNCTION_INFO_V1(l2_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
Datum Datum
l2_distance(PG_FUNCTION_ARGS) l2_distance(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0; double distance = 0.0;
double diff; double diff;
@@ -402,7 +441,7 @@ l2_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
diff = a->x[i] - b->x[i]; diff = ax[i] - bx[i];
distance += diff * diff; distance += diff * diff;
} }
@@ -413,12 +452,14 @@ l2_distance(PG_FUNCTION_ARGS)
* Get the L2 squared distance between vectors * Get the L2 squared distance between vectors
* This saves a sqrt calculation * This saves a sqrt calculation
*/ */
PG_FUNCTION_INFO_V1(vector_l2_squared_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
Datum Datum
vector_l2_squared_distance(PG_FUNCTION_ARGS) vector_l2_squared_distance(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0; double distance = 0.0;
double diff; double diff;
@@ -426,7 +467,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
diff = a->x[i] - b->x[i]; diff = ax[i] - bx[i];
distance += diff * diff; distance += diff * diff;
} }
@@ -436,18 +477,20 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
/* /*
* Get the inner product of two vectors * Get the inner product of two vectors
*/ */
PG_FUNCTION_INFO_V1(inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
Datum Datum
inner_product(PG_FUNCTION_ARGS) inner_product(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0; double distance = 0.0;
CheckDims(a, b); CheckDims(a, b);
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += a->x[i] * b->x[i]; distance += ax[i] * bx[i];
PG_RETURN_FLOAT8(distance); PG_RETURN_FLOAT8(distance);
} }
@@ -455,18 +498,20 @@ inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the negative inner product of two vectors * Get the negative inner product of two vectors
*/ */
PG_FUNCTION_INFO_V1(vector_negative_inner_product); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
Datum Datum
vector_negative_inner_product(PG_FUNCTION_ARGS) vector_negative_inner_product(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0; double distance = 0.0;
CheckDims(a, b); CheckDims(a, b);
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
distance += a->x[i] * b->x[i]; distance += ax[i] * bx[i];
PG_RETURN_FLOAT8(distance * -1); PG_RETURN_FLOAT8(distance * -1);
} }
@@ -474,12 +519,14 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
/* /*
* Get the cosine distance between two vectors * Get the cosine distance between two vectors
*/ */
PG_FUNCTION_INFO_V1(cosine_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
Datum Datum
cosine_distance(PG_FUNCTION_ARGS) cosine_distance(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
double distance = 0.0; double distance = 0.0;
double norma = 0.0; double norma = 0.0;
double normb = 0.0; double normb = 0.0;
@@ -488,9 +535,9 @@ cosine_distance(PG_FUNCTION_ARGS)
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
{ {
distance += a->x[i] * b->x[i]; distance += ax[i] * bx[i];
norma += a->x[i] * a->x[i]; norma += ax[i] * ax[i];
normb += b->x[i] * b->x[i]; normb += bx[i] * bx[i];
} }
PG_RETURN_FLOAT8(1 - (distance / (sqrt(norma) * sqrt(normb)))); PG_RETURN_FLOAT8(1 - (distance / (sqrt(norma) * sqrt(normb))));
@@ -501,7 +548,7 @@ cosine_distance(PG_FUNCTION_ARGS)
* Currently uses angular distance since needs to satisfy triangle inequality * Currently uses angular distance since needs to satisfy triangle inequality
* Assumes inputs are unit vectors (skips norm) * Assumes inputs are unit vectors (skips norm)
*/ */
PG_FUNCTION_INFO_V1(vector_spherical_distance); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
Datum Datum
vector_spherical_distance(PG_FUNCTION_ARGS) vector_spherical_distance(PG_FUNCTION_ARGS)
{ {
@@ -526,7 +573,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
/* /*
* Get the dimensions of a vector * Get the dimensions of a vector
*/ */
PG_FUNCTION_INFO_V1(vector_dims); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
Datum Datum
vector_dims(PG_FUNCTION_ARGS) vector_dims(PG_FUNCTION_ARGS)
{ {
@@ -538,15 +585,16 @@ vector_dims(PG_FUNCTION_ARGS)
/* /*
* Get the L2 norm of a vector * Get the L2 norm of a vector
*/ */
PG_FUNCTION_INFO_V1(vector_norm); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
Datum Datum
vector_norm(PG_FUNCTION_ARGS) vector_norm(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
float *ax = a->x;
double norm = 0.0; double norm = 0.0;
for (int i = 0; i < a->dim; i++) for (int i = 0; i < a->dim; i++)
norm += a->x[i] * a->x[i]; norm += ax[i] * ax[i];
PG_RETURN_FLOAT8(sqrt(norm)); PG_RETURN_FLOAT8(sqrt(norm));
} }
@@ -554,20 +602,23 @@ vector_norm(PG_FUNCTION_ARGS)
/* /*
* Add vectors * Add vectors
*/ */
PG_FUNCTION_INFO_V1(vector_add); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
Datum Datum
vector_add(PG_FUNCTION_ARGS) vector_add(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
Vector *result; Vector *result;
int i; float *rx;
CheckDims(a, b); CheckDims(a, b);
result = InitVector(a->dim); result = InitVector(a->dim);
for (i = 0; i < a->dim; i++) rx = result->x;
result->x[i] = a->x[i] + b->x[i]; for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] + bx[i];
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -575,20 +626,23 @@ vector_add(PG_FUNCTION_ARGS)
/* /*
* Subtract vectors * Subtract vectors
*/ */
PG_FUNCTION_INFO_V1(vector_sub); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
Datum Datum
vector_sub(PG_FUNCTION_ARGS) vector_sub(PG_FUNCTION_ARGS)
{ {
Vector *a = PG_GETARG_VECTOR_P(0); Vector *a = PG_GETARG_VECTOR_P(0);
Vector *b = PG_GETARG_VECTOR_P(1); Vector *b = PG_GETARG_VECTOR_P(1);
float *ax = a->x;
float *bx = b->x;
Vector *result; Vector *result;
int i; float *rx;
CheckDims(a, b); CheckDims(a, b);
result = InitVector(a->dim); result = InitVector(a->dim);
for (i = 0; i < a->dim; i++) rx = result->x;
result->x[i] = a->x[i] - b->x[i]; for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] - bx[i];
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -617,7 +671,7 @@ vector_cmp_internal(Vector * a, Vector * b)
/* /*
* Less than * Less than
*/ */
PG_FUNCTION_INFO_V1(vector_lt); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
Datum Datum
vector_lt(PG_FUNCTION_ARGS) vector_lt(PG_FUNCTION_ARGS)
{ {
@@ -630,7 +684,7 @@ vector_lt(PG_FUNCTION_ARGS)
/* /*
* Less than or equal * Less than or equal
*/ */
PG_FUNCTION_INFO_V1(vector_le); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
Datum Datum
vector_le(PG_FUNCTION_ARGS) vector_le(PG_FUNCTION_ARGS)
{ {
@@ -643,7 +697,7 @@ vector_le(PG_FUNCTION_ARGS)
/* /*
* Equal * Equal
*/ */
PG_FUNCTION_INFO_V1(vector_eq); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
Datum Datum
vector_eq(PG_FUNCTION_ARGS) vector_eq(PG_FUNCTION_ARGS)
{ {
@@ -656,7 +710,7 @@ vector_eq(PG_FUNCTION_ARGS)
/* /*
* Not equal * Not equal
*/ */
PG_FUNCTION_INFO_V1(vector_ne); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
Datum Datum
vector_ne(PG_FUNCTION_ARGS) vector_ne(PG_FUNCTION_ARGS)
{ {
@@ -669,7 +723,7 @@ vector_ne(PG_FUNCTION_ARGS)
/* /*
* Greater than or equal * Greater than or equal
*/ */
PG_FUNCTION_INFO_V1(vector_ge); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
Datum Datum
vector_ge(PG_FUNCTION_ARGS) vector_ge(PG_FUNCTION_ARGS)
{ {
@@ -682,7 +736,7 @@ vector_ge(PG_FUNCTION_ARGS)
/* /*
* Greater than * Greater than
*/ */
PG_FUNCTION_INFO_V1(vector_gt); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
Datum Datum
vector_gt(PG_FUNCTION_ARGS) vector_gt(PG_FUNCTION_ARGS)
{ {
@@ -695,7 +749,7 @@ vector_gt(PG_FUNCTION_ARGS)
/* /*
* Compare vectors * Compare vectors
*/ */
PG_FUNCTION_INFO_V1(vector_cmp); PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
Datum Datum
vector_cmp(PG_FUNCTION_ARGS) vector_cmp(PG_FUNCTION_ARGS)
{ {

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT ARRAY[1,2,3]::vector; SELECT ARRAY[1,2,3]::vector;
array array
--------- ---------

View File

@@ -1,10 +1,8 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3)); CREATE TABLE t2 (val vector(3));
\copy t TO '/tmp/data.bin' WITH (FORMAT binary) \copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary) \copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val; SELECT * FROM t2 ORDER BY val;
val val
--------- ---------

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector + '[4,5,6]'; SELECT '[1,2,3]'::vector + '[4,5,6]';
?column? ?column?
---------- ----------

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector; SELECT '[1,2,3]'::vector;
vector vector
--------- ---------
@@ -12,6 +10,12 @@ SELECT '[-1,2,3]'::vector;
[-1,2,3] [-1,2,3]
(1 row) (1 row)
SELECT '[1.23456]'::vector;
vector
-----------
[1.23456]
(1 row)
SELECT '[hello,1]'::vector; SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "hello" ERROR: invalid input syntax for type vector: "hello"
LINE 1: SELECT '[hello,1]'::vector; LINE 1: SELECT '[hello,1]'::vector;

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 0); CREATE INDEX ON t USING ivfflat (val) WITH (lists = 0);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3)); CREATE UNLOGGED TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));

View File

@@ -1,6 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT ARRAY[1,2,3]::vector; SELECT ARRAY[1,2,3]::vector;
SELECT ARRAY[1.0,2.0,3.0]::vector; SELECT ARRAY[1.0,2.0,3.0]::vector;
SELECT ARRAY[1,2,3]::float4[]::vector; SELECT ARRAY[1,2,3]::float4[]::vector;

View File

@@ -1,13 +1,10 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL); INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
CREATE TABLE t2 (val vector(3)); CREATE TABLE t2 (val vector(3));
\copy t TO '/tmp/data.bin' WITH (FORMAT binary) \copy t TO 'results/data.bin' WITH (FORMAT binary)
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary) \copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
SELECT * FROM t2 ORDER BY val; SELECT * FROM t2 ORDER BY val;

View File

@@ -1,6 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector + '[4,5,6]'; SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[1,2,3]'::vector - '[4,5,6]'; SELECT '[1,2,3]'::vector - '[4,5,6]';

View File

@@ -1,8 +1,6 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SELECT '[1,2,3]'::vector; SELECT '[1,2,3]'::vector;
SELECT '[-1,2,3]'::vector; SELECT '[-1,2,3]'::vector;
SELECT '[1.23456]'::vector;
SELECT '[hello,1]'::vector; SELECT '[hello,1]'::vector;
SELECT '[NaN,1]'::vector; SELECT '[NaN,1]'::vector;
SELECT '[Infinity,1]'::vector; SELECT '[Infinity,1]'::vector;

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE TABLE t (val vector(3)); CREATE TABLE t (val vector(3));

View File

@@ -1,5 +1,3 @@
SET client_min_messages = warning;
CREATE EXTENSION IF NOT EXISTS vector;
SET enable_seqscan = off; SET enable_seqscan = off;
CREATE UNLOGGED TABLE t (val vector(3)); CREATE UNLOGGED TABLE t (val vector(3));

View File

@@ -51,6 +51,11 @@ sub test_index_replay
return; return;
} }
# Use ARRAY[random(), random(), random(), ...] over
# SELECT array_agg(random()) FROM generate_series(1, $dim)
# to generate different values for each row
my $array_sql = join(",", ('random()') x $dim);
# Initialize primary node # Initialize primary node
$node_primary = get_new_node('primary'); $node_primary = get_new_node('primary');
$node_primary->init(allows_streaming => 1); $node_primary->init(allows_streaming => 1);
@@ -74,7 +79,7 @@ $node_replica->start;
$node_primary->safe_psql("postgres", "CREATE EXTENSION vector;"); $node_primary->safe_psql("postgres", "CREATE EXTENSION vector;");
$node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));"); $node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
$node_primary->safe_psql("postgres", $node_primary->safe_psql("postgres",
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series(1, 100000) i;" "INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
); );
$node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);"); $node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
@@ -90,7 +95,7 @@ for my $i (1 .. 10)
test_index_replay("vacuum $i"); test_index_replay("vacuum $i");
my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000); my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000);
$node_primary->safe_psql("postgres", $node_primary->safe_psql("postgres",
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series($start, $end) i;" "INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series($start, $end) i;"
); );
test_index_replay("insert $i"); test_index_replay("insert $i");
} }

View File

@@ -2,7 +2,11 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More tests => 3; use Test::More tests => 5;
my $dim = 768;
my $array_sql = join(",", ('random()') x $dim);
# Initialize node # Initialize node
my $node = get_new_node('node'); my $node = get_new_node('node');
@@ -11,9 +15,9 @@ $node->start;
# Create table and index # Create table and index
$node->safe_psql("postgres", "CREATE EXTENSION vector;"); $node->safe_psql("postgres", "CREATE EXTENSION vector;");
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(768));"); $node->safe_psql("postgres", "CREATE TABLE tst (v vector($dim));");
$node->safe_psql("postgres", $node->safe_psql("postgres",
"INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10000) i;" "INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
); );
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);"); $node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
@@ -24,10 +28,18 @@ $node->pgbench(
[qr{^$}], [qr{^$}],
"concurrent INSERTs", "concurrent INSERTs",
{ {
"007_concurrent" => q( "007_inserts" => "INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10) i;"
BEGIN;
INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10) i;
COMMIT;
),
} }
); );
my $expected = 10000 + 5 * 100 * 10;
my $count = $node->safe_psql("postgres", "SELECT COUNT(*) FROM tst;");
is($count, $expected);
$count = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET ivfflat.probes = 100;
SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
));
is($count, $expected);

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat access method' comment = 'vector data type and ivfflat access method'
default_version = '0.3.0' default_version = '0.3.2'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true