mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 12:07:34 +08:00
Compare commits
55 Commits
v0.3.0
...
windows-pr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb127fce8d | ||
|
|
7ce4e1f53e | ||
|
|
6e41c81412 | ||
|
|
98ec05145b | ||
|
|
e302c99d4a | ||
|
|
d6ab29e772 | ||
|
|
6ad96fe8ca | ||
|
|
e52ef4f2b5 | ||
|
|
8c81213592 | ||
|
|
f77b044303 | ||
|
|
fab7b50c3a | ||
|
|
9e2e726010 | ||
|
|
2621f9f947 | ||
|
|
4fbe55f97e | ||
|
|
9c9489e888 | ||
|
|
d376011087 | ||
|
|
5107e057ad | ||
|
|
1c1812aeac | ||
|
|
da12d9d8b0 | ||
|
|
de26e1e78a | ||
|
|
8691d2ad53 | ||
|
|
573200c188 | ||
|
|
cd7cab804f | ||
|
|
84a8bdd661 | ||
|
|
7a13a6cd3c | ||
|
|
35d5cb9f06 | ||
|
|
bc6b9e4bba | ||
|
|
d605948be4 | ||
|
|
37a784d3f6 | ||
|
|
c74bb5495b | ||
|
|
cc35af160e | ||
|
|
a7f712b5a4 | ||
|
|
a532d82bda | ||
|
|
ccec96ba54 | ||
|
|
b7c5849ba7 | ||
|
|
135147893a | ||
|
|
00391953dd | ||
|
|
81bba1f37e | ||
|
|
2d8b7e5f81 | ||
|
|
cebae5e5ff | ||
|
|
4c3332fc4d | ||
|
|
386a3b5dd5 | ||
|
|
fb819eb8b2 | ||
|
|
3f8407460e | ||
|
|
059e88dda5 | ||
|
|
12f158d4bb | ||
|
|
2a3c0a769a | ||
|
|
349d844fa1 | ||
|
|
a57ca81d8f | ||
|
|
dc4a79a7e7 | ||
|
|
1652a130c5 | ||
|
|
a6743bd206 | ||
|
|
e25627074f | ||
|
|
b3cad93f3a | ||
|
|
379a76098e |
87
.github/workflows/build.yml
vendored
87
.github/workflows/build.yml
vendored
@@ -1,38 +1,65 @@
|
|||||||
name: build
|
name: build
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
ubuntu:
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ubuntu-latest
|
||||||
|
if: ${{ !startsWith(github.ref_name, 'windows') }}
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-latest]
|
|
||||||
postgres: [15, 14, 13, 12, 11, 10]
|
postgres: [15, 14, 13, 12, 11, 10]
|
||||||
include:
|
|
||||||
- os: macos-latest
|
|
||||||
postgres: 14
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: ankane/setup-postgres@v1
|
- uses: ankane/setup-postgres@v1
|
||||||
with:
|
with:
|
||||||
postgres-version: ${{ matrix.postgres }}
|
postgres-version: ${{ matrix.postgres }}
|
||||||
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
|
dev-files: true
|
||||||
run: sudo apt-get update && sudo apt-get install postgresql-server-dev-${{ matrix.postgres }} libipc-run-perl
|
- run: make
|
||||||
- run: make
|
- run: |
|
||||||
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
|
export PG_CONFIG=`which pg_config`
|
||||||
run: |
|
sudo --preserve-env=PG_CONFIG make install
|
||||||
export PG_CONFIG=`which pg_config`
|
- run: make installcheck
|
||||||
sudo --preserve-env=PG_CONFIG make install
|
- if: ${{ failure() }}
|
||||||
- if: ${{ startsWith(matrix.os, 'macos') }}
|
run: cat regression.diffs
|
||||||
run: make install
|
- run: |
|
||||||
- run: make installcheck
|
sudo apt-get update
|
||||||
- if: ${{ failure() }}
|
sudo apt-get install libipc-run-perl
|
||||||
run: cat regression.diffs
|
make prove_installcheck
|
||||||
- if: ${{ startsWith(matrix.os, 'ubuntu') }}
|
mac:
|
||||||
run: make prove_installcheck
|
runs-on: macos-latest
|
||||||
- if: ${{ startsWith(matrix.os, 'macos') }}
|
if: ${{ !startsWith(github.ref_name, 'windows') }}
|
||||||
run: |
|
steps:
|
||||||
brew install cpanm && cpanm IPC::Run
|
- uses: actions/checkout@v3
|
||||||
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
|
- uses: ankane/setup-postgres@v1
|
||||||
tar xf REL_14_5.tar.gz
|
with:
|
||||||
make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
|
postgres-version: 14
|
||||||
|
- run: make
|
||||||
|
- run: make install
|
||||||
|
- run: make installcheck
|
||||||
|
- if: ${{ failure() }}
|
||||||
|
run: cat regression.diffs
|
||||||
|
- run: |
|
||||||
|
brew install cpanm
|
||||||
|
cpanm IPC::Run
|
||||||
|
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
|
||||||
|
tar xf REL_14_5.tar.gz
|
||||||
|
make prove_installcheck PROVE=prove PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
|
||||||
|
windows:
|
||||||
|
runs-on: windows-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- uses: ankane/setup-postgres@v1
|
||||||
|
with:
|
||||||
|
postgres-version: 14
|
||||||
|
- run: |
|
||||||
|
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
|
||||||
|
nmake /NOLOGO /F Makefile.win
|
||||||
|
nmake /NOLOGO /F Makefile.win install
|
||||||
|
curl -Ls -o REL_14_5.tar.gz https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
|
||||||
|
7z x REL_14_5.tar.gz
|
||||||
|
7z x REL_14_5.tar
|
||||||
|
ls ./postgres-REL_14_5/src/test/perl
|
||||||
|
set PROVE=prove
|
||||||
|
set PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl"
|
||||||
|
nmake /NOLOGO /F Makefile.win prove_installcheck
|
||||||
|
shell: cmd
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -5,3 +5,4 @@
|
|||||||
regression.*
|
regression.*
|
||||||
*.o
|
*.o
|
||||||
*.so
|
*.so
|
||||||
|
*.bc
|
||||||
|
|||||||
17
CHANGELOG.md
17
CHANGELOG.md
@@ -1,3 +1,20 @@
|
|||||||
|
## 0.4.0 (unreleased)
|
||||||
|
|
||||||
|
- Changed text representation for vector elements to match `real`
|
||||||
|
- Improved accuracy of text parsing for certain inputs
|
||||||
|
- Added experimental support for Windows
|
||||||
|
|
||||||
|
## 0.3.2 (2022-11-22)
|
||||||
|
|
||||||
|
- Fixed `invalid memory alloc request size` error
|
||||||
|
|
||||||
|
## 0.3.1 (2022-11-02)
|
||||||
|
|
||||||
|
If upgrading from 0.2.7 or 0.3.0, [recreate](https://github.com/pgvector/pgvector#031) all `ivfflat` indexes after upgrading to ensure all data is indexed.
|
||||||
|
|
||||||
|
- Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7)
|
||||||
|
- Fixed segmentation fault with index creation when lists > 6500
|
||||||
|
|
||||||
## 0.3.0 (2022-10-15)
|
## 0.3.0 (2022-10-15)
|
||||||
|
|
||||||
- Added support for Postgres 15
|
- Added support for Postgres 15
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"name": "vector",
|
"name": "vector",
|
||||||
"abstract": "Open-source vector similarity search for Postgres",
|
"abstract": "Open-source vector similarity search for Postgres",
|
||||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||||
"version": "0.3.0",
|
"version": "0.3.2",
|
||||||
"maintainer": [
|
"maintainer": [
|
||||||
"Andrew Kane <andrew@ankane.org>"
|
"Andrew Kane <andrew@ankane.org>"
|
||||||
],
|
],
|
||||||
@@ -12,7 +12,7 @@
|
|||||||
"prereqs": {
|
"prereqs": {
|
||||||
"runtime": {
|
"runtime": {
|
||||||
"requires": {
|
"requires": {
|
||||||
"PostgreSQL": "10.0"
|
"PostgreSQL": "10.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
"vector": {
|
"vector": {
|
||||||
"file": "sql/vector.sql",
|
"file": "sql/vector.sql",
|
||||||
"docfile": "README.md",
|
"docfile": "README.md",
|
||||||
"version": "0.3.0",
|
"version": "0.3.2",
|
||||||
"abstract": "Open-source vector similarity search for Postgres"
|
"abstract": "Open-source vector similarity search for Postgres"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
4
Makefile
4
Makefile
@@ -1,5 +1,5 @@
|
|||||||
EXTENSION = vector
|
EXTENSION = vector
|
||||||
EXTVERSION = 0.3.0
|
EXTVERSION = 0.3.2
|
||||||
|
|
||||||
MODULE_big = vector
|
MODULE_big = vector
|
||||||
DATA = $(wildcard sql/*--*.sql)
|
DATA = $(wildcard sql/*--*.sql)
|
||||||
@@ -7,7 +7,7 @@ OBJS = src/ivfbuild.o src/ivfflat.o src/ivfinsert.o src/ivfkmeans.o src/ivfscan.
|
|||||||
|
|
||||||
TESTS = $(wildcard test/sql/*.sql)
|
TESTS = $(wildcard test/sql/*.sql)
|
||||||
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
|
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
|
||||||
REGRESS_OPTS = --inputdir=test
|
REGRESS_OPTS = --inputdir=test --load-extension=vector
|
||||||
|
|
||||||
OPTFLAGS = -march=native
|
OPTFLAGS = -march=native
|
||||||
|
|
||||||
|
|||||||
62
Makefile.win
Normal file
62
Makefile.win
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
EXTENSION = vector
|
||||||
|
EXTVERSION = 0.3.2
|
||||||
|
|
||||||
|
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||||
|
|
||||||
|
REGRESS = btree cast copy functions input ivfflat_cosine ivfflat_ip ivfflat_l2 ivfflat_options ivfflat_unlogged
|
||||||
|
REGRESS_OPTS = --inputdir=test --load-extension=vector
|
||||||
|
|
||||||
|
# For /arch flags
|
||||||
|
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-minimum-cpu-architecture
|
||||||
|
OPTFLAGS =
|
||||||
|
|
||||||
|
# For auto-vectorization:
|
||||||
|
# - MSVC (needs /O2 /fp:fast) - https://learn.microsoft.com/en-us/cpp/parallel/auto-parallelization-and-auto-vectorization?#auto-vectorizer
|
||||||
|
PG_CFLAGS = $(PG_CFLAGS) $(OPTFLAGS) /O2 /fp:fast
|
||||||
|
|
||||||
|
# Debug MSVC auto-vectorization
|
||||||
|
# https://learn.microsoft.com/en-us/cpp/error-messages/tool-errors/vectorizer-and-parallelizer-messages
|
||||||
|
# PG_CFLAGS = $(PG_CFLAGS) /Qvec-report:2
|
||||||
|
|
||||||
|
all: sql\$(EXTENSION)--$(EXTVERSION).sql
|
||||||
|
|
||||||
|
sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql
|
||||||
|
copy sql\$(EXTENSION).sql $@
|
||||||
|
|
||||||
|
# TODO use pg_config
|
||||||
|
BINDIR = $(PGROOT)\bin
|
||||||
|
INCLUDEDIR = $(PGROOT)\include
|
||||||
|
INCLUDEDIR_SERVER = $(PGROOT)\include\server
|
||||||
|
LIBDIR = $(PGROOT)\lib
|
||||||
|
PKGLIBDIR = $(PGROOT)\lib
|
||||||
|
SHAREDIR = $(PGROOT)\share
|
||||||
|
|
||||||
|
CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)"
|
||||||
|
|
||||||
|
CFLAGS = $(CFLAGS) $(PG_CFLAGS)
|
||||||
|
|
||||||
|
SHLIB = src\$(EXTENSION).dll
|
||||||
|
|
||||||
|
LIBS = "$(LIBDIR)\postgres.lib"
|
||||||
|
|
||||||
|
.c.obj:
|
||||||
|
$(CC) $(CFLAGS) /c $< /Fo$@
|
||||||
|
|
||||||
|
$(SHLIB): $(OBJS)
|
||||||
|
$(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB)
|
||||||
|
|
||||||
|
all: $(SHLIB)
|
||||||
|
|
||||||
|
install:
|
||||||
|
copy $(SHLIB) "$(PKGLIBDIR)"
|
||||||
|
copy $(EXTENSION).control "$(SHAREDIR)\extension"
|
||||||
|
copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension"
|
||||||
|
|
||||||
|
installcheck:
|
||||||
|
"$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS)
|
||||||
|
|
||||||
|
prove_installcheck:
|
||||||
|
rm -rf tmp_check
|
||||||
|
set PGPORT=65432
|
||||||
|
set PG_REGRESS="$(BINDIR)\pg_regress"
|
||||||
|
$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) test/t/*.pl
|
||||||
22
README.md
22
README.md
@@ -17,7 +17,7 @@ Supports L2 distance, inner product, and cosine distance
|
|||||||
Compile and install the extension (supports Postgres 10+)
|
Compile and install the extension (supports Postgres 10+)
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
|
git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
|
||||||
cd pgvector
|
cd pgvector
|
||||||
make
|
make
|
||||||
make install # may need sudo
|
make install # may need sudo
|
||||||
@@ -164,7 +164,7 @@ CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000
|
|||||||
|
|
||||||
### Vector Type
|
### Vector Type
|
||||||
|
|
||||||
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a float, and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions.
|
Each vector takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (like the `real` type in Postgres), and all elements must be finite (no `NaN`, `Infinity` or `-Infinity`). Vectors can have up to 1024 dimensions.
|
||||||
|
|
||||||
### Vector Operators
|
### Vector Operators
|
||||||
|
|
||||||
@@ -232,7 +232,7 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres).
|
|||||||
You can also build the image manually
|
You can also build the image manually
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git clone --branch v0.3.0 https://github.com/pgvector/pgvector.git
|
git clone --branch v0.3.2 https://github.com/pgvector/pgvector.git
|
||||||
cd pgvector
|
cd pgvector
|
||||||
docker build -t pgvector .
|
docker build -t pgvector .
|
||||||
```
|
```
|
||||||
@@ -270,6 +270,22 @@ Install the latest version and run:
|
|||||||
ALTER EXTENSION vector UPDATE;
|
ALTER EXTENSION vector UPDATE;
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Upgrade Notes
|
||||||
|
|
||||||
|
### 0.3.1
|
||||||
|
|
||||||
|
If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Postgres 12+
|
||||||
|
REINDEX INDEX CONCURRENTLY index_name;
|
||||||
|
|
||||||
|
-- Postgres < 12
|
||||||
|
CREATE INDEX CONCURRENTLY temp_name ON table USING ivfflat (column opclass);
|
||||||
|
DROP INDEX CONCURRENTLY index_name;
|
||||||
|
ALTER INDEX temp_name RENAME TO index_name;
|
||||||
|
```
|
||||||
|
|
||||||
## Thanks
|
## Thanks
|
||||||
|
|
||||||
Thanks to:
|
Thanks to:
|
||||||
|
|||||||
2
sql/vector--0.3.0--0.3.1.sql
Normal file
2
sql/vector--0.3.0--0.3.1.sql
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||||
|
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.1'" to load this file. \quit
|
||||||
2
sql/vector--0.3.1--0.3.2.sql
Normal file
2
sql/vector--0.3.1--0.3.2.sql
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||||
|
\echo Use "ALTER EXTENSION vector UPDATE TO '0.3.2'" to load this file. \quit
|
||||||
@@ -107,7 +107,7 @@ SampleRows(IvfflatBuildState * buildstate)
|
|||||||
|
|
||||||
buildstate->rowstoskip = -1;
|
buildstate->rowstoskip = -1;
|
||||||
|
|
||||||
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, random());
|
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
|
||||||
|
|
||||||
reservoir_init_selection_state(&buildstate->rstate, targsamples);
|
reservoir_init_selection_state(&buildstate->rstate, targsamples);
|
||||||
while (BlockSampler_HasMore(&buildstate->bs))
|
while (BlockSampler_HasMore(&buildstate->bs))
|
||||||
@@ -233,8 +233,8 @@ InsertTuples(Relation index, IvfflatBuildState * buildstate, ForkNumber forkNum)
|
|||||||
GenericXLogState *state;
|
GenericXLogState *state;
|
||||||
int list;
|
int list;
|
||||||
IndexTuple itup = NULL; /* silence compiler warning */
|
IndexTuple itup = NULL; /* silence compiler warning */
|
||||||
BlockNumber startPage = InvalidBlockNumber;
|
BlockNumber startPage;
|
||||||
BlockNumber insertPage = InvalidBlockNumber;
|
BlockNumber insertPage;
|
||||||
Size itemsz;
|
Size itemsz;
|
||||||
int i;
|
int i;
|
||||||
int64 inserted = 0;
|
int64 inserted = 0;
|
||||||
@@ -362,7 +362,7 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
|
|||||||
static void
|
static void
|
||||||
FreeBuildState(IvfflatBuildState * buildstate)
|
FreeBuildState(IvfflatBuildState * buildstate)
|
||||||
{
|
{
|
||||||
pfree(buildstate->centers);
|
VectorArrayFree(buildstate->centers);
|
||||||
pfree(buildstate->listInfo);
|
pfree(buildstate->listInfo);
|
||||||
pfree(buildstate->normvec);
|
pfree(buildstate->normvec);
|
||||||
|
|
||||||
@@ -401,7 +401,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
|
|||||||
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
|
IvfflatBench("k-means", IvfflatKmeans(buildstate->index, buildstate->samples, buildstate->centers));
|
||||||
|
|
||||||
/* Free samples before we allocate more memory */
|
/* Free samples before we allocate more memory */
|
||||||
pfree(buildstate->samples);
|
VectorArrayFree(buildstate->samples);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ ivfflatvalidate(Oid opclassoid)
|
|||||||
*
|
*
|
||||||
* See https://www.postgresql.org/docs/current/index-api.html
|
* See https://www.postgresql.org/docs/current/index-api.html
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(ivfflathandler);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(ivfflathandler);
|
||||||
Datum
|
Datum
|
||||||
ivfflathandler(PG_FUNCTION_ARGS)
|
ivfflathandler(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -3,19 +3,24 @@
|
|||||||
|
|
||||||
#include "postgres.h"
|
#include "postgres.h"
|
||||||
|
|
||||||
|
#if PG_VERSION_NUM < 100000
|
||||||
|
#error "Requires PostgreSQL 10+"
|
||||||
|
#endif
|
||||||
|
|
||||||
#include "access/generic_xlog.h"
|
#include "access/generic_xlog.h"
|
||||||
#include "access/reloptions.h"
|
#include "access/reloptions.h"
|
||||||
#include "nodes/execnodes.h"
|
#include "nodes/execnodes.h"
|
||||||
|
#include "port.h" /* for strtof() and random() */
|
||||||
#include "utils/sampling.h"
|
#include "utils/sampling.h"
|
||||||
#include "utils/tuplesort.h"
|
#include "utils/tuplesort.h"
|
||||||
#include "vector.h"
|
#include "vector.h"
|
||||||
|
|
||||||
#ifdef IVFFLAT_BENCH
|
#if PG_VERSION_NUM >= 150000
|
||||||
#include "portability/instr_time.h"
|
#include "common/pg_prng.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if PG_VERSION_NUM < 100000
|
#ifdef IVFFLAT_BENCH
|
||||||
#error "Requires PostgreSQL 10+"
|
#include "portability/instr_time.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* Support functions */
|
/* Support functions */
|
||||||
@@ -62,15 +67,26 @@
|
|||||||
#define IvfflatBench(name, code) (code)
|
#define IvfflatBench(name, code) (code)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if PG_VERSION_NUM >= 150000
|
||||||
|
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
|
||||||
|
#define RandomInt() pg_prng_uint32(&pg_global_prng_state)
|
||||||
|
#else
|
||||||
|
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
|
||||||
|
#define RandomInt() random()
|
||||||
|
#endif
|
||||||
|
|
||||||
/* Variables */
|
/* Variables */
|
||||||
extern int ivfflat_probes;
|
extern int ivfflat_probes;
|
||||||
|
|
||||||
|
/* Exported functions */
|
||||||
|
PGDLLEXPORT void _PG_init(void);
|
||||||
|
|
||||||
typedef struct VectorArrayData
|
typedef struct VectorArrayData
|
||||||
{
|
{
|
||||||
int length;
|
int length;
|
||||||
int maxlen;
|
int maxlen;
|
||||||
int dim;
|
int dim;
|
||||||
Vector items[FLEXIBLE_ARRAY_MEMBER];
|
Vector *items;
|
||||||
} VectorArrayData;
|
} VectorArrayData;
|
||||||
|
|
||||||
typedef VectorArrayData * VectorArray;
|
typedef VectorArrayData * VectorArray;
|
||||||
@@ -191,14 +207,14 @@ typedef struct IvfflatScanOpaqueData
|
|||||||
|
|
||||||
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
|
typedef IvfflatScanOpaqueData * IvfflatScanOpaque;
|
||||||
|
|
||||||
#define VECTOR_ARRAY_SIZE(_length, _dim) (offsetof(VectorArrayData, items) + _length * VECTOR_SIZE(_dim))
|
#define VECTOR_ARRAY_SIZE(_length, _dim) (sizeof(VectorArrayData) + (_length) * VECTOR_SIZE(_dim))
|
||||||
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) _arr + offsetof(VectorArrayData, items) + (_offset) * VECTOR_SIZE(_arr->dim))
|
#define VECTOR_ARRAY_OFFSET(_arr, _offset) ((char*) (_arr)->items + (_offset) * VECTOR_SIZE((_arr)->dim))
|
||||||
#define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset))
|
#define VectorArrayGet(_arr, _offset) ((Vector *) VECTOR_ARRAY_OFFSET(_arr, _offset))
|
||||||
#define VectorArraySet(_arr, _offset, _val) (memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE(_arr->dim)))
|
#define VectorArraySet(_arr, _offset, _val) memcpy(VECTOR_ARRAY_OFFSET(_arr, _offset), _val, VECTOR_SIZE((_arr)->dim))
|
||||||
|
|
||||||
/* Methods */
|
/* Methods */
|
||||||
void _PG_init(void);
|
|
||||||
VectorArray VectorArrayInit(int maxlen, int dimensions);
|
VectorArray VectorArrayInit(int maxlen, int dimensions);
|
||||||
|
void VectorArrayFree(VectorArray arr);
|
||||||
void PrintVectorArray(char *msg, VectorArray arr);
|
void PrintVectorArray(char *msg, VectorArray arr);
|
||||||
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
|
void IvfflatKmeans(Relation index, VectorArray samples, VectorArray centers);
|
||||||
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);
|
FmgrInfo *IvfflatOptionalProcInfo(Relation rel, uint16 procnum);
|
||||||
|
|||||||
@@ -97,26 +97,42 @@ InsertTuple(Relation rel, IndexTuple itup, Relation heapRel, Datum *values)
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
/* Add a new page */
|
Buffer metabuf;
|
||||||
Buffer newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
|
Buffer newbuf;
|
||||||
Page newpage = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE);
|
Page newpage;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* From ReadBufferExtended: Caller is responsible for ensuring
|
||||||
|
* that only one backend tries to extend a relation at the same
|
||||||
|
* time!
|
||||||
|
*/
|
||||||
|
metabuf = ReadBuffer(rel, IVFFLAT_METAPAGE_BLKNO);
|
||||||
|
LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE);
|
||||||
|
|
||||||
|
/* Add a new page */
|
||||||
|
newbuf = IvfflatNewBuffer(rel, MAIN_FORKNUM);
|
||||||
|
newpage = GenericXLogRegisterBuffer(state, newbuf, GENERIC_XLOG_FULL_IMAGE);
|
||||||
|
|
||||||
|
/* Init new page */
|
||||||
|
IvfflatInitPage(newbuf, newpage);
|
||||||
|
|
||||||
|
/* Update insert page */
|
||||||
insertPage = BufferGetBlockNumber(newbuf);
|
insertPage = BufferGetBlockNumber(newbuf);
|
||||||
|
|
||||||
/* Update previous buffer */
|
/* Update previous buffer */
|
||||||
IvfflatPageGetOpaque(page)->nextblkno = insertPage;
|
IvfflatPageGetOpaque(page)->nextblkno = insertPage;
|
||||||
|
|
||||||
/* Init page */
|
|
||||||
IvfflatInitPage(newbuf, newpage);
|
|
||||||
|
|
||||||
/* Commit */
|
/* Commit */
|
||||||
MarkBufferDirty(buf);
|
|
||||||
MarkBufferDirty(newbuf);
|
MarkBufferDirty(newbuf);
|
||||||
|
MarkBufferDirty(buf);
|
||||||
GenericXLogFinish(state);
|
GenericXLogFinish(state);
|
||||||
|
|
||||||
/* Unlock */
|
/* Unlock extend relation lock as early as possible */
|
||||||
UnlockReleaseBuffer(buf);
|
UnlockReleaseBuffer(metabuf);
|
||||||
|
|
||||||
|
/* Unlock rest */
|
||||||
UnlockReleaseBuffer(newbuf);
|
UnlockReleaseBuffer(newbuf);
|
||||||
|
UnlockReleaseBuffer(buf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,16 +5,6 @@
|
|||||||
#include "ivfflat.h"
|
#include "ivfflat.h"
|
||||||
#include "miscadmin.h"
|
#include "miscadmin.h"
|
||||||
|
|
||||||
#if PG_VERSION_NUM >= 150000
|
|
||||||
#include "common/pg_prng.h"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#if PG_VERSION_NUM >= 150000
|
|
||||||
#define RandomDouble() pg_prng_double(&pg_global_prng_state)
|
|
||||||
#else
|
|
||||||
#define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Initialize with kmeans++
|
* Initialize with kmeans++
|
||||||
*
|
*
|
||||||
@@ -26,7 +16,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
|
|||||||
FmgrInfo *procinfo;
|
FmgrInfo *procinfo;
|
||||||
Oid collation;
|
Oid collation;
|
||||||
int i;
|
int i;
|
||||||
int j;
|
int64 j;
|
||||||
double distance;
|
double distance;
|
||||||
double sum;
|
double sum;
|
||||||
double choice;
|
double choice;
|
||||||
@@ -39,7 +29,7 @@ InitCenters(Relation index, VectorArray samples, VectorArray centers, float *low
|
|||||||
collation = index->rd_indcollation[0];
|
collation = index->rd_indcollation[0];
|
||||||
|
|
||||||
/* Choose an initial center uniformly at random */
|
/* Choose an initial center uniformly at random */
|
||||||
VectorArraySet(centers, 0, VectorArrayGet(samples, random() % samples->length));
|
VectorArraySet(centers, 0, VectorArrayGet(samples, RandomInt() % samples->length));
|
||||||
centers->length++;
|
centers->length++;
|
||||||
|
|
||||||
for (j = 0; j < numSamples; j++)
|
for (j = 0; j < numSamples; j++)
|
||||||
@@ -182,8 +172,8 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
|||||||
Vector *vec;
|
Vector *vec;
|
||||||
Vector *newCenter;
|
Vector *newCenter;
|
||||||
int iteration;
|
int iteration;
|
||||||
int j;
|
int64 j;
|
||||||
int k;
|
int64 k;
|
||||||
int dimensions = centers->dim;
|
int dimensions = centers->dim;
|
||||||
int numCenters = centers->maxlen;
|
int numCenters = centers->maxlen;
|
||||||
int numSamples = samples->length;
|
int numSamples = samples->length;
|
||||||
@@ -227,6 +217,10 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
|||||||
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
|
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
|
||||||
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
|
totalSize / (1024 * 1024) + 1, maintenance_work_mem / 1024)));
|
||||||
|
|
||||||
|
/* Ensure indexing does not overflow */
|
||||||
|
if (numCenters * numCenters > INT_MAX)
|
||||||
|
elog(ERROR, "Indexing overflow detected. Please report a bug.");
|
||||||
|
|
||||||
/* Set support functions */
|
/* Set support functions */
|
||||||
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
|
procinfo = index_getprocinfo(index, 1, IVFFLAT_KMEANS_DISTANCE_PROC);
|
||||||
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
|
normprocinfo = IvfflatOptionalProcInfo(index, IVFFLAT_KMEANS_NORM_PROC);
|
||||||
@@ -239,7 +233,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
|||||||
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
|
lowerBound = palloc_extended(lowerBoundSize, MCXT_ALLOC_HUGE);
|
||||||
upperBound = palloc(upperBoundSize);
|
upperBound = palloc(upperBoundSize);
|
||||||
s = palloc(sSize);
|
s = palloc(sSize);
|
||||||
halfcdist = palloc(halfcdistSize);
|
halfcdist = palloc_extended(halfcdistSize, MCXT_ALLOC_HUGE);
|
||||||
newcdist = palloc(newcdistSize);
|
newcdist = palloc(newcdistSize);
|
||||||
|
|
||||||
newCenters = VectorArrayInit(numCenters, dimensions);
|
newCenters = VectorArrayInit(numCenters, dimensions);
|
||||||
@@ -259,8 +253,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
|||||||
minDistance = DBL_MAX;
|
minDistance = DBL_MAX;
|
||||||
closestCenter = -1;
|
closestCenter = -1;
|
||||||
|
|
||||||
vec = VectorArrayGet(samples, j);
|
|
||||||
|
|
||||||
/* Find closest center */
|
/* Find closest center */
|
||||||
for (k = 0; k < numCenters; k++)
|
for (k = 0; k < numCenters; k++)
|
||||||
{
|
{
|
||||||
@@ -451,7 +443,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
pfree(newCenters);
|
VectorArrayFree(newCenters);
|
||||||
pfree(centerCounts);
|
pfree(centerCounts);
|
||||||
pfree(closestCenters);
|
pfree(closestCenters);
|
||||||
pfree(lowerBound);
|
pfree(lowerBound);
|
||||||
|
|||||||
@@ -10,14 +10,25 @@
|
|||||||
VectorArray
|
VectorArray
|
||||||
VectorArrayInit(int maxlen, int dimensions)
|
VectorArrayInit(int maxlen, int dimensions)
|
||||||
{
|
{
|
||||||
VectorArray res = palloc0(VECTOR_ARRAY_SIZE(maxlen, dimensions));
|
VectorArray res = palloc(sizeof(VectorArrayData));
|
||||||
|
|
||||||
res->length = 0;
|
res->length = 0;
|
||||||
res->maxlen = maxlen;
|
res->maxlen = maxlen;
|
||||||
res->dim = dimensions;
|
res->dim = dimensions;
|
||||||
|
res->items = palloc_extended(maxlen * VECTOR_SIZE(dimensions), MCXT_ALLOC_ZERO | MCXT_ALLOC_HUGE);
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Free a vector array
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
VectorArrayFree(VectorArray arr)
|
||||||
|
{
|
||||||
|
pfree(arr->items);
|
||||||
|
pfree(arr);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Print vector array - useful for debugging
|
* Print vector array - useful for debugging
|
||||||
*/
|
*/
|
||||||
|
|||||||
150
src/vector.c
150
src/vector.c
@@ -13,7 +13,10 @@
|
|||||||
#include "utils/numeric.h"
|
#include "utils/numeric.h"
|
||||||
|
|
||||||
#if PG_VERSION_NUM >= 120000
|
#if PG_VERSION_NUM >= 120000
|
||||||
|
#include "common/shortest_dec.h"
|
||||||
#include "utils/float.h"
|
#include "utils/float.h"
|
||||||
|
#else
|
||||||
|
#include <float.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if PG_VERSION_NUM < 130000
|
#if PG_VERSION_NUM < 130000
|
||||||
@@ -106,14 +109,14 @@ PrintVector(char *msg, Vector * vector)
|
|||||||
/*
|
/*
|
||||||
* Convert textual representation to internal representation
|
* Convert textual representation to internal representation
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_in);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_in);
|
||||||
Datum
|
Datum
|
||||||
vector_in(PG_FUNCTION_ARGS)
|
vector_in(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
char *str = PG_GETARG_CSTRING(0);
|
char *str = PG_GETARG_CSTRING(0);
|
||||||
int32 typmod = PG_GETARG_INT32(2);
|
int32 typmod = PG_GETARG_INT32(2);
|
||||||
int i;
|
int i;
|
||||||
double x[VECTOR_MAX_DIM];
|
float x[VECTOR_MAX_DIM];
|
||||||
int dim = 0;
|
int dim = 0;
|
||||||
char *pt;
|
char *pt;
|
||||||
char *stringEnd;
|
char *stringEnd;
|
||||||
@@ -136,7 +139,8 @@ vector_in(PG_FUNCTION_ARGS)
|
|||||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||||
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
|
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
|
||||||
|
|
||||||
x[dim] = strtod(pt, &stringEnd);
|
/* Use strtof like float4in to avoid a double-rounding problem */
|
||||||
|
x[dim] = strtof(pt, &stringEnd);
|
||||||
CheckElement(x[dim]);
|
CheckElement(x[dim]);
|
||||||
dim++;
|
dim++;
|
||||||
|
|
||||||
@@ -182,35 +186,68 @@ vector_in(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert internal representation to textual representation
|
* Convert internal representation to textual representation
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_out);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_out);
|
||||||
Datum
|
Datum
|
||||||
vector_out(PG_FUNCTION_ARGS)
|
vector_out(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *vector = PG_GETARG_VECTOR_P(0);
|
Vector *vector = PG_GETARG_VECTOR_P(0);
|
||||||
StringInfoData buf;
|
|
||||||
int dim = vector->dim;
|
int dim = vector->dim;
|
||||||
|
char *buf;
|
||||||
|
char *ptr;
|
||||||
int i;
|
int i;
|
||||||
|
int n;
|
||||||
|
|
||||||
initStringInfo(&buf);
|
#if PG_VERSION_NUM < 120000
|
||||||
|
int ndig = FLT_DIG + extra_float_digits;
|
||||||
|
|
||||||
appendStringInfoChar(&buf, '[');
|
if (ndig < 1)
|
||||||
|
ndig = 1;
|
||||||
|
|
||||||
|
#define FLOAT_SHORTEST_DECIMAL_LEN (ndig + 10)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Need:
|
||||||
|
*
|
||||||
|
* dim * (FLOAT_SHORTEST_DECIMAL_LEN - 1) bytes for
|
||||||
|
* float_to_shortest_decimal_bufn
|
||||||
|
*
|
||||||
|
* dim - 1 bytes for separator
|
||||||
|
*
|
||||||
|
* 3 bytes for [, ], and \0
|
||||||
|
*/
|
||||||
|
buf = (char *) palloc(FLOAT_SHORTEST_DECIMAL_LEN * dim + 2);
|
||||||
|
ptr = buf;
|
||||||
|
|
||||||
|
*ptr = '[';
|
||||||
|
ptr++;
|
||||||
for (i = 0; i < dim; i++)
|
for (i = 0; i < dim; i++)
|
||||||
{
|
{
|
||||||
if (i > 0)
|
if (i > 0)
|
||||||
appendStringInfoString(&buf, ",");
|
{
|
||||||
|
*ptr = ',';
|
||||||
|
ptr++;
|
||||||
|
}
|
||||||
|
|
||||||
appendStringInfoString(&buf, float8out_internal(vector->x[i]));
|
#if PG_VERSION_NUM >= 120000
|
||||||
|
n = float_to_shortest_decimal_bufn(vector->x[i], ptr);
|
||||||
|
#else
|
||||||
|
n = sprintf(ptr, "%.*g", ndig, vector->x[i]);
|
||||||
|
#endif
|
||||||
|
ptr += n;
|
||||||
}
|
}
|
||||||
appendStringInfoChar(&buf, ']');
|
*ptr = ']';
|
||||||
|
ptr++;
|
||||||
|
*ptr = '\0';
|
||||||
|
|
||||||
PG_FREE_IF_COPY(vector, 0);
|
PG_FREE_IF_COPY(vector, 0);
|
||||||
PG_RETURN_CSTRING(buf.data);
|
PG_RETURN_CSTRING(buf);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Convert type modifier
|
* Convert type modifier
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_typmod_in);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_typmod_in);
|
||||||
Datum
|
Datum
|
||||||
vector_typmod_in(PG_FUNCTION_ARGS)
|
vector_typmod_in(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -241,7 +278,7 @@ vector_typmod_in(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert external binary representation to internal representation
|
* Convert external binary representation to internal representation
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_recv);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_recv);
|
||||||
Datum
|
Datum
|
||||||
vector_recv(PG_FUNCTION_ARGS)
|
vector_recv(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -273,7 +310,7 @@ vector_recv(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert internal representation to the external binary representation
|
* Convert internal representation to the external binary representation
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_send);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_send);
|
||||||
Datum
|
Datum
|
||||||
vector_send(PG_FUNCTION_ARGS)
|
vector_send(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -293,7 +330,7 @@ vector_send(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert vector to vector
|
* Convert vector to vector
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector);
|
||||||
Datum
|
Datum
|
||||||
vector(PG_FUNCTION_ARGS)
|
vector(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -308,7 +345,7 @@ vector(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert array to vector
|
* Convert array to vector
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(array_to_vector);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(array_to_vector);
|
||||||
Datum
|
Datum
|
||||||
array_to_vector(PG_FUNCTION_ARGS)
|
array_to_vector(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -366,7 +403,7 @@ array_to_vector(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Convert vector to float4[]
|
* Convert vector to float4[]
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_to_float4);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_to_float4);
|
||||||
Datum
|
Datum
|
||||||
vector_to_float4(PG_FUNCTION_ARGS)
|
vector_to_float4(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -389,12 +426,14 @@ vector_to_float4(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the L2 distance between vectors
|
* Get the L2 distance between vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(l2_distance);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance);
|
||||||
Datum
|
Datum
|
||||||
l2_distance(PG_FUNCTION_ARGS)
|
l2_distance(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
double distance = 0.0;
|
double distance = 0.0;
|
||||||
double diff;
|
double diff;
|
||||||
|
|
||||||
@@ -402,7 +441,7 @@ l2_distance(PG_FUNCTION_ARGS)
|
|||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
{
|
{
|
||||||
diff = a->x[i] - b->x[i];
|
diff = ax[i] - bx[i];
|
||||||
distance += diff * diff;
|
distance += diff * diff;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,12 +452,14 @@ l2_distance(PG_FUNCTION_ARGS)
|
|||||||
* Get the L2 squared distance between vectors
|
* Get the L2 squared distance between vectors
|
||||||
* This saves a sqrt calculation
|
* This saves a sqrt calculation
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_l2_squared_distance);
|
||||||
Datum
|
Datum
|
||||||
vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
double distance = 0.0;
|
double distance = 0.0;
|
||||||
double diff;
|
double diff;
|
||||||
|
|
||||||
@@ -426,7 +467,7 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
|||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
{
|
{
|
||||||
diff = a->x[i] - b->x[i];
|
diff = ax[i] - bx[i];
|
||||||
distance += diff * diff;
|
distance += diff * diff;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,18 +477,20 @@ vector_l2_squared_distance(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the inner product of two vectors
|
* Get the inner product of two vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(inner_product);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(inner_product);
|
||||||
Datum
|
Datum
|
||||||
inner_product(PG_FUNCTION_ARGS)
|
inner_product(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
double distance = 0.0;
|
double distance = 0.0;
|
||||||
|
|
||||||
CheckDims(a, b);
|
CheckDims(a, b);
|
||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
distance += a->x[i] * b->x[i];
|
distance += ax[i] * bx[i];
|
||||||
|
|
||||||
PG_RETURN_FLOAT8(distance);
|
PG_RETURN_FLOAT8(distance);
|
||||||
}
|
}
|
||||||
@@ -455,18 +498,20 @@ inner_product(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the negative inner product of two vectors
|
* Get the negative inner product of two vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_negative_inner_product);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_negative_inner_product);
|
||||||
Datum
|
Datum
|
||||||
vector_negative_inner_product(PG_FUNCTION_ARGS)
|
vector_negative_inner_product(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
double distance = 0.0;
|
double distance = 0.0;
|
||||||
|
|
||||||
CheckDims(a, b);
|
CheckDims(a, b);
|
||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
distance += a->x[i] * b->x[i];
|
distance += ax[i] * bx[i];
|
||||||
|
|
||||||
PG_RETURN_FLOAT8(distance * -1);
|
PG_RETURN_FLOAT8(distance * -1);
|
||||||
}
|
}
|
||||||
@@ -474,12 +519,14 @@ vector_negative_inner_product(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the cosine distance between two vectors
|
* Get the cosine distance between two vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(cosine_distance);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(cosine_distance);
|
||||||
Datum
|
Datum
|
||||||
cosine_distance(PG_FUNCTION_ARGS)
|
cosine_distance(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
double distance = 0.0;
|
double distance = 0.0;
|
||||||
double norma = 0.0;
|
double norma = 0.0;
|
||||||
double normb = 0.0;
|
double normb = 0.0;
|
||||||
@@ -488,9 +535,9 @@ cosine_distance(PG_FUNCTION_ARGS)
|
|||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
{
|
{
|
||||||
distance += a->x[i] * b->x[i];
|
distance += ax[i] * bx[i];
|
||||||
norma += a->x[i] * a->x[i];
|
norma += ax[i] * ax[i];
|
||||||
normb += b->x[i] * b->x[i];
|
normb += bx[i] * bx[i];
|
||||||
}
|
}
|
||||||
|
|
||||||
PG_RETURN_FLOAT8(1 - (distance / (sqrt(norma) * sqrt(normb))));
|
PG_RETURN_FLOAT8(1 - (distance / (sqrt(norma) * sqrt(normb))));
|
||||||
@@ -501,7 +548,7 @@ cosine_distance(PG_FUNCTION_ARGS)
|
|||||||
* Currently uses angular distance since needs to satisfy triangle inequality
|
* Currently uses angular distance since needs to satisfy triangle inequality
|
||||||
* Assumes inputs are unit vectors (skips norm)
|
* Assumes inputs are unit vectors (skips norm)
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_spherical_distance);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_spherical_distance);
|
||||||
Datum
|
Datum
|
||||||
vector_spherical_distance(PG_FUNCTION_ARGS)
|
vector_spherical_distance(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -526,7 +573,7 @@ vector_spherical_distance(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the dimensions of a vector
|
* Get the dimensions of a vector
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_dims);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_dims);
|
||||||
Datum
|
Datum
|
||||||
vector_dims(PG_FUNCTION_ARGS)
|
vector_dims(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -538,15 +585,16 @@ vector_dims(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Get the L2 norm of a vector
|
* Get the L2 norm of a vector
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_norm);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_norm);
|
||||||
Datum
|
Datum
|
||||||
vector_norm(PG_FUNCTION_ARGS)
|
vector_norm(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
|
float *ax = a->x;
|
||||||
double norm = 0.0;
|
double norm = 0.0;
|
||||||
|
|
||||||
for (int i = 0; i < a->dim; i++)
|
for (int i = 0; i < a->dim; i++)
|
||||||
norm += a->x[i] * a->x[i];
|
norm += ax[i] * ax[i];
|
||||||
|
|
||||||
PG_RETURN_FLOAT8(sqrt(norm));
|
PG_RETURN_FLOAT8(sqrt(norm));
|
||||||
}
|
}
|
||||||
@@ -554,20 +602,23 @@ vector_norm(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Add vectors
|
* Add vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_add);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_add);
|
||||||
Datum
|
Datum
|
||||||
vector_add(PG_FUNCTION_ARGS)
|
vector_add(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
Vector *result;
|
Vector *result;
|
||||||
int i;
|
float *rx;
|
||||||
|
|
||||||
CheckDims(a, b);
|
CheckDims(a, b);
|
||||||
|
|
||||||
result = InitVector(a->dim);
|
result = InitVector(a->dim);
|
||||||
for (i = 0; i < a->dim; i++)
|
rx = result->x;
|
||||||
result->x[i] = a->x[i] + b->x[i];
|
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||||
|
rx[i] = ax[i] + bx[i];
|
||||||
|
|
||||||
PG_RETURN_POINTER(result);
|
PG_RETURN_POINTER(result);
|
||||||
}
|
}
|
||||||
@@ -575,20 +626,23 @@ vector_add(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Subtract vectors
|
* Subtract vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_sub);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_sub);
|
||||||
Datum
|
Datum
|
||||||
vector_sub(PG_FUNCTION_ARGS)
|
vector_sub(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
Vector *a = PG_GETARG_VECTOR_P(0);
|
Vector *a = PG_GETARG_VECTOR_P(0);
|
||||||
Vector *b = PG_GETARG_VECTOR_P(1);
|
Vector *b = PG_GETARG_VECTOR_P(1);
|
||||||
|
float *ax = a->x;
|
||||||
|
float *bx = b->x;
|
||||||
Vector *result;
|
Vector *result;
|
||||||
int i;
|
float *rx;
|
||||||
|
|
||||||
CheckDims(a, b);
|
CheckDims(a, b);
|
||||||
|
|
||||||
result = InitVector(a->dim);
|
result = InitVector(a->dim);
|
||||||
for (i = 0; i < a->dim; i++)
|
rx = result->x;
|
||||||
result->x[i] = a->x[i] - b->x[i];
|
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||||
|
rx[i] = ax[i] - bx[i];
|
||||||
|
|
||||||
PG_RETURN_POINTER(result);
|
PG_RETURN_POINTER(result);
|
||||||
}
|
}
|
||||||
@@ -617,7 +671,7 @@ vector_cmp_internal(Vector * a, Vector * b)
|
|||||||
/*
|
/*
|
||||||
* Less than
|
* Less than
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_lt);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_lt);
|
||||||
Datum
|
Datum
|
||||||
vector_lt(PG_FUNCTION_ARGS)
|
vector_lt(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -630,7 +684,7 @@ vector_lt(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Less than or equal
|
* Less than or equal
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_le);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_le);
|
||||||
Datum
|
Datum
|
||||||
vector_le(PG_FUNCTION_ARGS)
|
vector_le(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -643,7 +697,7 @@ vector_le(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Equal
|
* Equal
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_eq);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_eq);
|
||||||
Datum
|
Datum
|
||||||
vector_eq(PG_FUNCTION_ARGS)
|
vector_eq(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -656,7 +710,7 @@ vector_eq(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Not equal
|
* Not equal
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_ne);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ne);
|
||||||
Datum
|
Datum
|
||||||
vector_ne(PG_FUNCTION_ARGS)
|
vector_ne(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -669,7 +723,7 @@ vector_ne(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Greater than or equal
|
* Greater than or equal
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_ge);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_ge);
|
||||||
Datum
|
Datum
|
||||||
vector_ge(PG_FUNCTION_ARGS)
|
vector_ge(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -682,7 +736,7 @@ vector_ge(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Greater than
|
* Greater than
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_gt);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_gt);
|
||||||
Datum
|
Datum
|
||||||
vector_gt(PG_FUNCTION_ARGS)
|
vector_gt(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
@@ -695,7 +749,7 @@ vector_gt(PG_FUNCTION_ARGS)
|
|||||||
/*
|
/*
|
||||||
* Compare vectors
|
* Compare vectors
|
||||||
*/
|
*/
|
||||||
PG_FUNCTION_INFO_V1(vector_cmp);
|
PGDLLEXPORT PG_FUNCTION_INFO_V1(vector_cmp);
|
||||||
Datum
|
Datum
|
||||||
vector_cmp(PG_FUNCTION_ARGS)
|
vector_cmp(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SELECT ARRAY[1,2,3]::vector;
|
SELECT ARRAY[1,2,3]::vector;
|
||||||
array
|
array
|
||||||
---------
|
---------
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
CREATE TABLE t2 (val vector(3));
|
CREATE TABLE t2 (val vector(3));
|
||||||
\copy t TO '/tmp/data.bin' WITH (FORMAT binary)
|
\copy t TO 'results/data.bin' WITH (FORMAT binary)
|
||||||
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary)
|
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
|
||||||
SELECT * FROM t2 ORDER BY val;
|
SELECT * FROM t2 ORDER BY val;
|
||||||
val
|
val
|
||||||
---------
|
---------
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SELECT '[1,2,3]'::vector + '[4,5,6]';
|
SELECT '[1,2,3]'::vector + '[4,5,6]';
|
||||||
?column?
|
?column?
|
||||||
----------
|
----------
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SELECT '[1,2,3]'::vector;
|
SELECT '[1,2,3]'::vector;
|
||||||
vector
|
vector
|
||||||
---------
|
---------
|
||||||
@@ -12,6 +10,12 @@ SELECT '[-1,2,3]'::vector;
|
|||||||
[-1,2,3]
|
[-1,2,3]
|
||||||
(1 row)
|
(1 row)
|
||||||
|
|
||||||
|
SELECT '[1.23456]'::vector;
|
||||||
|
vector
|
||||||
|
-----------
|
||||||
|
[1.23456]
|
||||||
|
(1 row)
|
||||||
|
|
||||||
SELECT '[hello,1]'::vector;
|
SELECT '[hello,1]'::vector;
|
||||||
ERROR: invalid input syntax for type vector: "hello"
|
ERROR: invalid input syntax for type vector: "hello"
|
||||||
LINE 1: SELECT '[hello,1]'::vector;
|
LINE 1: SELECT '[hello,1]'::vector;
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 0);
|
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 0);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
CREATE UNLOGGED TABLE t (val vector(3));
|
CREATE UNLOGGED TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -1,6 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
|
|
||||||
SELECT ARRAY[1,2,3]::vector;
|
SELECT ARRAY[1,2,3]::vector;
|
||||||
SELECT ARRAY[1.0,2.0,3.0]::vector;
|
SELECT ARRAY[1.0,2.0,3.0]::vector;
|
||||||
SELECT ARRAY[1,2,3]::float4[]::vector;
|
SELECT ARRAY[1,2,3]::float4[]::vector;
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
INSERT INTO t (val) VALUES ('[0,0,0]'), ('[1,2,3]'), ('[1,1,1]'), (NULL);
|
||||||
|
|
||||||
CREATE TABLE t2 (val vector(3));
|
CREATE TABLE t2 (val vector(3));
|
||||||
|
|
||||||
\copy t TO '/tmp/data.bin' WITH (FORMAT binary)
|
\copy t TO 'results/data.bin' WITH (FORMAT binary)
|
||||||
\copy t2 FROM '/tmp/data.bin' WITH (FORMAT binary)
|
\copy t2 FROM 'results/data.bin' WITH (FORMAT binary)
|
||||||
|
|
||||||
SELECT * FROM t2 ORDER BY val;
|
SELECT * FROM t2 ORDER BY val;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
|
|
||||||
SELECT '[1,2,3]'::vector + '[4,5,6]';
|
SELECT '[1,2,3]'::vector + '[4,5,6]';
|
||||||
SELECT '[1,2,3]'::vector - '[4,5,6]';
|
SELECT '[1,2,3]'::vector - '[4,5,6]';
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
|
|
||||||
SELECT '[1,2,3]'::vector;
|
SELECT '[1,2,3]'::vector;
|
||||||
SELECT '[-1,2,3]'::vector;
|
SELECT '[-1,2,3]'::vector;
|
||||||
|
SELECT '[1.23456]'::vector;
|
||||||
SELECT '[hello,1]'::vector;
|
SELECT '[hello,1]'::vector;
|
||||||
SELECT '[NaN,1]'::vector;
|
SELECT '[NaN,1]'::vector;
|
||||||
SELECT '[Infinity,1]'::vector;
|
SELECT '[Infinity,1]'::vector;
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE TABLE t (val vector(3));
|
CREATE TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
SET client_min_messages = warning;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS vector;
|
|
||||||
SET enable_seqscan = off;
|
SET enable_seqscan = off;
|
||||||
|
|
||||||
CREATE UNLOGGED TABLE t (val vector(3));
|
CREATE UNLOGGED TABLE t (val vector(3));
|
||||||
|
|||||||
@@ -51,6 +51,11 @@ sub test_index_replay
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Use ARRAY[random(), random(), random(), ...] over
|
||||||
|
# SELECT array_agg(random()) FROM generate_series(1, $dim)
|
||||||
|
# to generate different values for each row
|
||||||
|
my $array_sql = join(",", ('random()') x $dim);
|
||||||
|
|
||||||
# Initialize primary node
|
# Initialize primary node
|
||||||
$node_primary = get_new_node('primary');
|
$node_primary = get_new_node('primary');
|
||||||
$node_primary->init(allows_streaming => 1);
|
$node_primary->init(allows_streaming => 1);
|
||||||
@@ -74,7 +79,7 @@ $node_replica->start;
|
|||||||
$node_primary->safe_psql("postgres", "CREATE EXTENSION vector;");
|
$node_primary->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||||
$node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
$node_primary->safe_psql("postgres", "CREATE TABLE tst (i int4, v vector($dim));");
|
||||||
$node_primary->safe_psql("postgres",
|
$node_primary->safe_psql("postgres",
|
||||||
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series(1, 100000) i;"
|
"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series(1, 100000) i;"
|
||||||
);
|
);
|
||||||
$node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
|
$node_primary->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
|
||||||
|
|
||||||
@@ -90,7 +95,7 @@ for my $i (1 .. 10)
|
|||||||
test_index_replay("vacuum $i");
|
test_index_replay("vacuum $i");
|
||||||
my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000);
|
my ($start, $end) = (100001 + ($i - 1) * 10000, 100000 + $i * 10000);
|
||||||
$node_primary->safe_psql("postgres",
|
$node_primary->safe_psql("postgres",
|
||||||
"INSERT INTO tst SELECT i % 10, (SELECT array_agg(random()) FROM generate_series(1, $dim)) FROM generate_series($start, $end) i;"
|
"INSERT INTO tst SELECT i % 10, ARRAY[$array_sql] FROM generate_series($start, $end) i;"
|
||||||
);
|
);
|
||||||
test_index_replay("insert $i");
|
test_index_replay("insert $i");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,11 @@ use strict;
|
|||||||
use warnings;
|
use warnings;
|
||||||
use PostgresNode;
|
use PostgresNode;
|
||||||
use TestLib;
|
use TestLib;
|
||||||
use Test::More tests => 3;
|
use Test::More tests => 5;
|
||||||
|
|
||||||
|
my $dim = 768;
|
||||||
|
|
||||||
|
my $array_sql = join(",", ('random()') x $dim);
|
||||||
|
|
||||||
# Initialize node
|
# Initialize node
|
||||||
my $node = get_new_node('node');
|
my $node = get_new_node('node');
|
||||||
@@ -11,9 +15,9 @@ $node->start;
|
|||||||
|
|
||||||
# Create table and index
|
# Create table and index
|
||||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||||
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(768));");
|
$node->safe_psql("postgres", "CREATE TABLE tst (v vector($dim));");
|
||||||
$node->safe_psql("postgres",
|
$node->safe_psql("postgres",
|
||||||
"INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10000) i;"
|
"INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10000) i;"
|
||||||
);
|
);
|
||||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
|
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v);");
|
||||||
|
|
||||||
@@ -24,10 +28,18 @@ $node->pgbench(
|
|||||||
[qr{^$}],
|
[qr{^$}],
|
||||||
"concurrent INSERTs",
|
"concurrent INSERTs",
|
||||||
{
|
{
|
||||||
"007_concurrent" => q(
|
"007_inserts" => "INSERT INTO tst SELECT ARRAY[$array_sql] FROM generate_series(1, 10) i;"
|
||||||
BEGIN;
|
|
||||||
INSERT INTO tst SELECT (SELECT array_agg(random()) FROM generate_series(1, 768)) FROM generate_series(1, 10) i;
|
|
||||||
COMMIT;
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
my $expected = 10000 + 5 * 100 * 10;
|
||||||
|
|
||||||
|
my $count = $node->safe_psql("postgres", "SELECT COUNT(*) FROM tst;");
|
||||||
|
is($count, $expected);
|
||||||
|
|
||||||
|
$count = $node->safe_psql("postgres", qq(
|
||||||
|
SET enable_seqscan = off;
|
||||||
|
SET ivfflat.probes = 100;
|
||||||
|
SELECT COUNT(*) FROM (SELECT v FROM tst ORDER BY v <-> (SELECT v FROM tst LIMIT 1)) t;
|
||||||
|
));
|
||||||
|
is($count, $expected);
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
comment = 'vector data type and ivfflat access method'
|
comment = 'vector data type and ivfflat access method'
|
||||||
default_version = '0.3.0'
|
default_version = '0.3.2'
|
||||||
module_pathname = '$libdir/vector'
|
module_pathname = '$libdir/vector'
|
||||||
relocatable = true
|
relocatable = true
|
||||||
|
|||||||
Reference in New Issue
Block a user