Compare commits

..

1 Commits

Author SHA1 Message Date
Andrew Kane
052675044c Use more samples [skip ci] 2023-05-01 20:42:07 -07:00
34 changed files with 152 additions and 529 deletions

View File

@@ -8,8 +8,6 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- postgres: 16
os: ubuntu-22.04
- postgres: 15 - postgres: 15
os: ubuntu-22.04 os: ubuntu-22.04
- postgres: 14 - postgres: 14
@@ -27,8 +25,6 @@ jobs:
postgres-version: ${{ matrix.postgres }} postgres-version: ${{ matrix.postgres }}
dev-files: true dev-files: true
- run: make - run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
- run: | - run: |
export PG_CONFIG=`which pg_config` export PG_CONFIG=`which pg_config`
sudo --preserve-env=PG_CONFIG make install sudo --preserve-env=PG_CONFIG make install
@@ -48,8 +44,6 @@ jobs:
with: with:
postgres-version: 14 postgres-version: 14
- run: make - run: make
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
- run: make install - run: make install
- run: make installcheck - run: make installcheck
- if: ${{ failure() }} - if: ${{ failure() }}
@@ -60,7 +54,6 @@ jobs:
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
tar xf REL_14_5.tar.gz tar xf REL_14_5.tar.gz
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5" - run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
windows: windows:
runs-on: windows-latest runs-on: windows-latest
if: ${{ !startsWith(github.ref_name, 'mac') }} if: ${{ !startsWith(github.ref_name, 'mac') }}
@@ -77,23 +70,3 @@ jobs:
nmake /NOLOGO /F Makefile.win clean && ^ nmake /NOLOGO /F Makefile.win clean && ^
nmake /NOLOGO /F Makefile.win uninstall nmake /NOLOGO /F Makefile.win uninstall
shell: cmd shell: cmd
i386:
runs-on: ubuntu-latest
container:
image: debian:11
options: --platform linux/386
steps:
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-13 postgresql-server-dev-13 sudo
- run: service postgresql start
- run: |
git clone https://github.com/${{ github.repository }}.git pgvector
cd pgvector
git fetch origin ${{ github.ref }}
git reset --hard FETCH_HEAD
make
make install
chown -R postgres .
sudo -u postgres make installcheck
sudo -u postgres make prove_installcheck
env:
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare

View File

@@ -1,24 +1,6 @@
## 0.4.4 (2023-06-12) ## 0.4.2 (unreleased)
- Improved error message for malformed vector literal
- Fixed segmentation fault with text input
- Fixed consecutive delimiters with text input
## 0.4.3 (2023-06-10)
- Improved cost estimation
- Improved support for spaces with text input
- Fixed infinite and NaN values with binary input
- Fixed infinite values with vector addition and subtraction
- Fixed infinite values with list centers
- Fixed compilation error when `float8` is pass by reference
- Fixed compilation error on PowerPC
- Fixed segmentation fault with index creation on i386
## 0.4.2 (2023-05-13)
- Added notice when index created with little data - Added notice when index created with little data
- Fixed dimensions check for some direct function calls
- Fixed installation error with Postgres 12.0-12.2 - Fixed installation error with Postgres 12.0-12.2
## 0.4.1 (2023-03-21) ## 0.4.1 (2023-03-21)

View File

@@ -1,4 +1,4 @@
Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
Portions Copyright (c) 1994, The Regents of the University of California Portions Copyright (c) 1994, The Regents of the University of California

View File

@@ -2,7 +2,7 @@
"name": "vector", "name": "vector",
"abstract": "Open-source vector similarity search for Postgres", "abstract": "Open-source vector similarity search for Postgres",
"description": "Supports L2 distance, inner product, and cosine distance", "description": "Supports L2 distance, inner product, and cosine distance",
"version": "0.4.4", "version": "0.4.1",
"maintainer": [ "maintainer": [
"Andrew Kane <andrew@ankane.org>" "Andrew Kane <andrew@ankane.org>"
], ],
@@ -20,7 +20,7 @@
"vector": { "vector": {
"file": "sql/vector.sql", "file": "sql/vector.sql",
"docfile": "README.md", "docfile": "README.md",
"version": "0.4.4", "version": "0.4.1",
"abstract": "Open-source vector similarity search for Postgres" "abstract": "Open-source vector similarity search for Postgres"
} }
}, },

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.4.4 EXTVERSION = 0.4.1
MODULE_big = vector MODULE_big = vector
DATA = $(wildcard sql/*--*.sql) DATA = $(wildcard sql/*--*.sql)
@@ -19,11 +19,6 @@ ifeq ($(shell uname -s), Darwin)
endif endif
endif endif
# PowerPC doesn't support -march=native
ifneq ($(filter ppc64%, $(shell uname -m)), )
OPTFLAGS =
endif
# For auto-vectorization: # For auto-vectorization:
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html # - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html # - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
@@ -68,9 +63,3 @@ dist:
docker: docker:
docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest . docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
.PHONY: docker-release
docker-release:
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .

View File

@@ -1,5 +1,5 @@
EXTENSION = vector EXTENSION = vector
EXTVERSION = 0.4.4 EXTVERSION = 0.4.1
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj

View File

@@ -2,14 +2,12 @@
Open-source vector similarity search for Postgres Open-source vector similarity search for Postgres
Store all of your application data in one place. Supports: Supports
- exact and approximate nearest neighbor search - exact and approximate nearest neighbor search
- L2 distance, inner product, and cosine distance - L2 distance, inner product, and cosine distance
- any [language](#languages) with a Postgres client - any [language](#languages) with a Postgres client
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
[![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions) [![Build Status](https://github.com/pgvector/pgvector/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector/actions)
## Installation ## Installation
@@ -18,24 +16,24 @@ Compile and install the extension (supports Postgres 11+)
```sh ```sh
cd /tmp cd /tmp
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
make make
make install # may need sudo make install # may need sudo
``` ```
See the [installation notes](#installation-notes) if you run into issues Then load it in databases where you want to use it
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres) ```sql
## Getting Started
Enable the extension (do this once in each database where you want to use it)
```tsql
CREATE EXTENSION vector; CREATE EXTENSION vector;
``` ```
See the [installation notes](#installation-notes) if you run into issues
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
## Getting Started
Create a vector column with 3 dimensions Create a vector column with 3 dimensions
```sql ```sql
@@ -129,7 +127,7 @@ SELECT embedding <-> '[3,1,2]' AS distance FROM items;
For inner product, multiply by -1 (since `<#>` returns the negative inner product) For inner product, multiply by -1 (since `<#>` returns the negative inner product)
```tsql ```sql
SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items; SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
``` ```
@@ -163,7 +161,7 @@ Three keys to achieving good recall are:
1. Create the index *after* the table has some data 1. Create the index *after* the table has some data
2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows 2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `sqrt(lists)` 3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
Add an index for each distance function you want to use. Add an index for each distance function you want to use.
@@ -223,42 +221,26 @@ The phases are:
Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
### Filtering ### Partial Indexes
There are a few ways to index nearest neighbor queries with a `WHERE` clause Consider [partial indexes](https://www.postgresql.org/docs/current/indexes-partial.html) for queries with a `WHERE` clause
```sql ```sql
SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5; SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
``` ```
Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search can be indexed with:
```sql ```sql
CREATE INDEX ON items (category_id); CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
``` ```
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
```sql
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)
WHERE (category_id = 123);
```
Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
```sql ```sql
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id); CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
``` ```
## Hybrid Search
Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search ([Python example](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py)).
```sql
SELECT id, content FROM items, plainto_tsquery('hello search') query
WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
```
## Performance ## Performance
Use `EXPLAIN ANALYZE` to debug performance. Use `EXPLAIN ANALYZE` to debug performance.
@@ -277,7 +259,7 @@ SET max_parallel_workers_per_gather = 4;
If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance. If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
```tsql ```sql
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5; SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
``` ```
@@ -325,11 +307,10 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
#### What if I want to index vectors with more than 2,000 dimensions? #### What if I want to index vectors with more than 2,000 dimensions?
Youll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment. Two things you can try are:
#### Why am I seeing less results after adding an index? 1. use dimensionality reduction
2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
The index was likely created with too little data for the number of lists. Drop the index until the table has more data.
## Reference ## Reference
@@ -373,11 +354,7 @@ If your machine has multiple Postgres installations, specify the path to [pg_con
export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
``` ```
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use: Then re-run the installation instructions (run `make clean` before `make` if needed)
```sh
sudo --preserve-env=PG_CONFIG make install
```
### Missing Header ### Missing Header
@@ -386,7 +363,7 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
For Ubuntu and Debian, use: For Ubuntu and Debian, use:
```sh ```sh
sudo apt install postgresql-server-dev-15 sudo apt-get install postgresql-server-dev-15
``` ```
Note: Replace `15` with your Postgres server version Note: Replace `15` with your Postgres server version
@@ -397,7 +374,7 @@ Support for Windows is currently experimental. Use `nmake` to build:
```cmd ```cmd
set "PGROOT=C:\Program Files\PostgreSQL\15" set "PGROOT=C:\Program Files\PostgreSQL\15"
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
nmake /F Makefile.win nmake /F Makefile.win
nmake /F Makefile.win install nmake /F Makefile.win install
@@ -418,9 +395,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
You can also build the image manually: You can also build the image manually:
```sh ```sh
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
cd pgvector cd pgvector
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector . docker build -t pgvector .
``` ```
### Homebrew ### Homebrew
@@ -431,8 +408,6 @@ With Homebrew Postgres, you can use:
brew install pgvector brew install pgvector
``` ```
Note: This only adds it to the `postgresql@14` formula
### PGXN ### PGXN
Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with: Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with:
@@ -441,16 +416,6 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
pgxn install vector pgxn install vector
``` ```
### APT
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
```sh
sudo apt install postgresql-15-pgvector
```
Note: Replace `15` with your Postgres server version
### Yum ### Yum
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run: RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
@@ -473,17 +438,16 @@ conda install -c conda-forge pgvector
This method is [community-maintained](https://github.com/conda-forge/pgvector-feedstock) by [@mmcauliffe](https://github.com/mmcauliffe) This method is [community-maintained](https://github.com/conda-forge/pgvector-feedstock) by [@mmcauliffe](https://github.com/mmcauliffe)
### Postgres.app
Download the [latest release](https://postgresapp.com/downloads.html) with Postgres 15+.
## Hosted Postgres ## Hosted Postgres
pgvector is available on [these providers](https://github.com/pgvector/pgvector/issues/54). pgvector is available on [these providers](https://github.com/pgvector/pgvector/issues/54).
To request a new extension on other providers: To request a new extension on other providers:
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/managed-database/p/pgvector-extension-for-postgresql) - Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
- Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156) - Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
## Upgrading ## Upgrading

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.2'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.3'" to load this file. \quit

View File

@@ -1,2 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.4'" to load this file. \quit

View File

@@ -147,7 +147,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
{ {
double distance; double distance;
double minDistance = DBL_MAX; double minDistance = DBL_MAX;
int closestCenter = 0; int closestCenter = -1;
VectorArray centers = buildstate->centers; VectorArray centers = buildstate->centers;
TupleTableSlot *slot = buildstate->slot; TupleTableSlot *slot = buildstate->slot;
int i; int i;
@@ -419,7 +419,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
/* Target 50 samples per list, with at least 10000 samples */ /* Target 50 samples per list, with at least 10000 samples */
/* The number of samples has a large effect on index build time */ /* The number of samples has a large effect on index build time */
numSamples = buildstate->lists * 50; numSamples = buildstate->lists * 200;
if (numSamples < 10000) if (numSamples < 10000)
numSamples = 10000; numSamples = 10000;
@@ -438,8 +438,8 @@ ComputeCenters(IvfflatBuildState * buildstate)
{ {
ereport(NOTICE, ereport(NOTICE,
(errmsg("ivfflat index created with little data"), (errmsg("ivfflat index created with little data"),
errdetail("This will cause low recall."), errdetail("this will cause poor recall"),
errhint("Drop the index until the table has more data."))); errhint("drop the index until the table has more data")));
} }
} }
@@ -568,21 +568,6 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
} }
#endif #endif
/*
* Scan table for tuples to index
*/
static void
ScanTable(IvfflatBuildState * buildstate)
{
#if PG_VERSION_NUM >= 120000
buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, true, BuildCallback, (void *) buildstate, NULL);
#else
buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, BuildCallback, (void *) buildstate, NULL);
#endif
}
/* /*
* Create entry pages * Create entry pages
*/ */
@@ -590,7 +575,7 @@ static void
CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum) CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
{ {
AttrNumber attNums[] = {1}; AttrNumber attNums[] = {1};
Oid sortOperators[] = {Int4LessOperator}; Oid sortOperators[] = {Float8LessOperator};
Oid sortCollations[] = {InvalidOid}; Oid sortCollations[] = {InvalidOid};
bool nullsFirstFlags[] = {false}; bool nullsFirstFlags[] = {false};
@@ -600,17 +585,25 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
/* Add tuples to sort */ /* Add tuples to sort */
if (buildstate->heap != NULL) if (buildstate->heap != NULL)
IvfflatBench("assign tuples", ScanTable(buildstate)); {
#if PG_VERSION_NUM >= 120000
buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, true, BuildCallback, (void *) buildstate, NULL);
#else
buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
true, BuildCallback, (void *) buildstate, NULL);
#endif
}
/* Sort */ /* Sort */
IvfflatBench("sort tuples", tuplesort_performsort(buildstate->sortstate)); tuplesort_performsort(buildstate->sortstate);
#ifdef IVFFLAT_KMEANS_DEBUG #ifdef IVFFLAT_KMEANS_DEBUG
PrintKmeansMetrics(buildstate); PrintKmeansMetrics(buildstate);
#endif #endif
/* Insert */ /* Insert */
IvfflatBench("load tuples", InsertTuples(buildstate->index, buildstate, forkNum)); InsertTuples(buildstate->index, buildstate, forkNum);
tuplesort_end(buildstate->sortstate); tuplesort_end(buildstate->sortstate);
} }
@@ -628,7 +621,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
/* Create pages */ /* Create pages */
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum); CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo); CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
CreateEntryPages(buildstate, forkNum); IvfflatBench("CreateEntryPages", CreateEntryPages(buildstate, forkNum));
FreeBuildState(buildstate); FreeBuildState(buildstate);
} }

View File

@@ -7,7 +7,6 @@
#include "ivfflat.h" #include "ivfflat.h"
#include "utils/guc.h" #include "utils/guc.h"
#include "utils/selfuncs.h" #include "utils/selfuncs.h"
#include "utils/spccache.h"
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
#include "commands/progress.h" #include "commands/progress.h"
@@ -64,13 +63,13 @@ ivfflatbuildphasename(int64 phasenum)
static void static void
ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
Cost *indexStartupCost, Cost *indexTotalCost, Cost *indexStartupCost, Cost *indexTotalCost,
Selectivity *indexSelectivity, double *indexCorrelation, Selectivity *indexSelectivity, double *indexCorrelation
double *indexPages) ,double *indexPages
)
{ {
GenericCosts costs; GenericCosts costs;
int lists; int lists;
double ratio; double ratio;
double spc_seq_page_cost;
Relation indexRel; Relation indexRel;
#if PG_VERSION_NUM < 120000 #if PG_VERSION_NUM < 120000
List *qinfos; List *qinfos;
@@ -89,22 +88,6 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
MemSet(&costs, 0, sizeof(costs)); MemSet(&costs, 0, sizeof(costs));
indexRel = index_open(path->indexinfo->indexoid, NoLock);
lists = IvfflatGetLists(indexRel);
index_close(indexRel, NoLock);
/* Get the ratio of lists that we need to visit */
ratio = ((double) ivfflat_probes) / lists;
if (ratio > 1.0)
ratio = 1.0;
/*
* This gives us the subset of tuples to visit. This value is passed into
* the generic cost estimator to determine the number of pages to visit
* during the index scan.
*/
costs.numIndexTuples = path->indexinfo->tuples * ratio;
#if PG_VERSION_NUM >= 120000 #if PG_VERSION_NUM >= 120000
genericcostestimate(root, path, loop_count, &costs); genericcostestimate(root, path, loop_count, &costs);
#else #else
@@ -112,40 +95,22 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
genericcostestimate(root, path, loop_count, qinfos, &costs); genericcostestimate(root, path, loop_count, qinfos, &costs);
#endif #endif
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost); indexRel = index_open(path->indexinfo->indexoid, NoLock);
lists = IvfflatGetLists(indexRel);
index_close(indexRel, NoLock);
/* Adjust cost if needed since TOAST not included in seq scan cost */ ratio = ((double) ivfflat_probes) / lists;
if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5) if (ratio > 1)
{ ratio = 1;
/* Change all page cost from random to sequential */
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
/* Remove cost of extra pages */ costs.indexTotalCost *= ratio;
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
}
else
{
/* Change some page cost from random to sequential */
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
}
/* /* Startup cost and total cost are same */
* If the list selectivity is lower than what is returned from the generic
* cost estimator, use that.
*/
if (ratio < costs.indexSelectivity)
costs.indexSelectivity = ratio;
/* Use total cost since most work happens before first tuple is returned */
*indexStartupCost = costs.indexTotalCost; *indexStartupCost = costs.indexTotalCost;
*indexTotalCost = costs.indexTotalCost; *indexTotalCost = costs.indexTotalCost;
*indexSelectivity = costs.indexSelectivity; *indexSelectivity = costs.indexSelectivity;
*indexCorrelation = costs.indexCorrelation; *indexCorrelation = costs.indexCorrelation;
*indexPages = costs.numIndexPages; *indexPages = costs.numIndexPages;
elog(INFO, "ivfflatcostestimate = %f", costs.indexTotalCost);
/* Cost estimates for parallel workers applied outside of amcostestimate */
elog(INFO, "parallel_workers = %d, parallel aware = %d", path->path.parallel_workers, path->path.parallel_aware);
} }
/* /*
@@ -186,25 +151,6 @@ ivfflatvalidate(Oid opclassoid)
return true; return true;
} }
static Size
ivfflatestimateparallelscan()
{
elog(INFO, "ivfflatestimateparallelscan");
return 0;
}
static void
ivfflatinitparallelscan(void *target)
{
elog(INFO, "ivfflatinitparallelscan");
}
static void
ivfflatparallelrescan(IndexScanDesc scan)
{
elog(INFO, "ivfflatparallelrescan");
}
/* /*
* Define index handler * Define index handler
* *
@@ -232,7 +178,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amstorage = false; amroutine->amstorage = false;
amroutine->amclusterable = false; amroutine->amclusterable = false;
amroutine->ampredlocks = false; amroutine->ampredlocks = false;
amroutine->amcanparallel = true; amroutine->amcanparallel = false;
amroutine->amcaninclude = false; amroutine->amcaninclude = false;
#if PG_VERSION_NUM >= 130000 #if PG_VERSION_NUM >= 130000
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */ amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
@@ -266,9 +212,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
amroutine->amrestrpos = NULL; amroutine->amrestrpos = NULL;
/* Interface functions to support parallel index scans */ /* Interface functions to support parallel index scans */
amroutine->amestimateparallelscan = ivfflatestimateparallelscan; amroutine->amestimateparallelscan = NULL;
amroutine->aminitparallelscan = ivfflatinitparallelscan; amroutine->aminitparallelscan = NULL;
amroutine->amparallelrescan = ivfflatparallelrescan; amroutine->amparallelrescan = NULL;
PG_RETURN_POINTER(amroutine); PG_RETURN_POINTER(amroutine);
} }

View File

@@ -23,10 +23,6 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
OffsetNumber offno; OffsetNumber offno;
OffsetNumber maxoffno; OffsetNumber maxoffno;
/* Avoid compiler warning */
listInfo->blkno = nextblkno;
listInfo->offno = FirstOffsetNumber;
procinfo = index_getprocinfo(rel, 1, IVFFLAT_DISTANCE_PROC); procinfo = index_getprocinfo(rel, 1, IVFFLAT_DISTANCE_PROC);
collation = rel->rd_indcollation[0]; collation = rel->rd_indcollation[0];
@@ -43,7 +39,7 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno)); list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, values[0], PointerGetDatum(&list->center))); distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, values[0], PointerGetDatum(&list->center)));
if (distance < minDistance || !BlockNumberIsValid(*insertPage)) if (distance < minDistance)
{ {
*insertPage = list->insertPage; *insertPage = list->insertPage;
listInfo->blkno = nextblkno; listInfo->blkno = nextblkno;

View File

@@ -1,7 +1,6 @@
#include "postgres.h" #include "postgres.h"
#include <float.h> #include <float.h>
#include <math.h>
#include "ivfflat.h" #include "ivfflat.h"
#include "miscadmin.h" #include "miscadmin.h"
@@ -212,7 +211,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
/* Check memory requirements */ /* Check memory requirements */
/* Add one to error message to ceil */ /* Add one to error message to ceil */
if (totalSize > (Size) maintenance_work_mem * 1024L) if (totalSize / 1024 > maintenance_work_mem)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB", errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
@@ -252,7 +251,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
for (j = 0; j < numSamples; j++) for (j = 0; j < numSamples; j++)
{ {
minDistance = DBL_MAX; minDistance = DBL_MAX;
closestCenter = 0; closestCenter = -1;
/* Find closest center */ /* Find closest center */
for (k = 0; k < numCenters; k++) for (k = 0; k < numCenters; k++)
@@ -399,14 +398,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
if (centerCounts[j] > 0) if (centerCounts[j] > 0)
{ {
/* Double avoids overflow, but requires more memory */
/* TODO Update bounds */
for (k = 0; k < dimensions; k++)
{
if (isinf(vec->x[k]))
vec->x[k] = vec->x[k] > 0 ? FLT_MAX : -FLT_MAX;
}
for (k = 0; k < dimensions; k++) for (k = 0; k < dimensions; k++)
vec->x[k] /= centerCounts[j]; vec->x[k] /= centerCounts[j];
} }
@@ -470,29 +461,12 @@ CheckCenters(Relation index, VectorArray centers)
{ {
FmgrInfo *normprocinfo; FmgrInfo *normprocinfo;
Oid collation; Oid collation;
Vector *vec;
int i; int i;
int j;
double norm; double norm;
if (centers->length != centers->maxlen) if (centers->length != centers->maxlen)
elog(ERROR, "Not enough centers. Please report a bug."); elog(ERROR, "Not enough centers. Please report a bug.");
/* Ensure no NaN or infinite values */
for (i = 0; i < centers->length; i++)
{
vec = VectorArrayGet(centers, i);
for (j = 0; j < vec->dim; j++)
{
if (isnan(vec->x[j]))
elog(ERROR, "NaN detected. Please report a bug.");
if (isinf(vec->x[j]))
elog(ERROR, "Infinite value detected. Please report a bug.");
}
}
/* Ensure no duplicate centers */ /* Ensure no duplicate centers */
/* Fine to sort in-place */ /* Fine to sort in-place */
qsort(centers->items, centers->length, VECTOR_SIZE(centers->dim), CompareVectors); qsort(centers->items, centers->length, VECTOR_SIZE(centers->dim), CompareVectors);

View File

@@ -126,9 +126,6 @@ GetScanItems(IndexScanDesc scan, Datum value)
*/ */
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD); BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
if (scan->parallel_scan != NULL)
elog(INFO, "parallel scan");
/* Search closest probes lists */ /* Search closest probes lists */
while (!pairingheap_is_empty(so->listQueue)) while (!pairingheap_is_empty(so->listQueue))
{ {
@@ -173,14 +170,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
} }
} }
FreeAccessStrategy(bas);
/* TODO Scan more lists */ /* TODO Scan more lists */
if (tuples < 100) if (tuples < 100)
ereport(DEBUG1, ereport(DEBUG1,
(errmsg("index scan found few tuples"), (errmsg("index scan found few tuples"),
errdetail("Index may have been created with little data."), errdetail("index may have been created without data or lists is too high"),
errhint("Recreate the index and possibly decrease lists."))); errhint("recreate the index and possibly decrease lists")));
tuplesort_performsort(so->sortstate); tuplesort_performsort(so->sortstate);
} }

View File

@@ -132,8 +132,6 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
} }
} }
FreeAccessStrategy(bas);
return stats; return stats;
} }

View File

@@ -42,7 +42,7 @@ CheckDims(Vector * a, Vector * b)
} }
/* /*
* Ensure expected dimensions * Ensure expected dimension
*/ */
static inline void static inline void
CheckExpectedDim(int32 typmod, int dim) CheckExpectedDim(int32 typmod, int dim)
@@ -53,9 +53,7 @@ CheckExpectedDim(int32 typmod, int dim)
errmsg("expected %d dimensions, not %d", typmod, dim))); errmsg("expected %d dimensions, not %d", typmod, dim)));
} }
/*
* Ensure valid dimensions
*/
static inline void static inline void
CheckDim(int dim) CheckDim(int dim)
{ {
@@ -81,28 +79,13 @@ CheckElement(float value)
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("NaN not allowed in vector"))); errmsg("NaN not allowed in vector")));
if (isinf(value)) if (isinf(value))
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("infinite value not allowed in vector"))); errmsg("infinite value not allowed in vector")));
} }
/*
* Check for whitespace, since array_isspace() is static
*/
static inline bool
vector_isspace(char ch)
{
if (ch == ' ' ||
ch == '\t' ||
ch == '\n' ||
ch == '\r' ||
ch == '\v' ||
ch == '\f')
return true;
return false;
}
/* /*
* Check state array * Check state array
*/ */
@@ -127,6 +110,30 @@ float_overflow_error(void)
} }
#endif #endif
/*
* Print vector - useful for debugging
*/
void
PrintVector(char *msg, Vector * vector)
{
StringInfoData buf;
int dim = vector->dim;
int i;
initStringInfo(&buf);
appendStringInfoChar(&buf, '[');
for (i = 0; i < dim; i++)
{
if (i > 0)
appendStringInfoString(&buf, ",");
appendStringInfoString(&buf, float8out_internal(vector->x[i]));
}
appendStringInfoChar(&buf, ']');
elog(INFO, "%s = %s", msg, buf.data);
}
/* /*
* Convert textual representation to internal representation * Convert textual representation to internal representation
*/ */
@@ -142,15 +149,11 @@ vector_in(PG_FUNCTION_ARGS)
char *pt; char *pt;
char *stringEnd; char *stringEnd;
Vector *result; Vector *result;
char *lit = pstrdup(str);
while (vector_isspace(*str))
str++;
if (*str != '[') if (*str != '[')
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit), errmsg("malformed vector literal: \"%s\"", str),
errdetail("Vector contents must start with \"[\"."))); errdetail("Vector contents must start with \"[\".")));
str++; str++;
@@ -164,15 +167,6 @@ vector_in(PG_FUNCTION_ARGS)
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM))); errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
while (vector_isspace(*pt))
pt++;
/* Check for empty string like float4in */
if (*pt == '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
/* Use strtof like float4in to avoid a double-rounding problem */ /* Use strtof like float4in to avoid a double-rounding problem */
x[dim] = strtof(pt, &stringEnd); x[dim] = strtof(pt, &stringEnd);
CheckElement(x[dim]); CheckElement(x[dim]);
@@ -181,53 +175,33 @@ vector_in(PG_FUNCTION_ARGS)
if (stringEnd == pt) if (stringEnd == pt)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit))); errmsg("invalid input syntax for type vector: \"%s\"", pt)));
while (vector_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0' && *stringEnd != ']') if (*stringEnd != '\0' && *stringEnd != ']')
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type vector: \"%s\"", lit))); errmsg("invalid input syntax for type vector: \"%s\"", pt)));
pt = strtok(NULL, ","); pt = strtok(NULL, ",");
} }
if (stringEnd == NULL || *stringEnd != ']') if (*stringEnd != ']')
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit), errmsg("malformed vector literal"),
errdetail("Unexpected end of input."))); errdetail("Unexpected end of input.")));
stringEnd++; if (stringEnd[1] != '\0')
/* Only whitespace is allowed after the closing brace */
while (vector_isspace(*stringEnd))
stringEnd++;
if (*stringEnd != '\0')
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit), errmsg("malformed vector literal"),
errdetail("Junk after closing right brace."))); errdetail("Junk after closing right brace.")));
/* Ensure no consecutive delimiters since strtok skips */
for (pt = lit + 1; *pt != '\0'; pt++)
{
if (pt[-1] == ',' && *pt == ',')
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed vector literal: \"%s\"", lit)));
}
if (dim < 1) if (dim < 1)
ereport(ERROR, ereport(ERROR,
(errcode(ERRCODE_DATA_EXCEPTION), (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("vector must have at least 1 dimension"))); errmsg("vector must have at least 1 dimension")));
pfree(lit);
CheckExpectedDim(typmod, dim); CheckExpectedDim(typmod, dim);
result = InitVector(dim); result = InitVector(dim);
@@ -298,18 +272,6 @@ vector_out(PG_FUNCTION_ARGS)
PG_RETURN_CSTRING(buf); PG_RETURN_CSTRING(buf);
} }
/*
* Print vector - useful for debugging
*/
void
PrintVector(char *msg, Vector * vector)
{
char *out = DatumGetPointer(DirectFunctionCall1(vector_out, PointerGetDatum(vector)));
elog(INFO, "%s = %s", msg, out);
pfree(out);
}
/* /*
* Convert type modifier * Convert type modifier
*/ */
@@ -368,10 +330,7 @@ vector_recv(PG_FUNCTION_ARGS)
result = InitVector(dim); result = InitVector(dim);
for (i = 0; i < dim; i++) for (i = 0; i < dim; i++)
{
result->x[i] = pq_getmsgfloat4(buf); result->x[i] = pq_getmsgfloat4(buf);
CheckElement(result->x[i]);
}
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -437,7 +396,9 @@ array_to_vector(PG_FUNCTION_ARGS)
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign); get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp); deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp);
if (typmod == -1)
CheckDim(nelemsp); CheckDim(nelemsp);
else
CheckExpectedDim(typmod, nelemsp); CheckExpectedDim(typmod, nelemsp);
result = InitVector(nelemsp); result = InitVector(nelemsp);
@@ -448,7 +409,6 @@ array_to_vector(PG_FUNCTION_ARGS)
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
errmsg("array must not containing NULLs"))); errmsg("array must not containing NULLs")));
/* TODO Move outside loop in 0.5.0 */
if (ARR_ELEMTYPE(array) == INT4OID) if (ARR_ELEMTYPE(array) == INT4OID)
result->x[i] = DatumGetInt32(elemsp[i]); result->x[i] = DatumGetInt32(elemsp[i]);
else if (ARR_ELEMTYPE(array) == FLOAT8OID) else if (ARR_ELEMTYPE(array) == FLOAT8OID)
@@ -476,19 +436,17 @@ Datum
vector_to_float4(PG_FUNCTION_ARGS) vector_to_float4(PG_FUNCTION_ARGS)
{ {
Vector *vec = PG_GETARG_VECTOR_P(0); Vector *vec = PG_GETARG_VECTOR_P(0);
Datum *datums; Datum *d;
ArrayType *result; ArrayType *result;
int i; int i;
datums = (Datum *) palloc(sizeof(Datum) * vec->dim); d = (Datum *) palloc(sizeof(Datum) * vec->dim);
for (i = 0; i < vec->dim; i++) for (i = 0; i < vec->dim; i++)
datums[i] = Float4GetDatum(vec->x[i]); d[i] = Float4GetDatum(vec->x[i]);
/* Use TYPALIGN_INT for float4 */ /* Use TYPALIGN_INT for float4 */
result = construct_array(datums, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT); result = construct_array(d, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT);
pfree(datums);
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -700,13 +658,6 @@ vector_add(PG_FUNCTION_ARGS)
for (int i = 0, imax = a->dim; i < imax; i++) for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] + bx[i]; rx[i] = ax[i] + bx[i];
/* Check for overflow */
for (int i = 0, imax = a->dim; i < imax; i++)
{
if (isinf(rx[i]))
float_overflow_error();
}
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -733,13 +684,6 @@ vector_sub(PG_FUNCTION_ARGS)
for (int i = 0, imax = a->dim; i < imax; i++) for (int i = 0, imax = a->dim; i < imax; i++)
rx[i] = ax[i] - bx[i]; rx[i] = ax[i] - bx[i];
/* Check for overflow */
for (int i = 0, imax = a->dim; i < imax; i++)
{
if (isinf(rx[i]))
float_overflow_error();
}
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);
} }
@@ -885,12 +829,12 @@ vector_accum(PG_FUNCTION_ARGS)
n = statevalues[0] + 1.0; n = statevalues[0] + 1.0;
statedatums = CreateStateDatums(dim); statedatums = CreateStateDatums(dim);
statedatums[0] = Float8GetDatum(n); statedatums[0] = Float8GetDatumFast(n);
if (newarr) if (newarr)
{ {
for (int i = 0; i < dim; i++) for (int i = 0; i < dim; i++)
statedatums[i + 1] = Float8GetDatum((double) x[i]); statedatums[i + 1] = Float8GetDatumFast((double) x[i]);
} }
else else
{ {
@@ -898,11 +842,10 @@ vector_accum(PG_FUNCTION_ARGS)
{ {
double v = statevalues[i + 1] + x[i]; double v = statevalues[i + 1] + x[i];
/* Check for overflow */
if (isinf(v)) if (isinf(v))
float_overflow_error(); float_overflow_error();
statedatums[i + 1] = Float8GetDatum(v); statedatums[i + 1] = Float8GetDatumFast(v);
} }
} }
@@ -947,7 +890,7 @@ vector_combine(PG_FUNCTION_ARGS)
dim = STATE_DIMS(statearray2); dim = STATE_DIMS(statearray2);
statedatums = CreateStateDatums(dim); statedatums = CreateStateDatums(dim);
for (int i = 1; i <= dim; i++) for (int i = 1; i <= dim; i++)
statedatums[i] = Float8GetDatum(statevalues2[i]); statedatums[i] = Float8GetDatumFast(statevalues2[i]);
} }
else if (n2 == 0.0) else if (n2 == 0.0)
{ {
@@ -955,7 +898,7 @@ vector_combine(PG_FUNCTION_ARGS)
dim = STATE_DIMS(statearray1); dim = STATE_DIMS(statearray1);
statedatums = CreateStateDatums(dim); statedatums = CreateStateDatums(dim);
for (int i = 1; i <= dim; i++) for (int i = 1; i <= dim; i++)
statedatums[i] = Float8GetDatum(statevalues1[i]); statedatums[i] = Float8GetDatumFast(statevalues1[i]);
} }
else else
{ {
@@ -967,15 +910,14 @@ vector_combine(PG_FUNCTION_ARGS)
{ {
double v = statevalues1[i] + statevalues2[i]; double v = statevalues1[i] + statevalues2[i];
/* Check for overflow */
if (isinf(v)) if (isinf(v))
float_overflow_error(); float_overflow_error();
statedatums[i] = Float8GetDatum(v); statedatums[i] = Float8GetDatumFast(v);
} }
} }
statedatums[0] = Float8GetDatum(n); statedatums[0] = Float8GetDatumFast(n);
result = construct_array(statedatums, dim + 1, result = construct_array(statedatums, dim + 1,
FLOAT8OID, FLOAT8OID,
@@ -998,6 +940,7 @@ vector_avg(PG_FUNCTION_ARGS)
float8 n; float8 n;
uint16 dim; uint16 dim;
Vector *result; Vector *result;
float v;
/* Check array before using */ /* Check array before using */
statevalues = CheckStateArray(statearray, "vector_avg"); statevalues = CheckStateArray(statearray, "vector_avg");
@@ -1009,12 +952,12 @@ vector_avg(PG_FUNCTION_ARGS)
/* Create vector */ /* Create vector */
dim = STATE_DIMS(statearray); dim = STATE_DIMS(statearray);
CheckDim(dim);
result = InitVector(dim); result = InitVector(dim);
for (int i = 0; i < dim; i++) for (int i = 0; i < dim; i++)
{ {
result->x[i] = statevalues[i + 1] / n; v = statevalues[i + 1] / n;
CheckElement(result->x[i]); CheckElement(v);
result->x[i] = v;
} }
PG_RETURN_POINTER(result); PG_RETURN_POINTER(result);

View File

@@ -46,8 +46,6 @@ SELECT '[1,2,3]'::vector::real[];
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n; SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions ERROR: vector cannot have more than 16000 dimensions
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
ERROR: vector cannot have more than 16000 dimensions
-- ensure no error -- ensure no error
SELECT ARRAY[1,2,3] = ARRAY[1,2,3]; SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
?column? ?column?

View File

@@ -4,16 +4,12 @@ SELECT '[1,2,3]'::vector + '[4,5,6]';
[5,7,9] [5,7,9]
(1 row) (1 row)
SELECT '[3e38]'::vector + '[3e38]';
ERROR: value out of range: overflow
SELECT '[1,2,3]'::vector - '[4,5,6]'; SELECT '[1,2,3]'::vector - '[4,5,6]';
?column? ?column?
------------ ------------
[-3,-3,-3] [-3,-3,-3]
(1 row) (1 row)
SELECT '[-3e38]'::vector - '[3e38]';
ERROR: value out of range: overflow
SELECT vector_dims('[1,2,3]'); SELECT vector_dims('[1,2,3]');
vector_dims vector_dims
------------- -------------
@@ -106,5 +102,3 @@ SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v; SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
ERROR: expected 2 dimensions, not 1 ERROR: expected 2 dimensions, not 1
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
ERROR: vector cannot have more than 16000 dimensions

View File

@@ -4,22 +4,10 @@ SELECT '[1,2,3]'::vector;
[1,2,3] [1,2,3]
(1 row) (1 row)
SELECT '[-1,-2,-3]'::vector; SELECT '[-1,2,3]'::vector;
vector vector
------------ ----------
[-1,-2,-3] [-1,2,3]
(1 row)
SELECT '[1.,2.,3.]'::vector;
vector
---------
[1,2,3]
(1 row)
SELECT ' [ 1, 2 , 3 ] '::vector;
vector
---------
[1,2,3]
(1 row) (1 row)
SELECT '[1.23456]'::vector; SELECT '[1.23456]'::vector;
@@ -29,7 +17,7 @@ SELECT '[1.23456]'::vector;
(1 row) (1 row)
SELECT '[hello,1]'::vector; SELECT '[hello,1]'::vector;
ERROR: invalid input syntax for type vector: "[hello,1]" ERROR: invalid input syntax for type vector: "hello"
LINE 1: SELECT '[hello,1]'::vector; LINE 1: SELECT '[hello,1]'::vector;
^ ^
SELECT '[NaN,1]'::vector; SELECT '[NaN,1]'::vector;
@@ -44,35 +32,13 @@ SELECT '[-Infinity,1]'::vector;
ERROR: infinite value not allowed in vector ERROR: infinite value not allowed in vector
LINE 1: SELECT '[-Infinity,1]'::vector; LINE 1: SELECT '[-Infinity,1]'::vector;
^ ^
SELECT '[1.5e38,-1.5e38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e+38,-1.5e+38]'::vector;
vector
--------------------
[1.5e+38,-1.5e+38]
(1 row)
SELECT '[1.5e-38,-1.5e-38]'::vector;
vector
--------------------
[1.5e-38,-1.5e-38]
(1 row)
SELECT '[4e38,1]'::vector;
ERROR: infinite value not allowed in vector
LINE 1: SELECT '[4e38,1]'::vector;
^
SELECT '[1,2,3'::vector; SELECT '[1,2,3'::vector;
ERROR: malformed vector literal: "[1,2,3" ERROR: malformed vector literal
LINE 1: SELECT '[1,2,3'::vector; LINE 1: SELECT '[1,2,3'::vector;
^ ^
DETAIL: Unexpected end of input. DETAIL: Unexpected end of input.
SELECT '[1,2,3]9'::vector; SELECT '[1,2,3]9'::vector;
ERROR: malformed vector literal: "[1,2,3]9" ERROR: malformed vector literal
LINE 1: SELECT '[1,2,3]9'::vector; LINE 1: SELECT '[1,2,3]9'::vector;
^ ^
DETAIL: Junk after closing right brace. DETAIL: Junk after closing right brace.
@@ -81,36 +47,14 @@ ERROR: malformed vector literal: "1,2,3"
LINE 1: SELECT '1,2,3'::vector; LINE 1: SELECT '1,2,3'::vector;
^ ^
DETAIL: Vector contents must start with "[". DETAIL: Vector contents must start with "[".
SELECT '['::vector;
ERROR: malformed vector literal: "["
LINE 1: SELECT '['::vector;
^
DETAIL: Unexpected end of input.
SELECT '[,'::vector;
ERROR: malformed vector literal: "[,"
LINE 1: SELECT '[,'::vector;
^
DETAIL: Unexpected end of input.
SELECT '[]'::vector; SELECT '[]'::vector;
ERROR: vector must have at least 1 dimension ERROR: vector must have at least 1 dimension
LINE 1: SELECT '[]'::vector; LINE 1: SELECT '[]'::vector;
^ ^
SELECT '[1,]'::vector; SELECT '[1,]'::vector;
ERROR: invalid input syntax for type vector: "[1,]" ERROR: invalid input syntax for type vector: "]"
LINE 1: SELECT '[1,]'::vector; LINE 1: SELECT '[1,]'::vector;
^ ^
SELECT '[1a]'::vector;
ERROR: invalid input syntax for type vector: "[1a]"
LINE 1: SELECT '[1a]'::vector;
^
SELECT '[1,,3]'::vector;
ERROR: malformed vector literal: "[1,,3]"
LINE 1: SELECT '[1,,3]'::vector;
^
SELECT '[1, ,3]'::vector;
ERROR: invalid input syntax for type vector: "[1, ,3]"
LINE 1: SELECT '[1, ,3]'::vector;
^
SELECT '[1,2,3]'::vector(2); SELECT '[1,2,3]'::vector(2);
ERROR: expected 2 dimensions, not 3 ERROR: expected 2 dimensions, not 3
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]); SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);

View File

@@ -10,7 +10,6 @@ SELECT '{-Infinity}'::real[]::vector;
SELECT '{}'::real[]::vector; SELECT '{}'::real[]::vector;
SELECT '[1,2,3]'::vector::real[]; SELECT '[1,2,3]'::vector::real[];
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n; SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
-- ensure no error -- ensure no error
SELECT ARRAY[1,2,3] = ARRAY[1,2,3]; SELECT ARRAY[1,2,3] = ARRAY[1,2,3];

View File

@@ -1,7 +1,5 @@
SELECT '[1,2,3]'::vector + '[4,5,6]'; SELECT '[1,2,3]'::vector + '[4,5,6]';
SELECT '[3e38]'::vector + '[3e38]';
SELECT '[1,2,3]'::vector - '[4,5,6]'; SELECT '[1,2,3]'::vector - '[4,5,6]';
SELECT '[-3e38]'::vector - '[3e38]';
SELECT vector_dims('[1,2,3]'); SELECT vector_dims('[1,2,3]');
@@ -26,4 +24,3 @@ SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v; SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v; SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v; SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;

View File

@@ -1,26 +1,15 @@
SELECT '[1,2,3]'::vector; SELECT '[1,2,3]'::vector;
SELECT '[-1,-2,-3]'::vector; SELECT '[-1,2,3]'::vector;
SELECT '[1.,2.,3.]'::vector;
SELECT ' [ 1, 2 , 3 ] '::vector;
SELECT '[1.23456]'::vector; SELECT '[1.23456]'::vector;
SELECT '[hello,1]'::vector; SELECT '[hello,1]'::vector;
SELECT '[NaN,1]'::vector; SELECT '[NaN,1]'::vector;
SELECT '[Infinity,1]'::vector; SELECT '[Infinity,1]'::vector;
SELECT '[-Infinity,1]'::vector; SELECT '[-Infinity,1]'::vector;
SELECT '[1.5e38,-1.5e38]'::vector;
SELECT '[1.5e+38,-1.5e+38]'::vector;
SELECT '[1.5e-38,-1.5e-38]'::vector;
SELECT '[4e38,1]'::vector;
SELECT '[1,2,3'::vector; SELECT '[1,2,3'::vector;
SELECT '[1,2,3]9'::vector; SELECT '[1,2,3]9'::vector;
SELECT '1,2,3'::vector; SELECT '1,2,3'::vector;
SELECT '['::vector;
SELECT '[,'::vector;
SELECT '[]'::vector; SELECT '[]'::vector;
SELECT '[1,]'::vector; SELECT '[1,]'::vector;
SELECT '[1a]'::vector;
SELECT '[1,,3]'::vector;
SELECT '[1, ,3]'::vector;
SELECT '[1,2,3]'::vector(2); SELECT '[1,2,3]'::vector(2);
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]); SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);

View File

@@ -1,16 +0,0 @@
-- SET force_parallel_mode = on;
SET parallel_setup_cost = 10;
SET parallel_tuple_cost = 0.000001;
SET min_parallel_table_scan_size = 1;
SET min_parallel_index_scan_size = 1;
CREATE TABLE t (id integer, val vector(3));
ALTER TABLE t ALTER COLUMN val SET STORAGE PLAIN;
INSERT INTO t (id, val) SELECT n, ARRAY[random(), random(), random()] FROM generate_series(1,1000000) n;
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 10);
SET ivfflat.probes = 4;
EXPLAIN SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
DROP TABLE t;

View File

@@ -5,7 +5,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 31;
my $dim = 32; my $dim = 32;
@@ -95,5 +95,3 @@ for my $i (1 .. 10)
); );
test_index_replay("insert $i"); test_index_replay("insert $i");
} }
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 1;
my $dim = 3; my $dim = 3;
@@ -39,5 +39,3 @@ $node->safe_psql("postgres",
# Check size # Check size
my $new_size = $node->safe_psql("postgres", "SELECT pg_total_relation_size('tst_v_idx');"); my $new_size = $node->safe_psql("postgres", "SELECT pg_total_relation_size('tst_v_idx');");
is($size, $new_size, "size does not change"); is($size, $new_size, "size does not change");
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 9;
my $node; my $node;
my @queries = (); my @queries = ();
@@ -15,13 +15,6 @@ sub test_recall
my $correct = 0; my $correct = 0;
my $total = 0; my $total = 0;
my $explain = $node->safe_psql("postgres", qq(
SET enable_seqscan = off;
SET ivfflat.probes = $probes;
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
));
like($explain, qr/Index Scan/);
for my $i (0 .. $#queries) { for my $i (0 .. $#queries) {
my $actual = $node->safe_psql("postgres", qq( my $actual = $node->safe_psql("postgres", qq(
SET enable_seqscan = off; SET enable_seqscan = off;
@@ -79,9 +72,9 @@ foreach (@operators) {
# Add index # Add index
my $opclass; my $opclass;
if ($operator eq "<->") { if ($operator == "<->") {
$opclass = "vector_l2_ops"; $opclass = "vector_l2_ops";
} elsif ($operator eq "<#>") { } elsif ($operator == "<#>") {
$opclass = "vector_ip_ops"; $opclass = "vector_ip_ops";
} else { } else {
$opclass = "vector_cosine_ops"; $opclass = "vector_cosine_ops";
@@ -89,13 +82,7 @@ foreach (@operators) {
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);"); $node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
# Test approximate results # Test approximate results
if ($operator ne "<#>") {
# TODO fix test
test_recall(1, 0.75, $operator); test_recall(1, 0.75, $operator);
test_recall(10, 0.95, $operator); test_recall(10, 0.95, $operator);
} test_recall(100, 1.0, $operator);
# Account for equal distances
test_recall(100, 0.9975, $operator);
} }
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 3;
# Initialize node # Initialize node
my $node = get_new_node('node'); my $node = get_new_node('node');
@@ -34,5 +34,3 @@ $node->safe_psql("postgres",
# Test no error for duplicate centers # Test no error for duplicate centers
test_centers(10); test_centers(10);
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 60;
# Initialize node # Initialize node
my $node = get_new_node('node'); my $node = get_new_node('node');
@@ -23,9 +23,9 @@ foreach (@operators) {
# Add index # Add index
my $opclass; my $opclass;
if ($operator eq "<->") { if ($operator == "<->") {
$opclass = "vector_l2_ops"; $opclass = "vector_l2_ops";
} elsif ($operator eq "<#>") { } elsif ($operator == "<#>") {
$opclass = "vector_ip_ops"; $opclass = "vector_ip_ops";
} else { } else {
$opclass = "vector_cosine_ops"; $opclass = "vector_cosine_ops";
@@ -43,5 +43,3 @@ foreach (@operators) {
is($res, $query); is($res, $query);
} }
} }
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 3;
# Initialize node # Initialize node
my $node = get_new_node('node'); my $node = get_new_node('node');
@@ -29,5 +29,3 @@ my ($ret, $stdout, $stderr) = $node->psql("postgres",
"CREATE INDEX lists10000 ON tst USING ivfflat (v) WITH (lists = 10000);" "CREATE INDEX lists10000 ON tst USING ivfflat (v) WITH (lists = 10000);"
); );
like($stderr, qr/memory required is/); like($stderr, qr/memory required is/);
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 7;
my $dim = 768; my $dim = 768;
@@ -53,5 +53,3 @@ $count = $node->safe_psql("postgres", qq(
)); ));
is($count, $expected); is($count, $expected);
is(idx_scan(), 1); is(idx_scan(), 1);
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 5;
# Initialize node # Initialize node
my $node = get_new_node('node'); my $node = get_new_node('node');
@@ -33,5 +33,3 @@ is($avg, "[$r1,$r2,$r3]");
# Test explain # Test explain
my $explain = $node->safe_psql("postgres", "EXPLAIN SELECT AVG(v) FROM tst;"); my $explain = $node->safe_psql("postgres", "EXPLAIN SELECT AVG(v) FROM tst;");
like($explain, qr/Partial Aggregate/); like($explain, qr/Partial Aggregate/);
done_testing();

View File

@@ -2,7 +2,7 @@ use strict;
use warnings; use warnings;
use PostgresNode; use PostgresNode;
use TestLib; use TestLib;
use Test::More; use Test::More tests => 1;
my $dim = 1024; my $dim = 1024;
@@ -30,5 +30,3 @@ my ($ret, $stdout, $stderr) = $node->psql("postgres",
"INSERT INTO tst SELECT array_agg(n), array_agg(n), array_agg(n) FROM generate_series(1, $dim) n" "INSERT INTO tst SELECT array_agg(n), array_agg(n), array_agg(n) FROM generate_series(1, $dim) n"
); );
like($stderr, qr/row is too big/); like($stderr, qr/row is too big/);
done_testing();

View File

@@ -1,4 +1,4 @@
comment = 'vector data type and ivfflat access method' comment = 'vector data type and ivfflat access method'
default_version = '0.4.4' default_version = '0.4.1'
module_pathname = '$libdir/vector' module_pathname = '$libdir/vector'
relocatable = true relocatable = true