mirror of
https://github.com/pgvector/pgvector.git
synced 2026-07-22 20:15:46 +08:00
Compare commits
2 Commits
parallel-i
...
dynamic-pr
| Author | SHA1 | Date | |
|---|---|---|---|
|
7132c9b111 | ||
|
|
878e1e6c3a |
27
.github/workflows/build.yml
vendored
27
.github/workflows/build.yml
vendored
@@ -8,8 +8,6 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- postgres: 16
|
||||
os: ubuntu-22.04
|
||||
- postgres: 15
|
||||
os: ubuntu-22.04
|
||||
- postgres: 14
|
||||
@@ -27,8 +25,6 @@ jobs:
|
||||
postgres-version: ${{ matrix.postgres }}
|
||||
dev-files: true
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
- run: |
|
||||
export PG_CONFIG=`which pg_config`
|
||||
sudo --preserve-env=PG_CONFIG make install
|
||||
@@ -48,8 +44,6 @@ jobs:
|
||||
with:
|
||||
postgres-version: 14
|
||||
- run: make
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter
|
||||
- run: make install
|
||||
- run: make installcheck
|
||||
- if: ${{ failure() }}
|
||||
@@ -60,7 +54,6 @@ jobs:
|
||||
wget -q https://github.com/postgres/postgres/archive/refs/tags/REL_14_5.tar.gz
|
||||
tar xf REL_14_5.tar.gz
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres-REL_14_5/src/test/perl" PERL5LIB="/Users/runner/perl5/lib/perl5"
|
||||
- run: make clean && /usr/local/opt/llvm@15/bin/scan-build --status-bugs make
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
if: ${{ !startsWith(github.ref_name, 'mac') }}
|
||||
@@ -77,23 +70,3 @@ jobs:
|
||||
nmake /NOLOGO /F Makefile.win clean && ^
|
||||
nmake /NOLOGO /F Makefile.win uninstall
|
||||
shell: cmd
|
||||
i386:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: debian:11
|
||||
options: --platform linux/386
|
||||
steps:
|
||||
- run: apt-get update && apt-get install -y build-essential git libipc-run-perl postgresql-13 postgresql-server-dev-13 sudo
|
||||
- run: service postgresql start
|
||||
- run: |
|
||||
git clone https://github.com/${{ github.repository }}.git pgvector
|
||||
cd pgvector
|
||||
git fetch origin ${{ github.ref }}
|
||||
git reset --hard FETCH_HEAD
|
||||
make
|
||||
make install
|
||||
chown -R postgres .
|
||||
sudo -u postgres make installcheck
|
||||
sudo -u postgres make prove_installcheck
|
||||
env:
|
||||
PG_CFLAGS: -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
|
||||
20
CHANGELOG.md
20
CHANGELOG.md
@@ -1,24 +1,6 @@
|
||||
## 0.4.4 (2023-06-12)
|
||||
|
||||
- Improved error message for malformed vector literal
|
||||
- Fixed segmentation fault with text input
|
||||
- Fixed consecutive delimiters with text input
|
||||
|
||||
## 0.4.3 (2023-06-10)
|
||||
|
||||
- Improved cost estimation
|
||||
- Improved support for spaces with text input
|
||||
- Fixed infinite and NaN values with binary input
|
||||
- Fixed infinite values with vector addition and subtraction
|
||||
- Fixed infinite values with list centers
|
||||
- Fixed compilation error when `float8` is pass by reference
|
||||
- Fixed compilation error on PowerPC
|
||||
- Fixed segmentation fault with index creation on i386
|
||||
|
||||
## 0.4.2 (2023-05-13)
|
||||
## 0.4.2 (unreleased)
|
||||
|
||||
- Added notice when index created with little data
|
||||
- Fixed dimensions check for some direct function calls
|
||||
- Fixed installation error with Postgres 12.0-12.2
|
||||
|
||||
## 0.4.1 (2023-03-21)
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
||||
Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
|
||||
Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
|
||||
|
||||
Portions Copyright (c) 1994, The Regents of the University of California
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.4.4",
|
||||
"version": "0.4.1",
|
||||
"maintainer": [
|
||||
"Andrew Kane <andrew@ankane.org>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.4.4",
|
||||
"version": "0.4.1",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
13
Makefile
13
Makefile
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.4.4
|
||||
EXTVERSION = 0.4.1
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*.sql)
|
||||
@@ -19,11 +19,6 @@ ifeq ($(shell uname -s), Darwin)
|
||||
endif
|
||||
endif
|
||||
|
||||
# PowerPC doesn't support -march=native
|
||||
ifneq ($(filter ppc64%, $(shell uname -m)), )
|
||||
OPTFLAGS =
|
||||
endif
|
||||
|
||||
# For auto-vectorization:
|
||||
# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html
|
||||
# - Clang (could use pragma instead) - https://llvm.org/docs/Vectorizers.html
|
||||
@@ -68,9 +63,3 @@ dist:
|
||||
|
||||
docker:
|
||||
docker build --pull --no-cache --platform linux/amd64 -t ankane/pgvector:latest .
|
||||
|
||||
.PHONY: docker-release
|
||||
|
||||
docker-release:
|
||||
docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 -t ankane/pgvector:latest .
|
||||
docker buildx build --push --platform linux/amd64,linux/arm64 -t ankane/pgvector:v$(EXTVERSION) .
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.4.4
|
||||
EXTVERSION = 0.4.1
|
||||
|
||||
OBJS = src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\vector.obj
|
||||
|
||||
|
||||
94
README.md
94
README.md
@@ -2,14 +2,12 @@
|
||||
|
||||
Open-source vector similarity search for Postgres
|
||||
|
||||
Store all of your application data in one place. Supports:
|
||||
Supports
|
||||
|
||||
- exact and approximate nearest neighbor search
|
||||
- L2 distance, inner product, and cosine distance
|
||||
- any [language](#languages) with a Postgres client
|
||||
|
||||
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
|
||||
|
||||
[](https://github.com/pgvector/pgvector/actions)
|
||||
|
||||
## Installation
|
||||
@@ -18,24 +16,24 @@ Compile and install the extension (supports Postgres 11+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
```
|
||||
|
||||
See the [installation notes](#installation-notes) if you run into issues
|
||||
Then load it in databases where you want to use it
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres)
|
||||
|
||||
## Getting Started
|
||||
|
||||
Enable the extension (do this once in each database where you want to use it)
|
||||
|
||||
```tsql
|
||||
```sql
|
||||
CREATE EXTENSION vector;
|
||||
```
|
||||
|
||||
See the [installation notes](#installation-notes) if you run into issues
|
||||
|
||||
You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [Yum](#yum), or [conda-forge](#conda-forge)
|
||||
|
||||
## Getting Started
|
||||
|
||||
Create a vector column with 3 dimensions
|
||||
|
||||
```sql
|
||||
@@ -129,7 +127,7 @@ SELECT embedding <-> '[3,1,2]' AS distance FROM items;
|
||||
|
||||
For inner product, multiply by -1 (since `<#>` returns the negative inner product)
|
||||
|
||||
```tsql
|
||||
```sql
|
||||
SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items;
|
||||
```
|
||||
|
||||
@@ -163,7 +161,7 @@ Three keys to achieving good recall are:
|
||||
|
||||
1. Create the index *after* the table has some data
|
||||
2. Choose an appropriate number of lists - a good place to start is `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows
|
||||
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `sqrt(lists)`
|
||||
3. When querying, specify an appropriate number of [probes](#query-options) (higher is better for recall, lower is better for speed) - a good place to start is `lists / 10` for up to 1M rows and `sqrt(lists)` for over 1M rows
|
||||
|
||||
Add an index for each distance function you want to use.
|
||||
|
||||
@@ -223,42 +221,26 @@ The phases are:
|
||||
|
||||
Note: `tuples_done` and `tuples_total` are only populated during the `loading tuples` phase
|
||||
|
||||
### Filtering
|
||||
### Partial Indexes
|
||||
|
||||
There are a few ways to index nearest neighbor queries with a `WHERE` clause
|
||||
Consider [partial indexes](https://www.postgresql.org/docs/current/indexes-partial.html) for queries with a `WHERE` clause
|
||||
|
||||
```sql
|
||||
SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
Create an index on one [or more](https://www.postgresql.org/docs/current/indexes-multicolumn.html) of the `WHERE` columns for exact search
|
||||
can be indexed with:
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items (category_id);
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) WHERE (category_id = 123);
|
||||
```
|
||||
|
||||
Or a [partial index](https://www.postgresql.org/docs/current/indexes-partial.html) on the vector column for approximate search
|
||||
|
||||
```sql
|
||||
CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)
|
||||
WHERE (category_id = 123);
|
||||
```
|
||||
|
||||
Use [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) for approximate search on many different values of the `WHERE` columns
|
||||
To index many different values of `category_id`, consider [partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) on `category_id`.
|
||||
|
||||
```sql
|
||||
CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id);
|
||||
```
|
||||
|
||||
## Hybrid Search
|
||||
|
||||
Use together with Postgres [full-text search](https://www.postgresql.org/docs/current/textsearch-intro.html) for hybrid search ([Python example](https://github.com/pgvector/pgvector-python/blob/master/examples/hybrid_search.py)).
|
||||
|
||||
```sql
|
||||
SELECT id, content FROM items, plainto_tsquery('hello search') query
|
||||
WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5;
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Use `EXPLAIN ANALYZE` to debug performance.
|
||||
@@ -277,7 +259,7 @@ SET max_parallel_workers_per_gather = 4;
|
||||
|
||||
If vectors are normalized to length 1 (like [OpenAI embeddings](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use)), use inner product for best performance.
|
||||
|
||||
```tsql
|
||||
```sql
|
||||
SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
```
|
||||
|
||||
@@ -325,7 +307,10 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p
|
||||
|
||||
#### What if I want to index vectors with more than 2,000 dimensions?
|
||||
|
||||
You’ll need to use [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction) at the moment.
|
||||
Two things you can try are:
|
||||
|
||||
1. use dimensionality reduction
|
||||
2. compile Postgres with a larger block size (`./configure --with-blocksize=32`) and edit the limit in `src/ivfflat.h`
|
||||
|
||||
#### Why am I seeing less results after adding an index?
|
||||
|
||||
@@ -373,11 +358,7 @@ If your machine has multiple Postgres installations, specify the path to [pg_con
|
||||
export PG_CONFIG=/Applications/Postgres.app/Contents/Versions/latest/bin/pg_config
|
||||
```
|
||||
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use:
|
||||
|
||||
```sh
|
||||
sudo --preserve-env=PG_CONFIG make install
|
||||
```
|
||||
Then re-run the installation instructions (run `make clean` before `make` if needed)
|
||||
|
||||
### Missing Header
|
||||
|
||||
@@ -386,7 +367,7 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`,
|
||||
For Ubuntu and Debian, use:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-server-dev-15
|
||||
sudo apt-get install postgresql-server-dev-15
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
@@ -397,7 +378,7 @@ Support for Windows is currently experimental. Use `nmake` to build:
|
||||
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\15"
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -418,9 +399,9 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.4.1 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --build-arg PG_MAJOR=15 -t myuser/pgvector .
|
||||
docker build -t pgvector .
|
||||
```
|
||||
|
||||
### Homebrew
|
||||
@@ -431,8 +412,6 @@ With Homebrew Postgres, you can use:
|
||||
brew install pgvector
|
||||
```
|
||||
|
||||
Note: This only adds it to the `postgresql@14` formula
|
||||
|
||||
### PGXN
|
||||
|
||||
Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) with:
|
||||
@@ -441,16 +420,6 @@ Install from the [PostgreSQL Extension Network](https://pgxn.org/dist/vector) wi
|
||||
pgxn install vector
|
||||
```
|
||||
|
||||
### APT
|
||||
|
||||
Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run:
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql-15-pgvector
|
||||
```
|
||||
|
||||
Note: Replace `15` with your Postgres server version
|
||||
|
||||
### Yum
|
||||
|
||||
RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run:
|
||||
@@ -473,17 +442,16 @@ conda install -c conda-forge pgvector
|
||||
|
||||
This method is [community-maintained](https://github.com/conda-forge/pgvector-feedstock) by [@mmcauliffe](https://github.com/mmcauliffe)
|
||||
|
||||
### Postgres.app
|
||||
|
||||
Download the [latest release](https://postgresapp.com/downloads.html) with Postgres 15+.
|
||||
|
||||
## Hosted Postgres
|
||||
|
||||
pgvector is available on [these providers](https://github.com/pgvector/pgvector/issues/54).
|
||||
|
||||
To request a new extension on other providers:
|
||||
|
||||
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/managed-database/p/pgvector-extension-for-postgresql)
|
||||
- Amazon RDS - follow the instructions on [this page](https://aws.amazon.com/rds/postgresql/faqs/)
|
||||
- Google Cloud SQL - vote or comment on [this page](https://issuetracker.google.com/issues/265172065)
|
||||
- Azure Database - vote or comment on [this page](https://feedback.azure.com/d365community/idea/7b423322-6189-ed11-a81b-000d3ae49307)
|
||||
- DigitalOcean Managed Databases - vote or comment on [this page](https://ideas.digitalocean.com/app-framework-services/p/pgvector-extension-for-postgresql)
|
||||
- Heroku Postgres - vote or comment on [this page](https://github.com/heroku/roadmap/issues/156)
|
||||
|
||||
## Upgrading
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.2'" to load this file. \quit
|
||||
@@ -1,2 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.3'" to load this file. \quit
|
||||
@@ -1,2 +0,0 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.4.4'" to load this file. \quit
|
||||
@@ -147,7 +147,7 @@ AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState
|
||||
{
|
||||
double distance;
|
||||
double minDistance = DBL_MAX;
|
||||
int closestCenter = 0;
|
||||
int closestCenter = -1;
|
||||
VectorArray centers = buildstate->centers;
|
||||
TupleTableSlot *slot = buildstate->slot;
|
||||
int i;
|
||||
@@ -438,8 +438,8 @@ ComputeCenters(IvfflatBuildState * buildstate)
|
||||
{
|
||||
ereport(NOTICE,
|
||||
(errmsg("ivfflat index created with little data"),
|
||||
errdetail("This will cause low recall."),
|
||||
errhint("Drop the index until the table has more data.")));
|
||||
errdetail("this will cause poor recall"),
|
||||
errhint("drop the index until the table has more data")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,21 +568,6 @@ PrintKmeansMetrics(IvfflatBuildState * buildstate)
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Scan table for tuples to index
|
||||
*/
|
||||
static void
|
||||
ScanTable(IvfflatBuildState * buildstate)
|
||||
{
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
|
||||
true, true, BuildCallback, (void *) buildstate, NULL);
|
||||
#else
|
||||
buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
|
||||
true, BuildCallback, (void *) buildstate, NULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Create entry pages
|
||||
*/
|
||||
@@ -590,7 +575,7 @@ static void
|
||||
CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
|
||||
{
|
||||
AttrNumber attNums[] = {1};
|
||||
Oid sortOperators[] = {Int4LessOperator};
|
||||
Oid sortOperators[] = {Float8LessOperator};
|
||||
Oid sortCollations[] = {InvalidOid};
|
||||
bool nullsFirstFlags[] = {false};
|
||||
|
||||
@@ -600,17 +585,25 @@ CreateEntryPages(IvfflatBuildState * buildstate, ForkNumber forkNum)
|
||||
|
||||
/* Add tuples to sort */
|
||||
if (buildstate->heap != NULL)
|
||||
IvfflatBench("assign tuples", ScanTable(buildstate));
|
||||
{
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
buildstate->reltuples = table_index_build_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
|
||||
true, true, BuildCallback, (void *) buildstate, NULL);
|
||||
#else
|
||||
buildstate->reltuples = IndexBuildHeapScan(buildstate->heap, buildstate->index, buildstate->indexInfo,
|
||||
true, BuildCallback, (void *) buildstate, NULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Sort */
|
||||
IvfflatBench("sort tuples", tuplesort_performsort(buildstate->sortstate));
|
||||
tuplesort_performsort(buildstate->sortstate);
|
||||
|
||||
#ifdef IVFFLAT_KMEANS_DEBUG
|
||||
PrintKmeansMetrics(buildstate);
|
||||
#endif
|
||||
|
||||
/* Insert */
|
||||
IvfflatBench("load tuples", InsertTuples(buildstate->index, buildstate, forkNum));
|
||||
InsertTuples(buildstate->index, buildstate, forkNum);
|
||||
tuplesort_end(buildstate->sortstate);
|
||||
}
|
||||
|
||||
@@ -628,7 +621,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo,
|
||||
/* Create pages */
|
||||
CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum);
|
||||
CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo);
|
||||
CreateEntryPages(buildstate, forkNum);
|
||||
IvfflatBench("CreateEntryPages", CreateEntryPages(buildstate, forkNum));
|
||||
|
||||
FreeBuildState(buildstate);
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include "ivfflat.h"
|
||||
#include "utils/guc.h"
|
||||
#include "utils/selfuncs.h"
|
||||
#include "utils/spccache.h"
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
#include "commands/progress.h"
|
||||
@@ -64,13 +63,13 @@ ivfflatbuildphasename(int64 phasenum)
|
||||
static void
|
||||
ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
Cost *indexStartupCost, Cost *indexTotalCost,
|
||||
Selectivity *indexSelectivity, double *indexCorrelation,
|
||||
double *indexPages)
|
||||
Selectivity *indexSelectivity, double *indexCorrelation
|
||||
,double *indexPages
|
||||
)
|
||||
{
|
||||
GenericCosts costs;
|
||||
int lists;
|
||||
double ratio;
|
||||
double spc_seq_page_cost;
|
||||
Relation indexRel;
|
||||
#if PG_VERSION_NUM < 120000
|
||||
List *qinfos;
|
||||
@@ -89,22 +88,6 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
|
||||
MemSet(&costs, 0, sizeof(costs));
|
||||
|
||||
indexRel = index_open(path->indexinfo->indexoid, NoLock);
|
||||
lists = IvfflatGetLists(indexRel);
|
||||
index_close(indexRel, NoLock);
|
||||
|
||||
/* Get the ratio of lists that we need to visit */
|
||||
ratio = ((double) ivfflat_probes) / lists;
|
||||
if (ratio > 1.0)
|
||||
ratio = 1.0;
|
||||
|
||||
/*
|
||||
* This gives us the subset of tuples to visit. This value is passed into
|
||||
* the generic cost estimator to determine the number of pages to visit
|
||||
* during the index scan.
|
||||
*/
|
||||
costs.numIndexTuples = path->indexinfo->tuples * ratio;
|
||||
|
||||
#if PG_VERSION_NUM >= 120000
|
||||
genericcostestimate(root, path, loop_count, &costs);
|
||||
#else
|
||||
@@ -112,40 +95,22 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count,
|
||||
genericcostestimate(root, path, loop_count, qinfos, &costs);
|
||||
#endif
|
||||
|
||||
get_tablespace_page_costs(path->indexinfo->reltablespace, NULL, &spc_seq_page_cost);
|
||||
indexRel = index_open(path->indexinfo->indexoid, NoLock);
|
||||
lists = IvfflatGetLists(indexRel);
|
||||
index_close(indexRel, NoLock);
|
||||
|
||||
/* Adjust cost if needed since TOAST not included in seq scan cost */
|
||||
if (costs.numIndexPages > path->indexinfo->rel->pages && ratio < 0.5)
|
||||
{
|
||||
/* Change all page cost from random to sequential */
|
||||
costs.indexTotalCost -= costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
ratio = ((double) ivfflat_probes) / lists;
|
||||
if (ratio > 1)
|
||||
ratio = 1;
|
||||
|
||||
/* Remove cost of extra pages */
|
||||
costs.indexTotalCost -= (costs.numIndexPages - path->indexinfo->rel->pages) * spc_seq_page_cost;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Change some page cost from random to sequential */
|
||||
costs.indexTotalCost -= 0.5 * costs.numIndexPages * (costs.spc_random_page_cost - spc_seq_page_cost);
|
||||
}
|
||||
costs.indexTotalCost *= ratio;
|
||||
|
||||
/*
|
||||
* If the list selectivity is lower than what is returned from the generic
|
||||
* cost estimator, use that.
|
||||
*/
|
||||
if (ratio < costs.indexSelectivity)
|
||||
costs.indexSelectivity = ratio;
|
||||
|
||||
/* Use total cost since most work happens before first tuple is returned */
|
||||
/* Startup cost and total cost are same */
|
||||
*indexStartupCost = costs.indexTotalCost;
|
||||
*indexTotalCost = costs.indexTotalCost;
|
||||
*indexSelectivity = costs.indexSelectivity;
|
||||
*indexCorrelation = costs.indexCorrelation;
|
||||
*indexPages = costs.numIndexPages;
|
||||
|
||||
elog(INFO, "ivfflatcostestimate = %f", costs.indexTotalCost);
|
||||
/* Cost estimates for parallel workers applied outside of amcostestimate */
|
||||
elog(INFO, "parallel_workers = %d, parallel aware = %d", path->path.parallel_workers, path->path.parallel_aware);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -186,25 +151,6 @@ ivfflatvalidate(Oid opclassoid)
|
||||
return true;
|
||||
}
|
||||
|
||||
static Size
|
||||
ivfflatestimateparallelscan()
|
||||
{
|
||||
elog(INFO, "ivfflatestimateparallelscan");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
ivfflatinitparallelscan(void *target)
|
||||
{
|
||||
elog(INFO, "ivfflatinitparallelscan");
|
||||
}
|
||||
|
||||
static void
|
||||
ivfflatparallelrescan(IndexScanDesc scan)
|
||||
{
|
||||
elog(INFO, "ivfflatparallelrescan");
|
||||
}
|
||||
|
||||
/*
|
||||
* Define index handler
|
||||
*
|
||||
@@ -232,7 +178,7 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amstorage = false;
|
||||
amroutine->amclusterable = false;
|
||||
amroutine->ampredlocks = false;
|
||||
amroutine->amcanparallel = true;
|
||||
amroutine->amcanparallel = false;
|
||||
amroutine->amcaninclude = false;
|
||||
#if PG_VERSION_NUM >= 130000
|
||||
amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */
|
||||
@@ -266,9 +212,9 @@ ivfflathandler(PG_FUNCTION_ARGS)
|
||||
amroutine->amrestrpos = NULL;
|
||||
|
||||
/* Interface functions to support parallel index scans */
|
||||
amroutine->amestimateparallelscan = ivfflatestimateparallelscan;
|
||||
amroutine->aminitparallelscan = ivfflatinitparallelscan;
|
||||
amroutine->amparallelrescan = ivfflatparallelrescan;
|
||||
amroutine->amestimateparallelscan = NULL;
|
||||
amroutine->aminitparallelscan = NULL;
|
||||
amroutine->amparallelrescan = NULL;
|
||||
|
||||
PG_RETURN_POINTER(amroutine);
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ typedef struct IvfflatScanOpaqueData
|
||||
|
||||
/* Lists */
|
||||
pairingheap *listQueue;
|
||||
double minDistance;
|
||||
IvfflatScanList lists[FLEXIBLE_ARRAY_MEMBER]; /* must come last */
|
||||
} IvfflatScanOpaqueData;
|
||||
|
||||
|
||||
@@ -23,10 +23,6 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
|
||||
OffsetNumber offno;
|
||||
OffsetNumber maxoffno;
|
||||
|
||||
/* Avoid compiler warning */
|
||||
listInfo->blkno = nextblkno;
|
||||
listInfo->offno = FirstOffsetNumber;
|
||||
|
||||
procinfo = index_getprocinfo(rel, 1, IVFFLAT_DISTANCE_PROC);
|
||||
collation = rel->rd_indcollation[0];
|
||||
|
||||
@@ -43,7 +39,7 @@ FindInsertPage(Relation rel, Datum *values, BlockNumber *insertPage, ListInfo *
|
||||
list = (IvfflatList) PageGetItem(cpage, PageGetItemId(cpage, offno));
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(procinfo, collation, values[0], PointerGetDatum(&list->center)));
|
||||
|
||||
if (distance < minDistance || !BlockNumberIsValid(*insertPage))
|
||||
if (distance < minDistance)
|
||||
{
|
||||
*insertPage = list->insertPage;
|
||||
listInfo->blkno = nextblkno;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "ivfflat.h"
|
||||
#include "miscadmin.h"
|
||||
@@ -212,7 +211,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
/* Check memory requirements */
|
||||
/* Add one to error message to ceil */
|
||||
if (totalSize > (Size) maintenance_work_mem * 1024L)
|
||||
if (totalSize / 1024 > maintenance_work_mem)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||
errmsg("memory required is %zu MB, maintenance_work_mem is %d MB",
|
||||
@@ -252,7 +251,7 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
for (j = 0; j < numSamples; j++)
|
||||
{
|
||||
minDistance = DBL_MAX;
|
||||
closestCenter = 0;
|
||||
closestCenter = -1;
|
||||
|
||||
/* Find closest center */
|
||||
for (k = 0; k < numCenters; k++)
|
||||
@@ -399,14 +398,6 @@ ElkanKmeans(Relation index, VectorArray samples, VectorArray centers)
|
||||
|
||||
if (centerCounts[j] > 0)
|
||||
{
|
||||
/* Double avoids overflow, but requires more memory */
|
||||
/* TODO Update bounds */
|
||||
for (k = 0; k < dimensions; k++)
|
||||
{
|
||||
if (isinf(vec->x[k]))
|
||||
vec->x[k] = vec->x[k] > 0 ? FLT_MAX : -FLT_MAX;
|
||||
}
|
||||
|
||||
for (k = 0; k < dimensions; k++)
|
||||
vec->x[k] /= centerCounts[j];
|
||||
}
|
||||
@@ -470,29 +461,12 @@ CheckCenters(Relation index, VectorArray centers)
|
||||
{
|
||||
FmgrInfo *normprocinfo;
|
||||
Oid collation;
|
||||
Vector *vec;
|
||||
int i;
|
||||
int j;
|
||||
double norm;
|
||||
|
||||
if (centers->length != centers->maxlen)
|
||||
elog(ERROR, "Not enough centers. Please report a bug.");
|
||||
|
||||
/* Ensure no NaN or infinite values */
|
||||
for (i = 0; i < centers->length; i++)
|
||||
{
|
||||
vec = VectorArrayGet(centers, i);
|
||||
|
||||
for (j = 0; j < vec->dim; j++)
|
||||
{
|
||||
if (isnan(vec->x[j]))
|
||||
elog(ERROR, "NaN detected. Please report a bug.");
|
||||
|
||||
if (isinf(vec->x[j]))
|
||||
elog(ERROR, "Infinite value detected. Please report a bug.");
|
||||
}
|
||||
}
|
||||
|
||||
/* Ensure no duplicate centers */
|
||||
/* Fine to sort in-place */
|
||||
qsort(centers->items, centers->length, VECTOR_SIZE(centers->dim), CompareVectors);
|
||||
|
||||
@@ -60,6 +60,9 @@ GetScanLists(IndexScanDesc scan, Datum value)
|
||||
/* Use procinfo from the index instead of scan key for performance */
|
||||
distance = DatumGetFloat8(FunctionCall2Coll(so->procinfo, so->collation, PointerGetDatum(&list->center), value));
|
||||
|
||||
if (distance < so->minDistance)
|
||||
so->minDistance = distance;
|
||||
|
||||
if (listCount < so->probes)
|
||||
{
|
||||
scanlist = &so->lists[listCount];
|
||||
@@ -126,13 +129,16 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
*/
|
||||
BufferAccessStrategy bas = GetAccessStrategy(BAS_BULKREAD);
|
||||
|
||||
if (scan->parallel_scan != NULL)
|
||||
elog(INFO, "parallel scan");
|
||||
|
||||
/* Search closest probes lists */
|
||||
while (!pairingheap_is_empty(so->listQueue))
|
||||
{
|
||||
searchPage = ((IvfflatScanList *) pairingheap_remove_first(so->listQueue))->startPage;
|
||||
IvfflatScanList *scanlist = (IvfflatScanList *) pairingheap_remove_first(so->listQueue);
|
||||
|
||||
/* Query-aware dynamic pruning */
|
||||
if (fabs(scanlist->distance) > 1.5 * fabs(so->minDistance))
|
||||
continue;
|
||||
|
||||
searchPage = scanlist->startPage;
|
||||
|
||||
/* Search all entry pages for list */
|
||||
while (BlockNumberIsValid(searchPage))
|
||||
@@ -173,14 +179,12 @@ GetScanItems(IndexScanDesc scan, Datum value)
|
||||
}
|
||||
}
|
||||
|
||||
FreeAccessStrategy(bas);
|
||||
|
||||
/* TODO Scan more lists */
|
||||
if (tuples < 100)
|
||||
ereport(DEBUG1,
|
||||
(errmsg("index scan found few tuples"),
|
||||
errdetail("Index may have been created with little data."),
|
||||
errhint("Recreate the index and possibly decrease lists.")));
|
||||
errdetail("index may have been created without data or lists is too high"),
|
||||
errhint("recreate the index and possibly decrease lists")));
|
||||
|
||||
tuplesort_performsort(so->sortstate);
|
||||
}
|
||||
@@ -257,6 +261,7 @@ ivfflatrescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int
|
||||
|
||||
so->first = true;
|
||||
pairingheap_reset(so->listQueue);
|
||||
so->minDistance = DBL_MAX;
|
||||
|
||||
if (keys && scan->numberOfKeys > 0)
|
||||
memmove(scan->keyData, keys, scan->numberOfKeys * sizeof(ScanKeyData));
|
||||
|
||||
@@ -132,8 +132,6 @@ ivfflatbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
|
||||
}
|
||||
}
|
||||
|
||||
FreeAccessStrategy(bas);
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
|
||||
163
src/vector.c
163
src/vector.c
@@ -42,7 +42,7 @@ CheckDims(Vector * a, Vector * b)
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure expected dimensions
|
||||
* Ensure expected dimension
|
||||
*/
|
||||
static inline void
|
||||
CheckExpectedDim(int32 typmod, int dim)
|
||||
@@ -53,9 +53,7 @@ CheckExpectedDim(int32 typmod, int dim)
|
||||
errmsg("expected %d dimensions, not %d", typmod, dim)));
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure valid dimensions
|
||||
*/
|
||||
|
||||
static inline void
|
||||
CheckDim(int dim)
|
||||
{
|
||||
@@ -81,28 +79,13 @@ CheckElement(float value)
|
||||
(errcode(ERRCODE_DATA_EXCEPTION),
|
||||
errmsg("NaN not allowed in vector")));
|
||||
|
||||
|
||||
if (isinf(value))
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_DATA_EXCEPTION),
|
||||
errmsg("infinite value not allowed in vector")));
|
||||
}
|
||||
|
||||
/*
|
||||
* Check for whitespace, since array_isspace() is static
|
||||
*/
|
||||
static inline bool
|
||||
vector_isspace(char ch)
|
||||
{
|
||||
if (ch == ' ' ||
|
||||
ch == '\t' ||
|
||||
ch == '\n' ||
|
||||
ch == '\r' ||
|
||||
ch == '\v' ||
|
||||
ch == '\f')
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check state array
|
||||
*/
|
||||
@@ -127,6 +110,30 @@ float_overflow_error(void)
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Print vector - useful for debugging
|
||||
*/
|
||||
void
|
||||
PrintVector(char *msg, Vector * vector)
|
||||
{
|
||||
StringInfoData buf;
|
||||
int dim = vector->dim;
|
||||
int i;
|
||||
|
||||
initStringInfo(&buf);
|
||||
|
||||
appendStringInfoChar(&buf, '[');
|
||||
for (i = 0; i < dim; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
appendStringInfoString(&buf, ",");
|
||||
appendStringInfoString(&buf, float8out_internal(vector->x[i]));
|
||||
}
|
||||
appendStringInfoChar(&buf, ']');
|
||||
|
||||
elog(INFO, "%s = %s", msg, buf.data);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert textual representation to internal representation
|
||||
*/
|
||||
@@ -142,15 +149,11 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
char *pt;
|
||||
char *stringEnd;
|
||||
Vector *result;
|
||||
char *lit = pstrdup(str);
|
||||
|
||||
while (vector_isspace(*str))
|
||||
str++;
|
||||
|
||||
if (*str != '[')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("malformed vector literal: \"%s\"", lit),
|
||||
errmsg("malformed vector literal: \"%s\"", str),
|
||||
errdetail("Vector contents must start with \"[\".")));
|
||||
|
||||
str++;
|
||||
@@ -164,15 +167,6 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
|
||||
errmsg("vector cannot have more than %d dimensions", VECTOR_MAX_DIM)));
|
||||
|
||||
while (vector_isspace(*pt))
|
||||
pt++;
|
||||
|
||||
/* Check for empty string like float4in */
|
||||
if (*pt == '\0')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||
|
||||
/* Use strtof like float4in to avoid a double-rounding problem */
|
||||
x[dim] = strtof(pt, &stringEnd);
|
||||
CheckElement(x[dim]);
|
||||
@@ -181,53 +175,33 @@ vector_in(PG_FUNCTION_ARGS)
|
||||
if (stringEnd == pt)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||
|
||||
while (vector_isspace(*stringEnd))
|
||||
stringEnd++;
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", pt)));
|
||||
|
||||
if (*stringEnd != '\0' && *stringEnd != ']')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", lit)));
|
||||
errmsg("invalid input syntax for type vector: \"%s\"", pt)));
|
||||
|
||||
pt = strtok(NULL, ",");
|
||||
}
|
||||
|
||||
if (stringEnd == NULL || *stringEnd != ']')
|
||||
if (*stringEnd != ']')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("malformed vector literal: \"%s\"", lit),
|
||||
errmsg("malformed vector literal"),
|
||||
errdetail("Unexpected end of input.")));
|
||||
|
||||
stringEnd++;
|
||||
|
||||
/* Only whitespace is allowed after the closing brace */
|
||||
while (vector_isspace(*stringEnd))
|
||||
stringEnd++;
|
||||
|
||||
if (*stringEnd != '\0')
|
||||
if (stringEnd[1] != '\0')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("malformed vector literal: \"%s\"", lit),
|
||||
errmsg("malformed vector literal"),
|
||||
errdetail("Junk after closing right brace.")));
|
||||
|
||||
/* Ensure no consecutive delimiters since strtok skips */
|
||||
for (pt = lit + 1; *pt != '\0'; pt++)
|
||||
{
|
||||
if (pt[-1] == ',' && *pt == ',')
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
|
||||
errmsg("malformed vector literal: \"%s\"", lit)));
|
||||
}
|
||||
|
||||
if (dim < 1)
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_DATA_EXCEPTION),
|
||||
errmsg("vector must have at least 1 dimension")));
|
||||
|
||||
pfree(lit);
|
||||
|
||||
CheckExpectedDim(typmod, dim);
|
||||
|
||||
result = InitVector(dim);
|
||||
@@ -298,18 +272,6 @@ vector_out(PG_FUNCTION_ARGS)
|
||||
PG_RETURN_CSTRING(buf);
|
||||
}
|
||||
|
||||
/*
|
||||
* Print vector - useful for debugging
|
||||
*/
|
||||
void
|
||||
PrintVector(char *msg, Vector * vector)
|
||||
{
|
||||
char *out = DatumGetPointer(DirectFunctionCall1(vector_out, PointerGetDatum(vector)));
|
||||
|
||||
elog(INFO, "%s = %s", msg, out);
|
||||
pfree(out);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert type modifier
|
||||
*/
|
||||
@@ -368,10 +330,7 @@ vector_recv(PG_FUNCTION_ARGS)
|
||||
|
||||
result = InitVector(dim);
|
||||
for (i = 0; i < dim; i++)
|
||||
{
|
||||
result->x[i] = pq_getmsgfloat4(buf);
|
||||
CheckElement(result->x[i]);
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
@@ -437,8 +396,10 @@ array_to_vector(PG_FUNCTION_ARGS)
|
||||
get_typlenbyvalalign(ARR_ELEMTYPE(array), &typlen, &typbyval, &typalign);
|
||||
deconstruct_array(array, ARR_ELEMTYPE(array), typlen, typbyval, typalign, &elemsp, &nullsp, &nelemsp);
|
||||
|
||||
CheckDim(nelemsp);
|
||||
CheckExpectedDim(typmod, nelemsp);
|
||||
if (typmod == -1)
|
||||
CheckDim(nelemsp);
|
||||
else
|
||||
CheckExpectedDim(typmod, nelemsp);
|
||||
|
||||
result = InitVector(nelemsp);
|
||||
for (i = 0; i < nelemsp; i++)
|
||||
@@ -448,7 +409,6 @@ array_to_vector(PG_FUNCTION_ARGS)
|
||||
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
|
||||
errmsg("array must not containing NULLs")));
|
||||
|
||||
/* TODO Move outside loop in 0.5.0 */
|
||||
if (ARR_ELEMTYPE(array) == INT4OID)
|
||||
result->x[i] = DatumGetInt32(elemsp[i]);
|
||||
else if (ARR_ELEMTYPE(array) == FLOAT8OID)
|
||||
@@ -476,19 +436,17 @@ Datum
|
||||
vector_to_float4(PG_FUNCTION_ARGS)
|
||||
{
|
||||
Vector *vec = PG_GETARG_VECTOR_P(0);
|
||||
Datum *datums;
|
||||
Datum *d;
|
||||
ArrayType *result;
|
||||
int i;
|
||||
|
||||
datums = (Datum *) palloc(sizeof(Datum) * vec->dim);
|
||||
d = (Datum *) palloc(sizeof(Datum) * vec->dim);
|
||||
|
||||
for (i = 0; i < vec->dim; i++)
|
||||
datums[i] = Float4GetDatum(vec->x[i]);
|
||||
d[i] = Float4GetDatum(vec->x[i]);
|
||||
|
||||
/* Use TYPALIGN_INT for float4 */
|
||||
result = construct_array(datums, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT);
|
||||
|
||||
pfree(datums);
|
||||
result = construct_array(d, vec->dim, FLOAT4OID, sizeof(float4), true, TYPALIGN_INT);
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
@@ -700,13 +658,6 @@ vector_add(PG_FUNCTION_ARGS)
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] + bx[i];
|
||||
|
||||
/* Check for overflow */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
{
|
||||
if (isinf(rx[i]))
|
||||
float_overflow_error();
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
@@ -733,13 +684,6 @@ vector_sub(PG_FUNCTION_ARGS)
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
rx[i] = ax[i] - bx[i];
|
||||
|
||||
/* Check for overflow */
|
||||
for (int i = 0, imax = a->dim; i < imax; i++)
|
||||
{
|
||||
if (isinf(rx[i]))
|
||||
float_overflow_error();
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
}
|
||||
|
||||
@@ -885,12 +829,12 @@ vector_accum(PG_FUNCTION_ARGS)
|
||||
n = statevalues[0] + 1.0;
|
||||
|
||||
statedatums = CreateStateDatums(dim);
|
||||
statedatums[0] = Float8GetDatum(n);
|
||||
statedatums[0] = Float8GetDatumFast(n);
|
||||
|
||||
if (newarr)
|
||||
{
|
||||
for (int i = 0; i < dim; i++)
|
||||
statedatums[i + 1] = Float8GetDatum((double) x[i]);
|
||||
statedatums[i + 1] = Float8GetDatumFast((double) x[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -898,11 +842,10 @@ vector_accum(PG_FUNCTION_ARGS)
|
||||
{
|
||||
double v = statevalues[i + 1] + x[i];
|
||||
|
||||
/* Check for overflow */
|
||||
if (isinf(v))
|
||||
float_overflow_error();
|
||||
|
||||
statedatums[i + 1] = Float8GetDatum(v);
|
||||
statedatums[i + 1] = Float8GetDatumFast(v);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -947,7 +890,7 @@ vector_combine(PG_FUNCTION_ARGS)
|
||||
dim = STATE_DIMS(statearray2);
|
||||
statedatums = CreateStateDatums(dim);
|
||||
for (int i = 1; i <= dim; i++)
|
||||
statedatums[i] = Float8GetDatum(statevalues2[i]);
|
||||
statedatums[i] = Float8GetDatumFast(statevalues2[i]);
|
||||
}
|
||||
else if (n2 == 0.0)
|
||||
{
|
||||
@@ -955,7 +898,7 @@ vector_combine(PG_FUNCTION_ARGS)
|
||||
dim = STATE_DIMS(statearray1);
|
||||
statedatums = CreateStateDatums(dim);
|
||||
for (int i = 1; i <= dim; i++)
|
||||
statedatums[i] = Float8GetDatum(statevalues1[i]);
|
||||
statedatums[i] = Float8GetDatumFast(statevalues1[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -967,15 +910,14 @@ vector_combine(PG_FUNCTION_ARGS)
|
||||
{
|
||||
double v = statevalues1[i] + statevalues2[i];
|
||||
|
||||
/* Check for overflow */
|
||||
if (isinf(v))
|
||||
float_overflow_error();
|
||||
|
||||
statedatums[i] = Float8GetDatum(v);
|
||||
statedatums[i] = Float8GetDatumFast(v);
|
||||
}
|
||||
}
|
||||
|
||||
statedatums[0] = Float8GetDatum(n);
|
||||
statedatums[0] = Float8GetDatumFast(n);
|
||||
|
||||
result = construct_array(statedatums, dim + 1,
|
||||
FLOAT8OID,
|
||||
@@ -998,6 +940,7 @@ vector_avg(PG_FUNCTION_ARGS)
|
||||
float8 n;
|
||||
uint16 dim;
|
||||
Vector *result;
|
||||
float v;
|
||||
|
||||
/* Check array before using */
|
||||
statevalues = CheckStateArray(statearray, "vector_avg");
|
||||
@@ -1009,12 +952,12 @@ vector_avg(PG_FUNCTION_ARGS)
|
||||
|
||||
/* Create vector */
|
||||
dim = STATE_DIMS(statearray);
|
||||
CheckDim(dim);
|
||||
result = InitVector(dim);
|
||||
for (int i = 0; i < dim; i++)
|
||||
{
|
||||
result->x[i] = statevalues[i + 1] / n;
|
||||
CheckElement(result->x[i]);
|
||||
v = statevalues[i + 1] / n;
|
||||
CheckElement(v);
|
||||
result->x[i] = v;
|
||||
}
|
||||
|
||||
PG_RETURN_POINTER(result);
|
||||
|
||||
@@ -46,8 +46,6 @@ SELECT '[1,2,3]'::vector::real[];
|
||||
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
-- ensure no error
|
||||
SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
|
||||
?column?
|
||||
|
||||
@@ -4,16 +4,12 @@ SELECT '[1,2,3]'::vector + '[4,5,6]';
|
||||
[5,7,9]
|
||||
(1 row)
|
||||
|
||||
SELECT '[3e38]'::vector + '[3e38]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT '[1,2,3]'::vector - '[4,5,6]';
|
||||
?column?
|
||||
------------
|
||||
[-3,-3,-3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[-3e38]'::vector - '[3e38]';
|
||||
ERROR: value out of range: overflow
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
vector_dims
|
||||
-------------
|
||||
@@ -106,5 +102,3 @@ SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
|
||||
ERROR: expected 2 dimensions, not 1
|
||||
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
|
||||
ERROR: vector cannot have more than 16000 dimensions
|
||||
|
||||
@@ -4,22 +4,10 @@ SELECT '[1,2,3]'::vector;
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[-1,-2,-3]'::vector;
|
||||
vector
|
||||
------------
|
||||
[-1,-2,-3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.,2.,3.]'::vector;
|
||||
vector
|
||||
---------
|
||||
[1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT ' [ 1, 2 , 3 ] '::vector;
|
||||
vector
|
||||
---------
|
||||
[1,2,3]
|
||||
SELECT '[-1,2,3]'::vector;
|
||||
vector
|
||||
----------
|
||||
[-1,2,3]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.23456]'::vector;
|
||||
@@ -29,7 +17,7 @@ SELECT '[1.23456]'::vector;
|
||||
(1 row)
|
||||
|
||||
SELECT '[hello,1]'::vector;
|
||||
ERROR: invalid input syntax for type vector: "[hello,1]"
|
||||
ERROR: invalid input syntax for type vector: "hello"
|
||||
LINE 1: SELECT '[hello,1]'::vector;
|
||||
^
|
||||
SELECT '[NaN,1]'::vector;
|
||||
@@ -44,35 +32,13 @@ SELECT '[-Infinity,1]'::vector;
|
||||
ERROR: infinite value not allowed in vector
|
||||
LINE 1: SELECT '[-Infinity,1]'::vector;
|
||||
^
|
||||
SELECT '[1.5e38,-1.5e38]'::vector;
|
||||
vector
|
||||
--------------------
|
||||
[1.5e+38,-1.5e+38]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.5e+38,-1.5e+38]'::vector;
|
||||
vector
|
||||
--------------------
|
||||
[1.5e+38,-1.5e+38]
|
||||
(1 row)
|
||||
|
||||
SELECT '[1.5e-38,-1.5e-38]'::vector;
|
||||
vector
|
||||
--------------------
|
||||
[1.5e-38,-1.5e-38]
|
||||
(1 row)
|
||||
|
||||
SELECT '[4e38,1]'::vector;
|
||||
ERROR: infinite value not allowed in vector
|
||||
LINE 1: SELECT '[4e38,1]'::vector;
|
||||
^
|
||||
SELECT '[1,2,3'::vector;
|
||||
ERROR: malformed vector literal: "[1,2,3"
|
||||
ERROR: malformed vector literal
|
||||
LINE 1: SELECT '[1,2,3'::vector;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[1,2,3]9'::vector;
|
||||
ERROR: malformed vector literal: "[1,2,3]9"
|
||||
ERROR: malformed vector literal
|
||||
LINE 1: SELECT '[1,2,3]9'::vector;
|
||||
^
|
||||
DETAIL: Junk after closing right brace.
|
||||
@@ -81,36 +47,14 @@ ERROR: malformed vector literal: "1,2,3"
|
||||
LINE 1: SELECT '1,2,3'::vector;
|
||||
^
|
||||
DETAIL: Vector contents must start with "[".
|
||||
SELECT '['::vector;
|
||||
ERROR: malformed vector literal: "["
|
||||
LINE 1: SELECT '['::vector;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[,'::vector;
|
||||
ERROR: malformed vector literal: "[,"
|
||||
LINE 1: SELECT '[,'::vector;
|
||||
^
|
||||
DETAIL: Unexpected end of input.
|
||||
SELECT '[]'::vector;
|
||||
ERROR: vector must have at least 1 dimension
|
||||
LINE 1: SELECT '[]'::vector;
|
||||
^
|
||||
SELECT '[1,]'::vector;
|
||||
ERROR: invalid input syntax for type vector: "[1,]"
|
||||
ERROR: invalid input syntax for type vector: "]"
|
||||
LINE 1: SELECT '[1,]'::vector;
|
||||
^
|
||||
SELECT '[1a]'::vector;
|
||||
ERROR: invalid input syntax for type vector: "[1a]"
|
||||
LINE 1: SELECT '[1a]'::vector;
|
||||
^
|
||||
SELECT '[1,,3]'::vector;
|
||||
ERROR: malformed vector literal: "[1,,3]"
|
||||
LINE 1: SELECT '[1,,3]'::vector;
|
||||
^
|
||||
SELECT '[1, ,3]'::vector;
|
||||
ERROR: invalid input syntax for type vector: "[1, ,3]"
|
||||
LINE 1: SELECT '[1, ,3]'::vector;
|
||||
^
|
||||
SELECT '[1,2,3]'::vector(2);
|
||||
ERROR: expected 2 dimensions, not 3
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||
|
||||
@@ -10,7 +10,6 @@ SELECT '{-Infinity}'::real[]::vector;
|
||||
SELECT '{}'::real[]::vector;
|
||||
SELECT '[1,2,3]'::vector::real[];
|
||||
SELECT array_agg(n)::vector FROM generate_series(1, 16001) n;
|
||||
SELECT array_to_vector(array_agg(n), 16001, false) FROM generate_series(1, 16001) n;
|
||||
|
||||
-- ensure no error
|
||||
SELECT ARRAY[1,2,3] = ARRAY[1,2,3];
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
SELECT '[1,2,3]'::vector + '[4,5,6]';
|
||||
SELECT '[3e38]'::vector + '[3e38]';
|
||||
SELECT '[1,2,3]'::vector - '[4,5,6]';
|
||||
SELECT '[-3e38]'::vector - '[3e38]';
|
||||
|
||||
SELECT vector_dims('[1,2,3]');
|
||||
|
||||
@@ -26,4 +24,3 @@ SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]']) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2,3]'::vector, '[3,5,7]', NULL]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY[]::vector[]) v;
|
||||
SELECT avg(v) FROM unnest(ARRAY['[1,2]'::vector, '[3]']) v;
|
||||
SELECT vector_avg(array_agg(n)) FROM generate_series(1, 16002) n;
|
||||
|
||||
@@ -1,26 +1,15 @@
|
||||
SELECT '[1,2,3]'::vector;
|
||||
SELECT '[-1,-2,-3]'::vector;
|
||||
SELECT '[1.,2.,3.]'::vector;
|
||||
SELECT ' [ 1, 2 , 3 ] '::vector;
|
||||
SELECT '[-1,2,3]'::vector;
|
||||
SELECT '[1.23456]'::vector;
|
||||
SELECT '[hello,1]'::vector;
|
||||
SELECT '[NaN,1]'::vector;
|
||||
SELECT '[Infinity,1]'::vector;
|
||||
SELECT '[-Infinity,1]'::vector;
|
||||
SELECT '[1.5e38,-1.5e38]'::vector;
|
||||
SELECT '[1.5e+38,-1.5e+38]'::vector;
|
||||
SELECT '[1.5e-38,-1.5e-38]'::vector;
|
||||
SELECT '[4e38,1]'::vector;
|
||||
SELECT '[1,2,3'::vector;
|
||||
SELECT '[1,2,3]9'::vector;
|
||||
SELECT '1,2,3'::vector;
|
||||
SELECT '['::vector;
|
||||
SELECT '[,'::vector;
|
||||
SELECT '[]'::vector;
|
||||
SELECT '[1,]'::vector;
|
||||
SELECT '[1a]'::vector;
|
||||
SELECT '[1,,3]'::vector;
|
||||
SELECT '[1, ,3]'::vector;
|
||||
SELECT '[1,2,3]'::vector(2);
|
||||
|
||||
SELECT unnest('{"[1,2,3]", "[4,5,6]"}'::vector[]);
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
-- SET force_parallel_mode = on;
|
||||
SET parallel_setup_cost = 10;
|
||||
SET parallel_tuple_cost = 0.000001;
|
||||
SET min_parallel_table_scan_size = 1;
|
||||
SET min_parallel_index_scan_size = 1;
|
||||
|
||||
CREATE TABLE t (id integer, val vector(3));
|
||||
ALTER TABLE t ALTER COLUMN val SET STORAGE PLAIN;
|
||||
INSERT INTO t (id, val) SELECT n, ARRAY[random(), random(), random()] FROM generate_series(1,1000000) n;
|
||||
CREATE INDEX ON t USING ivfflat (val) WITH (lists = 10);
|
||||
SET ivfflat.probes = 4;
|
||||
|
||||
EXPLAIN SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
|
||||
SELECT * FROM t ORDER BY val <-> '[0.5,0.5,0.5]' LIMIT 5;
|
||||
|
||||
DROP TABLE t;
|
||||
@@ -5,7 +5,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 31;
|
||||
|
||||
my $dim = 32;
|
||||
|
||||
@@ -95,5 +95,3 @@ for my $i (1 .. 10)
|
||||
);
|
||||
test_index_replay("insert $i");
|
||||
}
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 1;
|
||||
|
||||
my $dim = 3;
|
||||
|
||||
@@ -39,5 +39,3 @@ $node->safe_psql("postgres",
|
||||
# Check size
|
||||
my $new_size = $node->safe_psql("postgres", "SELECT pg_total_relation_size('tst_v_idx');");
|
||||
is($size, $new_size, "size does not change");
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 9;
|
||||
|
||||
my $node;
|
||||
my @queries = ();
|
||||
@@ -15,13 +15,6 @@ sub test_recall
|
||||
my $correct = 0;
|
||||
my $total = 0;
|
||||
|
||||
my $explain = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
SET ivfflat.probes = $probes;
|
||||
EXPLAIN ANALYZE SELECT i FROM tst ORDER BY v $operator '$queries[0]' LIMIT $limit;
|
||||
));
|
||||
like($explain, qr/Index Scan/);
|
||||
|
||||
for my $i (0 .. $#queries) {
|
||||
my $actual = $node->safe_psql("postgres", qq(
|
||||
SET enable_seqscan = off;
|
||||
@@ -79,9 +72,9 @@ foreach (@operators) {
|
||||
|
||||
# Add index
|
||||
my $opclass;
|
||||
if ($operator eq "<->") {
|
||||
if ($operator == "<->") {
|
||||
$opclass = "vector_l2_ops";
|
||||
} elsif ($operator eq "<#>") {
|
||||
} elsif ($operator == "<#>") {
|
||||
$opclass = "vector_ip_ops";
|
||||
} else {
|
||||
$opclass = "vector_cosine_ops";
|
||||
@@ -89,13 +82,7 @@ foreach (@operators) {
|
||||
$node->safe_psql("postgres", "CREATE INDEX ON tst USING ivfflat (v $opclass);");
|
||||
|
||||
# Test approximate results
|
||||
if ($operator ne "<#>") {
|
||||
# TODO fix test
|
||||
test_recall(1, 0.75, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
}
|
||||
# Account for equal distances
|
||||
test_recall(100, 0.9975, $operator);
|
||||
test_recall(1, 0.75, $operator);
|
||||
test_recall(10, 0.95, $operator);
|
||||
test_recall(100, 1.0, $operator);
|
||||
}
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 3;
|
||||
|
||||
# Initialize node
|
||||
my $node = get_new_node('node');
|
||||
@@ -34,5 +34,3 @@ $node->safe_psql("postgres",
|
||||
|
||||
# Test no error for duplicate centers
|
||||
test_centers(10);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 60;
|
||||
|
||||
# Initialize node
|
||||
my $node = get_new_node('node');
|
||||
@@ -23,9 +23,9 @@ foreach (@operators) {
|
||||
|
||||
# Add index
|
||||
my $opclass;
|
||||
if ($operator eq "<->") {
|
||||
if ($operator == "<->") {
|
||||
$opclass = "vector_l2_ops";
|
||||
} elsif ($operator eq "<#>") {
|
||||
} elsif ($operator == "<#>") {
|
||||
$opclass = "vector_ip_ops";
|
||||
} else {
|
||||
$opclass = "vector_cosine_ops";
|
||||
@@ -43,5 +43,3 @@ foreach (@operators) {
|
||||
is($res, $query);
|
||||
}
|
||||
}
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 3;
|
||||
|
||||
# Initialize node
|
||||
my $node = get_new_node('node');
|
||||
@@ -29,5 +29,3 @@ my ($ret, $stdout, $stderr) = $node->psql("postgres",
|
||||
"CREATE INDEX lists10000 ON tst USING ivfflat (v) WITH (lists = 10000);"
|
||||
);
|
||||
like($stderr, qr/memory required is/);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 7;
|
||||
|
||||
my $dim = 768;
|
||||
|
||||
@@ -53,5 +53,3 @@ $count = $node->safe_psql("postgres", qq(
|
||||
));
|
||||
is($count, $expected);
|
||||
is(idx_scan(), 1);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 5;
|
||||
|
||||
# Initialize node
|
||||
my $node = get_new_node('node');
|
||||
@@ -33,5 +33,3 @@ is($avg, "[$r1,$r2,$r3]");
|
||||
# Test explain
|
||||
my $explain = $node->safe_psql("postgres", "EXPLAIN SELECT AVG(v) FROM tst;");
|
||||
like($explain, qr/Partial Aggregate/);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -2,7 +2,7 @@ use strict;
|
||||
use warnings;
|
||||
use PostgresNode;
|
||||
use TestLib;
|
||||
use Test::More;
|
||||
use Test::More tests => 1;
|
||||
|
||||
my $dim = 1024;
|
||||
|
||||
@@ -30,5 +30,3 @@ my ($ret, $stdout, $stderr) = $node->psql("postgres",
|
||||
"INSERT INTO tst SELECT array_agg(n), array_agg(n), array_agg(n) FROM generate_series(1, $dim) n"
|
||||
);
|
||||
like($stderr, qr/row is too big/);
|
||||
|
||||
done_testing();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat access method'
|
||||
default_version = '0.4.4'
|
||||
default_version = '0.4.1'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user