database/pgvecto.rs
1
0
Fork 0
mirror of https://github.com/tensorchord/pgvecto.rs.git synced 2025-08-07 03:22:55 +03:00
Code Activity

chore: fine-grained upgrade hint (#220)

Browse Source 
* chore: upgrade instruction for every index

Signed-off-by: usamoi <usamoi@outlook.com>

* fix: soft_version check

Signed-off-by: usamoi <usamoi@outlook.com>

* fix: index_stat view if need upgrade

Signed-off-by: usamoi <usamoi@outlook.com>

* fix: size info of write segment

Signed-off-by: usamoi <usamoi@outlook.com>

---------

Signed-off-by: usamoi <usamoi@outlook.com>
This commit is contained in:
Usamoi
2024-01-03 17:19:11 +08:00
committed by GitHub
parent 3ae8ff2ad9
commit d0349f1aa8
35 changed files with 538 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/release.yml vendored
View File

@@ -87,7 +87,7 @@ jobs:
sudo apt-get update sudo apt-get update
sudo apt-get -y install libpq-dev postgresql-${{ matrix.version }} postgresql-server-dev-${{ matrix.version }} sudo apt-get -y install libpq-dev postgresql-${{ matrix.version }} postgresql-server-dev-${{ matrix.version }}
sudo apt-get -y install clang-16 sudo apt-get -y install clang-16
cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2)
cargo pgrx init --pg${{ matrix.version }}=/usr/lib/postgresql/${{ matrix.version }}/bin/pg_config cargo pgrx init --pg${{ matrix.version }}=/usr/lib/postgresql/${{ matrix.version }}/bin/pg_config
if [[ "${{ matrix.arch }}" == "aarch64" ]]; then if [[ "${{ matrix.arch }}" == "aarch64" ]]; then
sudo apt-get -y install crossbuild-essential-arm64 sudo apt-get -y install crossbuild-essential-arm64

321
Cargo.lock generated
View File

@@ -26,21 +26,6 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "anstyle" name = "anstyle"
version = "1.0.4" version = "1.0.4"
@@ -471,9 +456,9 @@ dependencies = [
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.4.3" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "bytes" name = "bytes"
@@ -531,19 +516,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
"serde",
"windows-targets 0.48.5",
]
[[package]] [[package]]
name = "clang-sys" name = "clang-sys"
version = "1.6.1" version = "1.6.1"
@@ -557,9 +529,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.11" version = "4.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@@ -577,9 +549,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.11" version = "4.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"clap_lex", "clap_lex",
@@ -779,41 +751,6 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "darling"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.43",
]
[[package]]
name = "darling_macro"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core",
"quote",
"syn 2.0.43",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "5.5.3" version = "5.5.3"
@@ -821,22 +758,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"hashbrown 0.14.3", "hashbrown",
"lock_api", "lock_api",
"once_cell", "once_cell",
"parking_lot_core", "parking_lot_core",
] ]
[[package]]
name = "deranged"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
dependencies = [
"powerfmt",
"serde",
]
[[package]] [[package]]
name = "detect" name = "detect"
version = "0.0.0" version = "0.0.0"
@@ -852,12 +779,6 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "difflib"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.10.7" version = "0.10.7"
@@ -1087,15 +1008,6 @@ dependencies = [
"miniz_oxide", "miniz_oxide",
] ]
[[package]]
name = "float-cmp"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@@ -1286,12 +1198,6 @@ dependencies = [
"byteorder", "byteorder",
] ]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.3" version = "0.14.3"
@@ -1323,12 +1229,6 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]] [[package]]
name = "hmac" name = "hmac"
version = "0.12.1" version = "0.12.1"
@@ -1429,35 +1329,6 @@ dependencies = [
"want", "want",
] ]
[[package]]
name = "iana-time-zone"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.4.0" version = "0.4.0"
@@ -1490,17 +1361,6 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.1.0" version = "2.1.0"
@@ -1508,8 +1368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.3", "hashbrown",
"serde",
] ]
[[package]] [[package]]
@@ -1534,13 +1393,13 @@ dependencies = [
[[package]] [[package]]
name = "is-terminal" name = "is-terminal"
version = "0.4.9" version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"rustix 0.38.28", "rustix 0.38.28",
"windows-sys 0.48.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@@ -1579,6 +1438,15 @@ dependencies = [
"either", "either",
] ]
[[package]]
name = "itertools"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
dependencies = [
"either",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.10" version = "1.0.10"
@@ -1614,7 +1482,7 @@ dependencies = [
"diff", "diff",
"ena", "ena",
"is-terminal", "is-terminal",
"itertools", "itertools 0.10.5",
"lalrpop-util", "lalrpop-util",
"petgraph", "petgraph",
"regex", "regex",
@@ -1750,9 +1618,9 @@ dependencies = [
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.6.4" version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]] [[package]]
name = "memmap2" name = "memmap2"
@@ -1806,9 +1674,9 @@ dependencies = [
[[package]] [[package]]
name = "mockall" name = "mockall"
version = "0.11.4" version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"downcast", "downcast",
@@ -1821,14 +1689,14 @@ dependencies = [
[[package]] [[package]]
name = "mockall_derive" name = "mockall_derive"
version = "0.11.4" version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 1.0.109", "syn 2.0.43",
] ]
[[package]] [[package]]
@@ -1869,12 +1737,6 @@ dependencies = [
"minimal-lexical", "minimal-lexical",
] ]
[[package]]
name = "normalize-line-endings"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
[[package]] [[package]]
name = "ntapi" name = "ntapi"
version = "0.4.1" version = "0.4.1"
@@ -2031,13 +1893,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
dependencies = [ dependencies = [
"fixedbitset", "fixedbitset",
"indexmap 2.1.0", "indexmap",
] ]
[[package]] [[package]]
name = "pgrx" name = "pgrx"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb44171122605250e719ca2ae49afb357bdb2fce4b3c876fcf2225165237328a"
dependencies = [ dependencies = [
"atomic-traits", "atomic-traits",
"bitflags 2.4.1", "bitflags 2.4.1",
@@ -2060,8 +1923,9 @@ dependencies = [
[[package]] [[package]]
name = "pgrx-macros" name = "pgrx-macros"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a18ac8628b7de2f29a93d0abdbdcaee95a0e0ef4b59fd4de99cc117e166e843b"
dependencies = [ dependencies = [
"pgrx-sql-entity-graph", "pgrx-sql-entity-graph",
"proc-macro2", "proc-macro2",
@@ -2071,8 +1935,9 @@ dependencies = [
[[package]] [[package]]
name = "pgrx-pg-config" name = "pgrx-pg-config"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acd45ac6eb1142c5690df63c4e0bdfb74f27c9f93a7af84f064dc2c0a2c2d6f7"
dependencies = [ dependencies = [
"cargo_toml", "cargo_toml",
"dirs", "dirs",
@@ -2088,8 +1953,9 @@ dependencies = [
[[package]] [[package]]
name = "pgrx-pg-sys" name = "pgrx-pg-sys"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81c6207939582934fc26fceb651cb5338e363c06ddc6b2d50ca71867f7c70ffe"
dependencies = [ dependencies = [
"bindgen", "bindgen",
"clang-sys", "clang-sys",
@@ -2111,8 +1977,9 @@ dependencies = [
[[package]] [[package]]
name = "pgrx-sql-entity-graph" name = "pgrx-sql-entity-graph"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a50083de83b1fac2484e8f2c2a7da5fed0193904e2578fa6c4ce02262c455c2b"
dependencies = [ dependencies = [
"convert_case", "convert_case",
"eyre", "eyre",
@@ -2125,8 +1992,9 @@ dependencies = [
[[package]] [[package]]
name = "pgrx-tests" name = "pgrx-tests"
version = "0.11.0" version = "0.11.2"
source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba0115cd80d9e3ca1d5d2a8ab8b7320d6ed614a53d025b86152696a8b3caa75"
dependencies = [ dependencies = [
"clap-cargo", "clap-cargo",
"eyre", "eyre",
@@ -2295,12 +2163,6 @@ dependencies = [
"postgres-protocol", "postgres-protocol",
] ]
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.17" version = "0.2.17"
@@ -2315,16 +2177,13 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]] [[package]]
name = "predicates" name = "predicates"
version = "2.1.5" version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0"
dependencies = [ dependencies = [
"difflib", "anstyle",
"float-cmp", "itertools 0.11.0",
"itertools",
"normalize-line-endings",
"predicates-core", "predicates-core",
"regex",
] ]
[[package]] [[package]]
@@ -2788,35 +2647,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "serde_with"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
dependencies = [
"base64",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.1.0",
"serde",
"serde_json",
"serde_with_macros",
"time",
]
[[package]]
name = "serde_with_macros"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.43",
]
[[package]] [[package]]
name = "service" name = "service"
version = "0.0.0" version = "0.0.0"
@@ -2844,8 +2674,6 @@ dependencies = [
"rustix 0.38.28", "rustix 0.38.28",
"serde", "serde",
"serde_json", "serde_json",
"serde_with",
"tempfile",
"thiserror", "thiserror",
"ulock-sys", "ulock-sys",
"uuid", "uuid",
@@ -2990,12 +2818,6 @@ dependencies = [
"unicode-normalization", "unicode-normalization",
] ]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.5.0" version = "2.5.0"
@@ -3110,35 +2932,6 @@ dependencies = [
"syn 2.0.43", "syn 2.0.43",
] ]
[[package]]
name = "time"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e"
dependencies = [
"deranged",
"itoa",
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f"
dependencies = [
"time-core",
]
[[package]] [[package]]
name = "tiny-keccak" name = "tiny-keccak"
version = "2.0.2" version = "2.0.2"
@@ -3259,7 +3052,7 @@ version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
dependencies = [ dependencies = [
"indexmap 2.1.0", "indexmap",
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
@@ -3490,6 +3283,7 @@ name = "vectors"
version = "0.0.0" version = "0.0.0"
dependencies = [ dependencies = [
"bincode", "bincode",
"bytemuck",
"byteorder", "byteorder",
"detect", "detect",
"env_logger", "env_logger",
@@ -3681,15 +3475,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.51.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
dependencies = [
"windows-targets 0.48.5",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.48.0" version = "0.48.0"

14
Cargo.toml
View File

@@ -27,19 +27,19 @@ byteorder.workspace = true
bincode.workspace = true bincode.workspace = true
half.workspace = true half.workspace = true
num-traits.workspace = true num-traits.workspace = true
rand.workspace = true
bytemuck.workspace = true
service = { path = "crates/service" } service = { path = "crates/service" }
detect = { path = "crates/detect" } detect = { path = "crates/detect" }
pgrx = { git = "https://github.com/tensorchord/pgrx.git", rev = "7c30e2023876c1efce613756f5ec81f3ab05696b", default-features = false, features = [ pgrx = { version = "0.11.2", default-features = false, features = [] }
] }
openai_api_rust = { git = "https://github.com/tensorchord/openai-api.git", rev = "228d54b6002e98257b3c81501a054942342f585f" } openai_api_rust = { git = "https://github.com/tensorchord/openai-api.git", rev = "228d54b6002e98257b3c81501a054942342f585f" }
env_logger = "0.10.0" env_logger = "0.10.0"
toml = "0.8.8" toml = "0.8.8"
rand = "0.8.5"
[dev-dependencies] [dev-dependencies]
pgrx-tests = { git = "https://github.com/tensorchord/pgrx.git", rev = "7c30e2023876c1efce613756f5ec81f3ab05696b" } pgrx-tests = "0.11.2"
httpmock = "0.6" httpmock = "0.6"
mockall = "0.11.4" mockall = "0.12"
[lints] [lints]
clippy.too_many_arguments = "allow" clippy.too_many_arguments = "allow"
@@ -62,7 +62,8 @@ serde = "~1.0"
serde_json = "1" serde_json = "1"
thiserror = "~1.0" thiserror = "~1.0"
bincode = "~1.3" bincode = "~1.3"
byteorder = "~1.4" byteorder = "~1.5"
bytemuck = { version = "~1.14", features = ["extern_crate_alloc"] }
half = { version = "~2.3", features = [ half = { version = "~2.3", features = [
"bytemuck", "bytemuck",
"num-traits", "num-traits",
@@ -72,6 +73,7 @@ half = { version = "~2.3", features = [
num-traits = "~0.2" num-traits = "~0.2"
validator = { version = "~0.16", features = ["derive"] } validator = { version = "~0.16", features = ["derive"] }
rustix = { version = "~0.38", features = ["fs", "net", "mm"] } rustix = { version = "~0.38", features = ["fs", "net", "mm"] }
rand = "~0.8"
[profile.dev] [profile.dev]
panic = "unwind" panic = "unwind"

6
crates/service/Cargo.toml
View File

@@ -15,22 +15,20 @@ byteorder.workspace = true
bincode.workspace = true bincode.workspace = true
half.workspace = true half.workspace = true
num-traits.workspace = true num-traits.workspace = true
rand.workspace = true
bytemuck.workspace = true
c = { path = "../c" } c = { path = "../c" }
detect = { path = "../detect" } detect = { path = "../detect" }
rand = "0.8.5"
crc32fast = "1.3.2" crc32fast = "1.3.2"
crossbeam = "0.8.2" crossbeam = "0.8.2"
dashmap = "5.4.0" dashmap = "5.4.0"
parking_lot = "0.12.1" parking_lot = "0.12.1"
memoffset = "0.9.0" memoffset = "0.9.0"
tempfile = "3.6.0"
arrayvec = { version = "0.7.3", features = ["serde"] } arrayvec = { version = "0.7.3", features = ["serde"] }
memmap2 = "0.9.0" memmap2 = "0.9.0"
rayon = "1.6.1" rayon = "1.6.1"
uuid = { version = "1.6.1", features = ["serde"] } uuid = { version = "1.6.1", features = ["serde"] }
arc-swap = "1.6.0" arc-swap = "1.6.0"
bytemuck = { version = "1.14.0", features = ["extern_crate_alloc"] }
serde_with = "3.4.0"
multiversion = "0.7.3" multiversion = "0.7.3"
[target.'cfg(target_os = "macos")'.dependencies] [target.'cfg(target_os = "macos")'.dependencies]

50
crates/service/src/index/mod.rs
View File

@@ -64,7 +64,7 @@ pub struct IndexOptions {
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub struct SegmentSizeInfo { pub struct SegmentStat {
pub id: Uuid, pub id: Uuid,
#[serde(rename = "type")] #[serde(rename = "type")]
pub typ: String, pub typ: String,
@@ -73,13 +73,13 @@ pub struct SegmentSizeInfo {
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct IndexStat { pub enum IndexStat {
pub indexing: bool, Normal {
pub sealed: Vec<u32>, indexing: bool,
pub growing: Vec<u32>, segments: Vec<SegmentStat>,
pub write: u32, options: IndexOptions,
pub options: IndexOptions, },
pub sizes: Vec<SegmentSizeInfo>, Upgrade,
} }
pub struct Index<S: G> { pub struct Index<S: G> {
@@ -97,6 +97,11 @@ impl<S: G> Index<S> {
pub fn create(path: PathBuf, options: IndexOptions) -> Arc<Self> { pub fn create(path: PathBuf, options: IndexOptions) -> Arc<Self> {
assert!(options.validate().is_ok()); assert!(options.validate().is_ok());
std::fs::create_dir(&path).unwrap(); std::fs::create_dir(&path).unwrap();
std::fs::write(
path.join("options"),
serde_json::to_string::<IndexOptions>(&options).unwrap(),
)
.unwrap();
std::fs::create_dir(path.join("segments")).unwrap(); std::fs::create_dir(path.join("segments")).unwrap();
let startup = FileAtomic::create( let startup = FileAtomic::create(
path.join("startup"), path.join("startup"),
@@ -132,7 +137,10 @@ impl<S: G> Index<S> {
OptimizerSealing::new(index.clone()).spawn(); OptimizerSealing::new(index.clone()).spawn();
index index
} }
pub fn open(path: PathBuf, options: IndexOptions) -> Arc<Self> { pub fn open(path: PathBuf) -> Arc<Self> {
let options =
serde_json::from_slice::<IndexOptions>(&std::fs::read(path.join("options")).unwrap())
.unwrap();
let tracker = Arc::new(IndexTracker { path: path.clone() }); let tracker = Arc::new(IndexTracker { path: path.clone() });
let startup = FileAtomic::<IndexStartup>::open(path.join("startup")); let startup = FileAtomic::<IndexStartup>::open(path.join("startup"));
clean( clean(
@@ -244,18 +252,22 @@ impl<S: G> Index<S> {
} }
pub fn stat(&self) -> IndexStat { pub fn stat(&self) -> IndexStat {
let view = self.view(); let view = self.view();
IndexStat { IndexStat::Normal {
indexing: self.instant_index.load() < self.instant_write.load(), indexing: self.instant_index.load() < self.instant_write.load(),
sealed: view.sealed.values().map(|x| x.len()).collect(),
growing: view.growing.values().map(|x| x.len()).collect(),
write: view.write.as_ref().map(|(_, x)| x.len()).unwrap_or(0),
options: self.options().clone(), options: self.options().clone(),
sizes: view segments: {
.sealed let mut segments = Vec::new();
.values() for sealed in view.sealed.values() {
.map(|x| x.size()) segments.push(sealed.stat_sealed());
.chain(view.growing.values().map(|x| x.size())) }
.collect(), for growing in view.growing.values() {
segments.push(growing.stat_growing());
}
if let Some(write) = view.write.as_ref().map(|(_, x)| x) {
segments.push(write.stat_write());
}
segments
},
} }
} }
} }

20
crates/service/src/index/segments/growing.rs
View File

@@ -1,9 +1,7 @@
#![allow(clippy::all)] // Clippy bug.
use super::SegmentTracker; use super::SegmentTracker;
use crate::index::IndexOptions; use crate::index::IndexOptions;
use crate::index::IndexTracker; use crate::index::IndexTracker;
use crate::index::SegmentSizeInfo; use crate::index::SegmentStat;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::dir_ops::sync_dir; use crate::utils::dir_ops::sync_dir;
use crate::utils::file_wal::FileWal; use crate::utils::file_wal::FileWal;
@@ -12,8 +10,7 @@ use serde::{Deserialize, Serialize};
use std::cell::UnsafeCell; use std::cell::UnsafeCell;
use std::mem::MaybeUninit; use std::mem::MaybeUninit;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::AtomicUsize; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::Ordering;
use std::sync::Arc; use std::sync::Arc;
use thiserror::Error; use thiserror::Error;
use uuid::Uuid; use uuid::Uuid;
@@ -44,6 +41,7 @@ impl<S: G> GrowingSegment<S> {
sync_dir(&path); sync_dir(&path);
Arc::new(Self { Arc::new(Self {
uuid, uuid,
#[allow(clippy::uninit_vec)]
vec: unsafe { vec: unsafe {
let mut vec = Vec::with_capacity(capacity as usize); let mut vec = Vec::with_capacity(capacity as usize);
vec.set_len(capacity as usize); vec.set_len(capacity as usize);
@@ -141,14 +139,22 @@ impl<S: G> GrowingSegment<S> {
pub fn len(&self) -> u32 { pub fn len(&self) -> u32 {
self.len.load(Ordering::Acquire) as u32 self.len.load(Ordering::Acquire) as u32
} }
pub fn size(&self) -> SegmentSizeInfo { pub fn stat_growing(&self) -> SegmentStat {
SegmentSizeInfo { SegmentStat {
id: self.uuid, id: self.uuid,
typ: "growing".to_string(), typ: "growing".to_string(),
length: self.len() as usize, length: self.len() as usize,
size: (self.len() as u64) * (std::mem::size_of::<Log<S>>() as u64), size: (self.len() as u64) * (std::mem::size_of::<Log<S>>() as u64),
} }
} }
pub fn stat_write(&self) -> SegmentStat {
SegmentStat {
id: self.uuid,
typ: "write".to_string(),
length: self.len() as usize,
size: (self.len() as u64) * (std::mem::size_of::<Log<S>>() as u64),
}
}
pub fn vector(&self, i: u32) -> &[S::Scalar] { pub fn vector(&self, i: u32) -> &[S::Scalar] {
let i = i as usize; let i = i as usize;
if i >= self.len.load(Ordering::Acquire) { if i >= self.len.load(Ordering::Acquire) {

17
crates/service/src/index/segments/sealed.rs
View File

@@ -1,7 +1,7 @@
use super::growing::GrowingSegment; use super::growing::GrowingSegment;
use super::SegmentTracker; use super::SegmentTracker;
use crate::index::indexing::{DynamicIndexIter, DynamicIndexing}; use crate::index::indexing::{DynamicIndexIter, DynamicIndexing};
use crate::index::{IndexOptions, IndexTracker, SegmentSizeInfo}; use crate::index::{IndexOptions, IndexTracker, SegmentStat};
use crate::prelude::*; use crate::prelude::*;
use crate::utils::dir_ops::{dir_size, sync_dir}; use crate::utils::dir_ops::{dir_size, sync_dir};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -58,22 +58,15 @@ impl<S: G> SealedSegment<S> {
pub fn len(&self) -> u32 { pub fn len(&self) -> u32 {
self.indexing.len() self.indexing.len()
} }
pub fn size(&self) -> SegmentSizeInfo { pub fn stat_sealed(&self) -> SegmentStat {
let mut info = SegmentSizeInfo { let path = self._tracker.path.join("indexing");
SegmentStat {
id: self.uuid, id: self.uuid,
typ: "sealed".to_string(), typ: "sealed".to_string(),
length: self.len() as usize, length: self.len() as usize,
size: 0, size: dir_size(&path).unwrap(),
};
let path = self._tracker.path.join("indexing");
match dir_size(&path) {
Ok(size) => info.size = size as u64,
Err(e) => {
panic!("Failed to get size of {:?}: {}", path, e);
} }
} }
info
}
pub fn vector(&self, i: u32) -> &[S::Scalar] { pub fn vector(&self, i: u32) -> &[S::Scalar] {
self.indexing.vector(i) self.indexing.vector(i)
} }

46
crates/service/src/instance/metadata.rs Normal file
View File

@@ -0,0 +1,46 @@
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::path::Path;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum MetadataError {
#[error("Invalid version.")]
InvalidVersion,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Metadata {
#[serde(default)]
pub version: Option<u64>,
#[serde(default)]
pub soft_version: Option<u64>,
}
impl Metadata {
const VERSION: u64 = 2;
const SOFT_VERSION: u64 = 1;
}
impl Metadata {
pub fn write(path: impl AsRef<Path>) {
let metadata = Metadata {
version: Some(Self::VERSION),
soft_version: Some(Self::SOFT_VERSION),
};
let contents = serde_json::to_string(&metadata).unwrap();
std::fs::write(path, contents).unwrap();
}
pub fn read(path: impl AsRef<Path>) -> Result<(), Box<dyn Error>> {
use MetadataError::*;
let contents = std::fs::read_to_string(path)?;
let metadata = serde_json::from_str::<Metadata>(&contents)?;
if Self::VERSION != metadata.version.ok_or(InvalidVersion)? {
return Err(Box::new(InvalidVersion));
}
if Self::SOFT_VERSION < metadata.soft_version.ok_or(InvalidVersion)? {
return Err(Box::new(InvalidVersion));
}
Ok(())
}
}

137
crates/service/src/worker/instance.rs → crates/service/src/instance/mod.rs
View File

@@ -1,3 +1,5 @@
pub mod metadata;
use crate::index::segments::SearchGucs; use crate::index::segments::SearchGucs;
use crate::index::Index; use crate::index::Index;
use crate::index::IndexOptions; use crate::index::IndexOptions;
@@ -16,67 +18,120 @@ pub enum Instance {
F16Cos(Arc<Index<F16Cos>>), F16Cos(Arc<Index<F16Cos>>),
F16Dot(Arc<Index<F16Dot>>), F16Dot(Arc<Index<F16Dot>>),
F16L2(Arc<Index<F16L2>>), F16L2(Arc<Index<F16L2>>),
Upgrade,
} }
impl Instance { impl Instance {
pub fn create(path: PathBuf, options: IndexOptions) -> Self { pub fn create(path: PathBuf, options: IndexOptions) -> Self {
match (options.vector.d, options.vector.k) { match (options.vector.d, options.vector.k) {
(Distance::Cos, Kind::F32) => Self::F32Cos(Index::create(path, options)), (Distance::Cos, Kind::F32) => {
(Distance::Dot, Kind::F32) => Self::F32Dot(Index::create(path, options)), let index = Index::create(path.clone(), options);
(Distance::L2, Kind::F32) => Self::F32L2(Index::create(path, options)), self::metadata::Metadata::write(path.join("metadata"));
(Distance::Cos, Kind::F16) => Self::F16Cos(Index::create(path, options)), Self::F32Cos(index)
(Distance::Dot, Kind::F16) => Self::F16Dot(Index::create(path, options)), }
(Distance::L2, Kind::F16) => Self::F16L2(Index::create(path, options)), (Distance::Dot, Kind::F32) => {
let index = Index::create(path.clone(), options);
self::metadata::Metadata::write(path.join("metadata"));
Self::F32Dot(index)
}
(Distance::L2, Kind::F32) => {
let index = Index::create(path.clone(), options);
self::metadata::Metadata::write(path.join("metadata"));
Self::F32L2(index)
}
(Distance::Cos, Kind::F16) => {
let index = Index::create(path.clone(), options);
self::metadata::Metadata::write(path.join("metadata"));
Self::F16Cos(index)
}
(Distance::Dot, Kind::F16) => {
let index = Index::create(path.clone(), options);
self::metadata::Metadata::write(path.join("metadata"));
Self::F16Dot(index)
}
(Distance::L2, Kind::F16) => {
let index = Index::create(path.clone(), options);
self::metadata::Metadata::write(path.join("metadata"));
Self::F16L2(index)
} }
} }
pub fn open(path: PathBuf, options: IndexOptions) -> Self { }
pub fn open(path: PathBuf) -> Self {
if self::metadata::Metadata::read(path.join("metadata")).is_err() {
return Self::Upgrade;
}
let options =
serde_json::from_slice::<IndexOptions>(&std::fs::read(path.join("options")).unwrap())
.unwrap();
match (options.vector.d, options.vector.k) { match (options.vector.d, options.vector.k) {
(Distance::Cos, Kind::F32) => Self::F32Cos(Index::open(path, options)), (Distance::Cos, Kind::F32) => Self::F32Cos(Index::open(path)),
(Distance::Dot, Kind::F32) => Self::F32Dot(Index::open(path, options)), (Distance::Dot, Kind::F32) => Self::F32Dot(Index::open(path)),
(Distance::L2, Kind::F32) => Self::F32L2(Index::open(path, options)), (Distance::L2, Kind::F32) => Self::F32L2(Index::open(path)),
(Distance::Cos, Kind::F16) => Self::F16Cos(Index::open(path, options)), (Distance::Cos, Kind::F16) => Self::F16Cos(Index::open(path)),
(Distance::Dot, Kind::F16) => Self::F16Dot(Index::open(path, options)), (Distance::Dot, Kind::F16) => Self::F16Dot(Index::open(path)),
(Distance::L2, Kind::F16) => Self::F16L2(Index::open(path, options)), (Distance::L2, Kind::F16) => Self::F16L2(Index::open(path)),
} }
} }
pub fn options(&self) -> &IndexOptions { pub fn options(&self) -> Result<&IndexOptions, FriendlyError> {
match self { match self {
Instance::F32Cos(x) => x.options(), Instance::F32Cos(x) => Ok(x.options()),
Instance::F32Dot(x) => x.options(), Instance::F32Dot(x) => Ok(x.options()),
Instance::F32L2(x) => x.options(), Instance::F32L2(x) => Ok(x.options()),
Instance::F16Cos(x) => x.options(), Instance::F16Cos(x) => Ok(x.options()),
Instance::F16Dot(x) => x.options(), Instance::F16Dot(x) => Ok(x.options()),
Instance::F16L2(x) => x.options(), Instance::F16L2(x) => Ok(x.options()),
Instance::Upgrade => Err(FriendlyError::Upgrade2),
} }
} }
pub fn refresh(&self) { pub fn refresh(&self) -> Result<(), FriendlyError> {
match self { match self {
Instance::F32Cos(x) => x.refresh(), Instance::F32Cos(x) => {
Instance::F32Dot(x) => x.refresh(), x.refresh();
Instance::F32L2(x) => x.refresh(), Ok(())
Instance::F16Cos(x) => x.refresh(), }
Instance::F16Dot(x) => x.refresh(), Instance::F32Dot(x) => {
Instance::F16L2(x) => x.refresh(), x.refresh();
Ok(())
}
Instance::F32L2(x) => {
x.refresh();
Ok(())
}
Instance::F16Cos(x) => {
x.refresh();
Ok(())
}
Instance::F16Dot(x) => {
x.refresh();
Ok(())
}
Instance::F16L2(x) => {
x.refresh();
Ok(())
}
Instance::Upgrade => Err(FriendlyError::Upgrade2),
} }
} }
pub fn view(&self) -> InstanceView { pub fn view(&self) -> Result<InstanceView, FriendlyError> {
match self { match self {
Instance::F32Cos(x) => InstanceView::F32Cos(x.view()), Instance::F32Cos(x) => Ok(InstanceView::F32Cos(x.view())),
Instance::F32Dot(x) => InstanceView::F32Dot(x.view()), Instance::F32Dot(x) => Ok(InstanceView::F32Dot(x.view())),
Instance::F32L2(x) => InstanceView::F32L2(x.view()), Instance::F32L2(x) => Ok(InstanceView::F32L2(x.view())),
Instance::F16Cos(x) => InstanceView::F16Cos(x.view()), Instance::F16Cos(x) => Ok(InstanceView::F16Cos(x.view())),
Instance::F16Dot(x) => InstanceView::F16Dot(x.view()), Instance::F16Dot(x) => Ok(InstanceView::F16Dot(x.view())),
Instance::F16L2(x) => InstanceView::F16L2(x.view()), Instance::F16L2(x) => Ok(InstanceView::F16L2(x.view())),
Instance::Upgrade => Err(FriendlyError::Upgrade2),
} }
} }
pub fn stat(&self) -> IndexStat { pub fn stat(&self) -> Result<IndexStat, FriendlyError> {
match self { match self {
Instance::F32Cos(x) => x.stat(), Instance::F32Cos(x) => Ok(x.stat()),
Instance::F32Dot(x) => x.stat(), Instance::F32Dot(x) => Ok(x.stat()),
Instance::F32L2(x) => x.stat(), Instance::F32L2(x) => Ok(x.stat()),
Instance::F16Cos(x) => x.stat(), Instance::F16Cos(x) => Ok(x.stat()),
Instance::F16Dot(x) => x.stat(), Instance::F16Dot(x) => Ok(x.stat()),
Instance::F16L2(x) => x.stat(), Instance::F16L2(x) => Ok(x.stat()),
Instance::Upgrade => Ok(IndexStat::Upgrade),
} }
} }
} }

1
crates/service/src/lib.rs
View File

@@ -3,6 +3,7 @@
pub mod algorithms; pub mod algorithms;
pub mod index; pub mod index;
pub mod instance;
pub mod prelude; pub mod prelude;
pub mod worker; pub mod worker;

9
crates/service/src/prelude/error.rs
View File

@@ -69,10 +69,15 @@ Please check the full PostgreSQL log to get more information.\
")] ")]
Ipc, Ipc,
#[error("\ #[error("\
The extension is upgraded. However, the index files is outdated. The extension is upgraded so all index files are outdated.
ADVICE: Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\ ADVICE: Delete all index files. Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\
")] ")]
Upgrade, Upgrade,
#[error("\
The extension is upgraded so this index is outdated.
ADVICE: Rebuild the index. Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\
")]
Upgrade2,
} }
pub trait FriendlyErrorLike: Sized { pub trait FriendlyErrorLike: Sized {

2
crates/service/src/prelude/mod.rs
View File

@@ -11,6 +11,6 @@ pub use self::scalar::{F16, F32};
pub use self::filter::{Filter, Payload}; pub use self::filter::{Filter, Payload};
pub use self::heap::{Heap, HeapElement}; pub use self::heap::{Heap, HeapElement};
pub use self::sys::{Id, Pointer}; pub use self::sys::{Handle, Pointer};
pub use num_traits::{Float, Zero}; pub use num_traits::{Float, Zero};

10
crates/service/src/prelude/sys.rs
View File

@@ -2,27 +2,27 @@ use serde::{Deserialize, Serialize};
use std::{fmt::Display, num::ParseIntError, str::FromStr}; use std::{fmt::Display, num::ParseIntError, str::FromStr};
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct Id { pub struct Handle {
pub newtype: u32, pub newtype: u32,
} }
impl Id { impl Handle {
pub fn as_u32(self) -> u32 { pub fn as_u32(self) -> u32 {
self.newtype self.newtype
} }
} }
impl Display for Id { impl Display for Handle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_u32()) write!(f, "{}", self.as_u32())
} }
} }
impl FromStr for Id { impl FromStr for Handle {
type Err = ParseIntError; type Err = ParseIntError;
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Id { Ok(Handle {
newtype: u32::from_str(s)?, newtype: u32::from_str(s)?,
}) })
} }

4
crates/service/src/utils/dir_ops.rs
View File

@@ -7,7 +7,7 @@ pub fn sync_dir(path: impl AsRef<Path>) {
file.sync_all().expect("Failed to sync dir."); file.sync_all().expect("Failed to sync dir.");
} }
pub fn dir_size(dir: &Path) -> io::Result<usize> { pub fn dir_size(dir: &Path) -> io::Result<u64> {
let mut size = 0; let mut size = 0;
if dir.is_dir() { if dir.is_dir() {
for entry in read_dir(dir)? { for entry in read_dir(dir)? {
@@ -21,7 +21,7 @@ pub fn dir_size(dir: &Path) -> io::Result<usize> {
if path.is_dir() { if path.is_dir() {
size += dir_size(&path)?; size += dir_size(&path)?;
} else { } else {
size += entry.metadata()?.len() as usize; size += entry.metadata()?.len();
} }
} }
} }

4
crates/service/src/worker/metadata.rs
View File

@@ -18,7 +18,7 @@ pub struct Metadata {
} }
impl Metadata { impl Metadata {
const VERSION: u64 = 1; const VERSION: u64 = 2;
const SOFT_VERSION: u64 = 1; const SOFT_VERSION: u64 = 1;
} }
@@ -38,7 +38,7 @@ impl Metadata {
if Self::VERSION != metadata.version.ok_or(InvalidVersion)? { if Self::VERSION != metadata.version.ok_or(InvalidVersion)? {
return Err(Box::new(InvalidVersion)); return Err(Box::new(InvalidVersion));
} }
if Self::SOFT_VERSION <= metadata.soft_version.ok_or(InvalidVersion)? { if Self::SOFT_VERSION < metadata.soft_version.ok_or(InvalidVersion)? {
return Err(Box::new(InvalidVersion)); return Err(Box::new(InvalidVersion));
} }
Ok(()) Ok(())

98
crates/service/src/worker/mod.rs
View File

@@ -1,11 +1,10 @@
pub mod instance;
pub mod metadata; pub mod metadata;
use self::instance::Instance;
use crate::index::segments::SearchGucs; use crate::index::segments::SearchGucs;
use crate::index::IndexOptions; use crate::index::IndexOptions;
use crate::index::IndexStat; use crate::index::IndexStat;
use crate::index::OutdatedError; use crate::index::OutdatedError;
use crate::instance::Instance;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::clean::clean; use crate::utils::clean::clean;
use crate::utils::dir_ops::sync_dir; use crate::utils::dir_ops::sync_dir;
@@ -13,8 +12,7 @@ use crate::utils::file_atomic::FileAtomic;
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use parking_lot::Mutex; use parking_lot::Mutex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_with::DisplayFromStr; use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
@@ -49,12 +47,12 @@ impl Worker {
let startup = FileAtomic::<WorkerStartup>::open(path.join("startup")); let startup = FileAtomic::<WorkerStartup>::open(path.join("startup"));
clean( clean(
path.join("indexes"), path.join("indexes"),
startup.get().indexes.keys().map(|s| s.to_string()), startup.get().indexes.iter().map(|s| s.to_string()),
); );
let mut indexes = HashMap::new(); let mut indexes = HashMap::new();
for (&id, options) in startup.get().indexes.iter() { for &id in startup.get().indexes.iter() {
let path = path.join("indexes").join(id.to_string()); let path = path.join("indexes").join(id.to_string());
let index = Instance::open(path, options.clone()); let index = Instance::open(path);
indexes.insert(id, index); indexes.insert(id, index);
} }
let view = Arc::new(WorkerView { let view = Arc::new(WorkerView {
@@ -67,17 +65,17 @@ impl Worker {
view: ArcSwap::new(view), view: ArcSwap::new(view),
}) })
} }
pub fn call_create(&self, id: Id, options: IndexOptions) { pub fn call_create(&self, handle: Handle, options: IndexOptions) {
let mut protect = self.protect.lock(); let mut protect = self.protect.lock();
let index = Instance::create(self.path.join("indexes").join(id.to_string()), options); let index = Instance::create(self.path.join("indexes").join(handle.to_string()), options);
if protect.indexes.insert(id, index).is_some() { if protect.indexes.insert(handle, index).is_some() {
panic!("index {} already exists", id) panic!("index {} already exists", handle)
} }
protect.maintain(&self.view); protect.maintain(&self.view);
} }
pub fn call_search<F>( pub fn call_search<F>(
&self, &self,
id: Id, handle: Handle,
search: (DynamicVector, usize), search: (DynamicVector, usize),
gucs: SearchGucs, gucs: SearchGucs,
filter: F, filter: F,
@@ -86,80 +84,90 @@ impl Worker {
F: FnMut(Pointer) -> bool, F: FnMut(Pointer) -> bool,
{ {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
let view = index.view(); .indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
let view = index.view()?;
view.search(search.1, search.0, gucs, filter) view.search(search.1, search.0, gucs, filter)
} }
pub fn call_insert( pub fn call_insert(
&self, &self,
id: Id, handle: Handle,
insert: (DynamicVector, Pointer), insert: (DynamicVector, Pointer),
) -> Result<(), FriendlyError> { ) -> Result<(), FriendlyError> {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
.indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
loop { loop {
let view = index.view(); let view = index.view()?;
match view.insert(insert.0.clone(), insert.1)? { match view.insert(insert.0.clone(), insert.1)? {
Ok(()) => break Ok(()), Ok(()) => break Ok(()),
Err(OutdatedError(_)) => index.refresh(), Err(OutdatedError(_)) => index.refresh()?,
} }
} }
} }
pub fn call_delete<F>(&self, id: Id, f: F) -> Result<(), FriendlyError> pub fn call_delete<F>(&self, handle: Handle, f: F) -> Result<(), FriendlyError>
where where
F: FnMut(Pointer) -> bool, F: FnMut(Pointer) -> bool,
{ {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
let view = index.view(); .indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
let view = index.view()?;
view.delete(f); view.delete(f);
Ok(()) Ok(())
} }
pub fn call_flush(&self, id: Id) -> Result<(), FriendlyError> { pub fn call_flush(&self, handle: Handle) -> Result<(), FriendlyError> {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
let view = index.view(); .indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
let view = index.view()?;
view.flush(); view.flush();
Ok(()) Ok(())
} }
pub fn call_destory(&self, ids: Vec<Id>) { pub fn call_destory(&self, handle: Handle) {
let mut updated = false;
let mut protect = self.protect.lock(); let mut protect = self.protect.lock();
for id in ids { if protect.indexes.remove(&handle).is_some() {
updated |= protect.indexes.remove(&id).is_some();
}
if updated {
protect.maintain(&self.view); protect.maintain(&self.view);
} }
} }
pub fn call_stat(&self, id: Id) -> Result<IndexStat, FriendlyError> { pub fn call_stat(&self, handle: Handle) -> Result<IndexStat, FriendlyError> {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
Ok(index.stat()) .indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
index.stat()
} }
pub fn get_instance(&self, id: Id) -> Result<Instance, FriendlyError> { pub fn get_instance(&self, handle: Handle) -> Result<Instance, FriendlyError> {
let view = self.view.load_full(); let view = self.view.load_full();
let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; let index = view
.indexes
.get(&handle)
.ok_or(FriendlyError::UnknownIndex)?;
Ok(index.clone()) Ok(index.clone())
} }
} }
struct WorkerView { struct WorkerView {
indexes: HashMap<Id, Instance>, indexes: HashMap<Handle, Instance>,
} }
struct WorkerProtect { struct WorkerProtect {
startup: FileAtomic<WorkerStartup>, startup: FileAtomic<WorkerStartup>,
indexes: HashMap<Id, Instance>, indexes: HashMap<Handle, Instance>,
} }
impl WorkerProtect { impl WorkerProtect {
fn maintain(&mut self, swap: &ArcSwap<WorkerView>) { fn maintain(&mut self, swap: &ArcSwap<WorkerView>) {
let indexes = self let indexes = self.indexes.keys().copied().collect();
.indexes
.iter()
.map(|(&k, v)| (k, v.options().clone()))
.collect();
self.startup.set(WorkerStartup { indexes }); self.startup.set(WorkerStartup { indexes });
swap.swap(Arc::new(WorkerView { swap.swap(Arc::new(WorkerView {
indexes: self.indexes.clone(), indexes: self.indexes.clone(),
@@ -167,17 +175,15 @@ impl WorkerProtect {
} }
} }
#[serde_with::serde_as]
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
struct WorkerStartup { struct WorkerStartup {
#[serde_as(as = "HashMap<DisplayFromStr, _>")] indexes: HashSet<Handle>,
indexes: HashMap<Id, IndexOptions>,
} }
impl WorkerStartup { impl WorkerStartup {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
indexes: HashMap::new(), indexes: HashSet::new(),
} }
} }
} }

17
docs/indexing.md
View File

@@ -109,18 +109,19 @@ Options for table `product`.
We also provide a view `pg_vector_index_info` to monitor the progress of indexing. We also provide a view `pg_vector_index_info` to monitor the progress of indexing.
| Column | Type | Description | | Column | Type | Description |
| ------------ | ------ | --------------------------------------------- | | ------------ | ------ | ----------------------------------------------------------------------------------- |
| tablerelid | oid | The oid of the table. | | tablerelid | oid | The oid of the table. |
| indexrelid | oid | The oid of the index. | | indexrelid | oid | The oid of the index. |
| tablename | name | The name of the table. | | tablename | name | The name of the table. |
| indexname | name | The name of the index. | | indexname | name | The name of the index. |
| idx_indexing | bool | Whether the background thread is indexing. | | idx_status | text | Its value is `NORMAL` or `UPGRADE`. Whether this index is normal or needs upgrade. |
| idx_tuples | int8 | The number of tuples. | | idx_indexing | bool | Not null if `idx_status` is `NORMAL`. Whether the background thread is indexing. |
| idx_sealed | int8[] | The number of tuples in each sealed segment. | | idx_tuples | int8 | Not null if `idx_status` is `NORMAL`. The number of tuples. |
| idx_growing | int8[] | The number of tuples in each growing segment. | | idx_sealed | int8[] | Not null if `idx_status` is `NORMAL`. The number of tuples in each sealed segment. |
| idx_write | int8 | The number of tuples in write buffer. | | idx_growing | int8[] | Not null if `idx_status` is `NORMAL`. The number of tuples in each growing segment. |
| idx_size | int8 | The byte size for all the segments. | | idx_write | int8 | Not null if `idx_status` is `NORMAL`. The number of tuples in write buffer. |
| idx_config | text | The configuration of the index. | | idx_size | int8 | Not null if `idx_status` is `NORMAL`. The byte size for all the segments. |
| idx_config | text | Not null if `idx_status` is `NORMAL`. The configuration of the index. |
## Examples ## Examples

2
docs/installation.md
View File

@@ -77,7 +77,7 @@ cd pgvecto.rs
Install cargo-pgrx. Install cargo-pgrx.
```sh ```sh
cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2)
cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config
``` ```

40
docs/upgrade.md
View File

@@ -1,6 +1,6 @@
# Upgrade # Upgrade
## `The extension is upgraded. However, the index files is outdated.` ## `The extension is upgraded so all index files are outdated.`
You may see this error if you upgrade the extension. On this condition, you should follow these steps: You may see this error if you upgrade the extension. On this condition, you should follow these steps:
@@ -49,3 +49,41 @@ REINDEX INDEX t_val_idx1;
REINDEX INDEX t_val_idx2; REINDEX INDEX t_val_idx2;
REINDEX INDEX t_val_idx3; REINDEX INDEX t_val_idx3;
``` ```
## `The extension is upgraded so this index is outdated.`
You may see this error if you upgrade the extension. On this condition, you should follow these steps:
* Reindex.
You can list all indexes that needed to be reindexed with this command:
```sql
SELECT
I.oid AS indexrelid,
I.relname AS indexname
FROM pg_index X
JOIN pg_class I ON I.oid = X.indexrelid
JOIN pg_am A ON A.oid = I.relam
WHERE A.amname = 'vectors';
```
If you get the result like this:
```
indexrelid | indexname
------------+------------
17988 | t_val_idx
17989 | t_val_idx1
17990 | t_val_idx2
17991 | t_val_idx3
```
You will reindex them with this SQL:
```sql
REINDEX INDEX t_val_idx;
REINDEX INDEX t_val_idx1;
REINDEX INDEX t_val_idx2;
REINDEX INDEX t_val_idx3;
```

2
scripts/ci_setup.sh
View File

@@ -41,5 +41,5 @@ sudo chmod -R 777 `pg_config --sharedir`/extension
curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
cargo binstall sqllogictest-bin -y --force cargo binstall sqllogictest-bin -y --force
cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) --debug cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2) --debug
cargo pgrx init --pg$VERSION=$(which pg_config) cargo pgrx init --pg$VERSION=$(which pg_config)

2
scripts/envd.sh
View File

@@ -9,5 +9,5 @@ sudo chmod 777 /usr/share/postgresql/15/extension/
sudo chmod 777 /usr/lib/postgresql/15/lib/ sudo chmod 777 /usr/lib/postgresql/15/lib/
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2)
cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config

38
src/bgworker/normal.rs
View File

@@ -60,58 +60,62 @@ fn session(worker: Arc<Worker>, mut handler: RpcHandler) -> Result<(), IpcError>
use crate::ipc::server::RpcHandle; use crate::ipc::server::RpcHandle;
loop { loop {
match handler.handle()? { match handler.handle()? {
RpcHandle::Create { id, options, x } => { RpcHandle::Create { handle, options, x } => {
worker.call_create(id, options); worker.call_create(handle, options);
handler = x.leave()?; handler = x.leave()?;
} }
RpcHandle::Insert { id, insert, x } => match worker.call_insert(id, insert) { RpcHandle::Insert { handle, insert, x } => match worker.call_insert(handle, insert) {
Ok(()) => handler = x.leave()?, Ok(()) => handler = x.leave()?,
Err(res) => x.reset(res)?, Err(res) => x.reset(res)?,
}, },
RpcHandle::Delete { id, mut x } => match worker.call_delete(id, |p| x.next(p).unwrap()) RpcHandle::Delete { handle, mut x } => {
{ match worker.call_delete(handle, |p| x.next(p).unwrap()) {
Ok(()) => handler = x.leave()?, Ok(()) => handler = x.leave()?,
Err(res) => x.reset(res)?, Err(res) => x.reset(res)?,
}, }
}
RpcHandle::Search { RpcHandle::Search {
id, handle,
search, search,
prefilter: true, prefilter: true,
gucs, gucs,
mut x, mut x,
} => match worker.call_search(id, search, gucs, |p| x.check(p).unwrap()) { } => match worker.call_search(handle, search, gucs, |p| x.check(p).unwrap()) {
Ok(res) => handler = x.leave(res)?, Ok(res) => handler = x.leave(res)?,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,
}, },
RpcHandle::Search { RpcHandle::Search {
id, handle,
search, search,
prefilter: false, prefilter: false,
gucs, gucs,
x, x,
} => match worker.call_search(id, search, gucs, |_| true) { } => match worker.call_search(handle, search, gucs, |_| true) {
Ok(res) => handler = x.leave(res)?, Ok(res) => handler = x.leave(res)?,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,
}, },
RpcHandle::Flush { id, x } => match worker.call_flush(id) { RpcHandle::Flush { handle, x } => match worker.call_flush(handle) {
Ok(()) => handler = x.leave()?, Ok(()) => handler = x.leave()?,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,
}, },
RpcHandle::Destory { ids, x } => { RpcHandle::Destory { handle, x } => {
worker.call_destory(ids); worker.call_destory(handle);
handler = x.leave()?; handler = x.leave()?;
} }
RpcHandle::Stat { id, x } => match worker.call_stat(id) { RpcHandle::Stat { handle, x } => match worker.call_stat(handle) {
Ok(res) => handler = x.leave(res)?, Ok(res) => handler = x.leave(res)?,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,
}, },
RpcHandle::Vbase { id, vbase, x } => { RpcHandle::Vbase { handle, vbase, x } => {
use crate::ipc::server::VbaseHandle::*; use crate::ipc::server::VbaseHandle::*;
let instance = match worker.get_instance(id) { let instance = match worker.get_instance(handle) {
Ok(x) => x,
Err(e) => x.reset(e)?,
};
let view = match instance.view() {
Ok(x) => x, Ok(x) => x,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,
}; };
let view = instance.view();
let mut it = match view.vbase(vbase.0, vbase.1) { let mut it = match view.vbase(vbase.0, vbase.1) {
Ok(x) => x, Ok(x) => x,
Err(e) => x.reset(e)?, Err(e) => x.reset(e)?,

6
src/index/am.rs
View File

@@ -205,7 +205,7 @@ pub unsafe extern "C" fn aminsert(
_index_info: *mut pgrx::pg_sys::IndexInfo, _index_info: *mut pgrx::pg_sys::IndexInfo,
) -> bool { ) -> bool {
let oid = (*index_relation).rd_node.relNode; let oid = (*index_relation).rd_node.relNode;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
let vector = from_datum(*values.add(0)); let vector = from_datum(*values.add(0));
am_update::update_insert(id, vector, *heap_tid); am_update::update_insert(id, vector, *heap_tid);
true true
@@ -227,7 +227,7 @@ pub unsafe extern "C" fn aminsert(
let oid = (*index_relation).rd_node.relNode; let oid = (*index_relation).rd_node.relNode;
#[cfg(feature = "pg16")] #[cfg(feature = "pg16")]
let oid = (*index_relation).rd_locator.relNumber; let oid = (*index_relation).rd_locator.relNumber;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
let vector = from_datum(*values.add(0)); let vector = from_datum(*values.add(0));
am_update::update_insert(id, vector, *heap_tid); am_update::update_insert(id, vector, *heap_tid);
true true
@@ -284,7 +284,7 @@ pub unsafe extern "C" fn ambulkdelete(
let oid = (*(*info).index).rd_node.relNode; let oid = (*(*info).index).rd_node.relNode;
#[cfg(feature = "pg16")] #[cfg(feature = "pg16")]
let oid = (*(*info).index).rd_locator.relNumber; let oid = (*(*info).index).rd_locator.relNumber;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
if let Some(callback) = callback { if let Some(callback) = callback {
am_update::update_delete(id, |pointer| { am_update::update_delete(id, |pointer| {
callback( callback(

6
src/index/am_build.rs
View File

@@ -21,7 +21,7 @@ pub unsafe fn build(
let oid = (*index).rd_node.relNode; let oid = (*index).rd_node.relNode;
#[cfg(feature = "pg16")] #[cfg(feature = "pg16")]
let oid = (*index).rd_locator.relNumber; let oid = (*index).rd_locator.relNumber;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
flush_if_commit(id); flush_if_commit(id);
let options = options(index); let options = options(index);
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
@@ -55,7 +55,7 @@ unsafe extern "C" fn callback(
) { ) {
let ctid = &(*htup).t_self; let ctid = &(*htup).t_self;
let oid = (*index_relation).rd_node.relNode; let oid = (*index_relation).rd_node.relNode;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
let state = &mut *(state as *mut Builder); let state = &mut *(state as *mut Builder);
let vector = from_datum(*values.add(0)); let vector = from_datum(*values.add(0));
let data = (vector, Pointer::from_sys(*ctid)); let data = (vector, Pointer::from_sys(*ctid));
@@ -78,7 +78,7 @@ unsafe extern "C" fn callback(
let oid = (*index_relation).rd_node.relNode; let oid = (*index_relation).rd_node.relNode;
#[cfg(feature = "pg16")] #[cfg(feature = "pg16")]
let oid = (*index_relation).rd_locator.relNumber; let oid = (*index_relation).rd_locator.relNumber;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
let state = &mut *(state as *mut Builder); let state = &mut *(state as *mut Builder);
let vector = from_datum(*values.add(0)); let vector = from_datum(*values.add(0));
let data = (vector, Pointer::from_sys(*ctid)); let data = (vector, Pointer::from_sys(*ctid));

2
src/index/am_scan.rs
View File

@@ -95,7 +95,7 @@ pub unsafe fn next_scan(scan: pgrx::pg_sys::IndexScanDesc) -> bool {
let oid = (*(*scan).indexRelation).rd_node.relNode; let oid = (*(*scan).indexRelation).rd_node.relNode;
#[cfg(feature = "pg16")] #[cfg(feature = "pg16")]
let oid = (*(*scan).indexRelation).rd_locator.relNumber; let oid = (*(*scan).indexRelation).rd_locator.relNumber;
let id = Id::from_sys(oid); let id = Handle::from_sys(oid);
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();

4
src/index/am_setup.rs
View File

@@ -9,7 +9,7 @@ use std::ffi::CStr;
use validator::Validate; use validator::Validate;
pub fn helper_offset() -> usize { pub fn helper_offset() -> usize {
std::mem::offset_of!(Helper, offset) bytemuck::offset_of!(Helper, offset)
} }
pub fn helper_size() -> usize { pub fn helper_size() -> usize {
@@ -101,7 +101,7 @@ pub unsafe fn options(index_relation: pgrx::pg_sys::Relation) -> IndexOptions {
options options
} }
#[derive(Copy, Clone, Debug)] #[derive(Copy, Clone, Debug, Default)]
#[repr(C)] #[repr(C)]
struct Helper { struct Helper {
pub vl_len_: i32, pub vl_len_: i32,

12
src/index/am_update.rs
View File

@@ -2,14 +2,14 @@ use crate::index::hook_transaction::flush_if_commit;
use crate::prelude::*; use crate::prelude::*;
use service::prelude::*; use service::prelude::*;
pub fn update_insert(id: Id, vector: DynamicVector, tid: pgrx::pg_sys::ItemPointerData) { pub fn update_insert(handle: Handle, vector: DynamicVector, tid: pgrx::pg_sys::ItemPointerData) {
flush_if_commit(id); flush_if_commit(handle);
let p = Pointer::from_sys(tid); let p = Pointer::from_sys(tid);
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
rpc.insert(id, (vector, p)); rpc.insert(handle, (vector, p));
} }
pub fn update_delete(id: Id, hook: impl Fn(Pointer) -> bool) { pub fn update_delete(handle: Handle, hook: impl Fn(Pointer) -> bool) {
struct Delete<H> { struct Delete<H> {
hook: H, hook: H,
} }
@@ -25,7 +25,7 @@ pub fn update_delete(id: Id, hook: impl Fn(Pointer) -> bool) {
let client_delete = Delete { hook }; let client_delete = Delete { hook };
flush_if_commit(id); flush_if_commit(handle);
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
rpc.delete(id, client_delete); rpc.delete(handle, client_delete);
} }

6
src/index/hook_transaction.rs
View File

@@ -2,7 +2,7 @@ use crate::utils::cells::PgRefCell;
use service::prelude::*; use service::prelude::*;
use std::collections::BTreeSet; use std::collections::BTreeSet;
static FLUSH_IF_COMMIT: PgRefCell<BTreeSet<Id>> = unsafe { PgRefCell::new(BTreeSet::new()) }; static FLUSH_IF_COMMIT: PgRefCell<BTreeSet<Handle>> = unsafe { PgRefCell::new(BTreeSet::new()) };
pub fn aborting() { pub fn aborting() {
*FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new(); *FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new();
@@ -21,6 +21,6 @@ pub fn committing() {
*FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new(); *FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new();
} }
pub fn flush_if_commit(id: Id) { pub fn flush_if_commit(handle: Handle) {
FLUSH_IF_COMMIT.borrow_mut().insert(id); FLUSH_IF_COMMIT.borrow_mut().insert(handle);
} }

16
src/index/hooks.rs
View File

@@ -42,12 +42,14 @@ unsafe fn xact_delete() {
let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _); let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _);
if n > 0 { if n > 0 {
let nodes = std::slice::from_raw_parts(ptr, n as usize); let nodes = std::slice::from_raw_parts(ptr, n as usize);
let ids = nodes let handles = nodes
.iter() .iter()
.map(|node| Id::from_sys(node.relNode)) .map(|node| Handle::from_sys(node.relNode))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
rpc.destory(ids); for handle in handles {
rpc.destory(handle);
}
} }
} }
@@ -57,11 +59,13 @@ unsafe fn xact_delete() {
let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _); let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _);
if n > 0 { if n > 0 {
let nodes = std::slice::from_raw_parts(ptr, n as usize); let nodes = std::slice::from_raw_parts(ptr, n as usize);
let ids = nodes let handles = nodes
.iter() .iter()
.map(|node| Id::from_sys(node.relNumber)) .map(|node| Handle::from_sys(node.relNumber))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
rpc.destory(ids); for handle in handles {
rpc.destory(handle);
}
} }
} }

76
src/index/views.rs
View File

@@ -4,6 +4,7 @@ use service::prelude::*;
pgrx::extension_sql!( pgrx::extension_sql!(
"\ "\
CREATE TYPE VectorIndexStat AS ( CREATE TYPE VectorIndexStat AS (
idx_status TEXT,
idx_indexing BOOL, idx_indexing BOOL,
idx_tuples BIGINT, idx_tuples BIGINT,
idx_sealed BIGINT[], idx_sealed BIGINT[],
@@ -17,36 +18,63 @@ CREATE TYPE VectorIndexStat AS (
#[pgrx::pg_extern(volatile, strict)] #[pgrx::pg_extern(volatile, strict)]
fn vector_stat(oid: pgrx::pg_sys::Oid) -> pgrx::composite_type!("VectorIndexStat") { fn vector_stat(oid: pgrx::pg_sys::Oid) -> pgrx::composite_type!("VectorIndexStat") {
let id = Id::from_sys(oid); use service::index::IndexStat;
let id = Handle::from_sys(oid);
let mut res = pgrx::prelude::PgHeapTuple::new_composite_type("VectorIndexStat").unwrap(); let mut res = pgrx::prelude::PgHeapTuple::new_composite_type("VectorIndexStat").unwrap();
let mut rpc = crate::ipc::client::borrow_mut(); let mut rpc = crate::ipc::client::borrow_mut();
let stat = rpc.stat(id); let stat = rpc.stat(id);
res.set_by_name("idx_indexing", stat.indexing).unwrap(); match stat {
res.set_by_name("idx_tuples", { IndexStat::Normal {
let mut tuples = 0; indexing,
tuples += stat.sealed.iter().map(|x| *x as i64).sum::<i64>(); options,
tuples += stat.growing.iter().map(|x| *x as i64).sum::<i64>(); segments,
tuples += stat.write as i64; } => {
tuples res.set_by_name("idx_status", "NORMAL").unwrap();
}) res.set_by_name("idx_indexing", indexing).unwrap();
.unwrap();
res.set_by_name("idx_sealed", {
let sealed = stat.sealed;
sealed.into_iter().map(|x| x as i64).collect::<Vec<_>>()
})
.unwrap();
res.set_by_name("idx_growing", {
let growing = stat.growing;
growing.into_iter().map(|x| x as i64).collect::<Vec<_>>()
})
.unwrap();
res.set_by_name("idx_write", stat.write as i64).unwrap();
res.set_by_name( res.set_by_name(
"idx_size", "idx_tuples",
stat.sizes.iter().map(|x| x.size as i64).sum::<i64>(), segments.iter().map(|x| x.length as i64).sum::<i64>(),
) )
.unwrap(); .unwrap();
res.set_by_name("idx_options", serde_json::to_string(&stat.options)) res.set_by_name(
"idx_sealed",
segments
.iter()
.filter(|x| x.typ == "sealed")
.map(|x| x.length as i64)
.collect::<Vec<_>>(),
)
.unwrap();
res.set_by_name(
"idx_growing",
segments
.iter()
.filter(|x| x.typ == "growing")
.map(|x| x.length as i64)
.collect::<Vec<_>>(),
)
.unwrap();
res.set_by_name(
"idx_write",
segments
.iter()
.filter(|x| x.typ == "write")
.map(|x| x.length as i64)
.sum::<i64>(),
)
.unwrap();
res.set_by_name(
"idx_size",
segments.iter().map(|x| x.size as i64).sum::<i64>(),
)
.unwrap();
res.set_by_name("idx_options", serde_json::to_string(&options))
.unwrap(); .unwrap();
res res
} }
IndexStat::Upgrade => {
res.set_by_name("idx_status", "UPGRADE").unwrap();
res
}
}
}

32
src/ipc/client/mod.rs
View File

@@ -54,21 +54,21 @@ impl Rpc {
} }
impl ClientGuard<Rpc> { impl ClientGuard<Rpc> {
pub fn create(&mut self, id: Id, options: IndexOptions) { pub fn create(&mut self, handle: Handle, options: IndexOptions) {
let packet = RpcPacket::Create { id, options }; let packet = RpcPacket::Create { handle, options };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let create::CreatePacket::Leave {} = self.socket.recv().friendly(); let create::CreatePacket::Leave {} = self.socket.recv().friendly();
} }
pub fn search( pub fn search(
&mut self, &mut self,
id: Id, handle: Handle,
search: (DynamicVector, usize), search: (DynamicVector, usize),
prefilter: bool, prefilter: bool,
gucs: SearchGucs, gucs: SearchGucs,
mut t: impl Search, mut t: impl Search,
) -> Vec<Pointer> { ) -> Vec<Pointer> {
let packet = RpcPacket::Search { let packet = RpcPacket::Search {
id, handle,
search, search,
prefilter, prefilter,
gucs, gucs,
@@ -87,8 +87,8 @@ impl ClientGuard<Rpc> {
} }
} }
} }
pub fn delete(&mut self, id: Id, mut t: impl Delete) { pub fn delete(&mut self, handle: Handle, mut t: impl Delete) {
let packet = RpcPacket::Delete { id }; let packet = RpcPacket::Delete { handle };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
loop { loop {
match self.socket.recv().friendly() { match self.socket.recv().friendly() {
@@ -103,29 +103,29 @@ impl ClientGuard<Rpc> {
} }
} }
} }
pub fn insert(&mut self, id: Id, insert: (DynamicVector, Pointer)) { pub fn insert(&mut self, handle: Handle, insert: (DynamicVector, Pointer)) {
let packet = RpcPacket::Insert { id, insert }; let packet = RpcPacket::Insert { handle, insert };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let insert::InsertPacket::Leave {} = self.socket.recv().friendly(); let insert::InsertPacket::Leave {} = self.socket.recv().friendly();
} }
pub fn flush(&mut self, id: Id) { pub fn flush(&mut self, handle: Handle) {
let packet = RpcPacket::Flush { id }; let packet = RpcPacket::Flush { handle };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let flush::FlushPacket::Leave {} = self.socket.recv().friendly(); let flush::FlushPacket::Leave {} = self.socket.recv().friendly();
} }
pub fn destory(&mut self, ids: Vec<Id>) { pub fn destory(&mut self, handle: Handle) {
let packet = RpcPacket::Destory { ids }; let packet = RpcPacket::Destory { handle };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let destory::DestoryPacket::Leave {} = self.socket.recv().friendly(); let destory::DestoryPacket::Leave {} = self.socket.recv().friendly();
} }
pub fn stat(&mut self, id: Id) -> IndexStat { pub fn stat(&mut self, handle: Handle) -> IndexStat {
let packet = RpcPacket::Stat { id }; let packet = RpcPacket::Stat { handle };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let stat::StatPacket::Leave { result } = self.socket.recv().friendly(); let stat::StatPacket::Leave { result } = self.socket.recv().friendly();
result result
} }
pub fn vbase(mut self, id: Id, vbase: (DynamicVector, usize)) -> ClientGuard<Vbase> { pub fn vbase(mut self, handle: Handle, vbase: (DynamicVector, usize)) -> ClientGuard<Vbase> {
let packet = RpcPacket::Vbase { id, vbase }; let packet = RpcPacket::Vbase { handle, vbase };
self.socket.send(packet).friendly(); self.socket.send(packet).friendly();
let vbase::VbaseErrorPacket {} = self.socket.recv().friendly(); let vbase::VbaseErrorPacket {} = self.socket.recv().friendly();
ClientGuard::map(self) ClientGuard::map(self)

16
src/ipc/packet/mod.rs
View File

@@ -15,33 +15,33 @@ use service::prelude::*;
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub enum RpcPacket { pub enum RpcPacket {
Create { Create {
id: Id, handle: Handle,
options: IndexOptions, options: IndexOptions,
}, },
Delete { Delete {
id: Id, handle: Handle,
}, },
Destory { Destory {
ids: Vec<Id>, handle: Handle,
}, },
Flush { Flush {
id: Id, handle: Handle,
}, },
Insert { Insert {
id: Id, handle: Handle,
insert: (DynamicVector, Pointer), insert: (DynamicVector, Pointer),
}, },
Search { Search {
id: Id, handle: Handle,
search: (DynamicVector, usize), search: (DynamicVector, usize),
prefilter: bool, prefilter: bool,
gucs: SearchGucs, gucs: SearchGucs,
}, },
Stat { Stat {
id: Id, handle: Handle,
}, },
Vbase { Vbase {
id: Id, handle: Handle,
vbase: (DynamicVector, usize), vbase: (DynamicVector, usize),
}, },
} }

48
src/ipc/server/mod.rs
View File

@@ -16,33 +16,33 @@ impl RpcHandler {
} }
pub fn handle(mut self) -> Result<RpcHandle, IpcError> { pub fn handle(mut self) -> Result<RpcHandle, IpcError> {
Ok(match self.socket.recv::<RpcPacket>()? { Ok(match self.socket.recv::<RpcPacket>()? {
RpcPacket::Create { id, options } => RpcHandle::Create { RpcPacket::Create { handle, options } => RpcHandle::Create {
id, handle,
options, options,
x: Create { x: Create {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Insert { id, insert } => RpcHandle::Insert { RpcPacket::Insert { handle, insert } => RpcHandle::Insert {
id, handle,
insert, insert,
x: Insert { x: Insert {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Delete { id } => RpcHandle::Delete { RpcPacket::Delete { handle } => RpcHandle::Delete {
id, handle,
x: Delete { x: Delete {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Search { RpcPacket::Search {
id, handle,
search, search,
prefilter, prefilter,
gucs, gucs,
} => RpcHandle::Search { } => RpcHandle::Search {
id, handle,
search, search,
prefilter, prefilter,
gucs, gucs,
@@ -50,26 +50,26 @@ impl RpcHandler {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Flush { id } => RpcHandle::Flush { RpcPacket::Flush { handle } => RpcHandle::Flush {
id, handle,
x: Flush { x: Flush {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Destory { ids } => RpcHandle::Destory { RpcPacket::Destory { handle } => RpcHandle::Destory {
ids, handle,
x: Destory { x: Destory {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Stat { id } => RpcHandle::Stat { RpcPacket::Stat { handle } => RpcHandle::Stat {
id, handle,
x: Stat { x: Stat {
socket: self.socket, socket: self.socket,
}, },
}, },
RpcPacket::Vbase { id, vbase } => RpcHandle::Vbase { RpcPacket::Vbase { handle, vbase } => RpcHandle::Vbase {
id, handle,
vbase, vbase,
x: Vbase { x: Vbase {
socket: self.socket, socket: self.socket,
@@ -81,40 +81,40 @@ impl RpcHandler {
pub enum RpcHandle { pub enum RpcHandle {
Create { Create {
id: Id, handle: Handle,
options: IndexOptions, options: IndexOptions,
x: Create, x: Create,
}, },
Search { Search {
id: Id, handle: Handle,
search: (DynamicVector, usize), search: (DynamicVector, usize),
prefilter: bool, prefilter: bool,
gucs: SearchGucs, gucs: SearchGucs,
x: Search, x: Search,
}, },
Insert { Insert {
id: Id, handle: Handle,
insert: (DynamicVector, Pointer), insert: (DynamicVector, Pointer),
x: Insert, x: Insert,
}, },
Delete { Delete {
id: Id, handle: Handle,
x: Delete, x: Delete,
}, },
Flush { Flush {
id: Id, handle: Handle,
x: Flush, x: Flush,
}, },
Destory { Destory {
ids: Vec<Id>, handle: Handle,
x: Destory, x: Destory,
}, },
Stat { Stat {
id: Id, handle: Handle,
x: Stat, x: Stat,
}, },
Vbase { Vbase {
id: Id, handle: Handle,
vbase: (DynamicVector, usize), vbase: (DynamicVector, usize),
x: Vbase, x: Vbase,
}, },

1
src/lib.rs
View File

@@ -1,7 +1,6 @@
//! Postgres vector extension. //! Postgres vector extension.
//! //!
//! Provides an easy-to-use extension for vector similarity search. //! Provides an easy-to-use extension for vector similarity search.
#![feature(offset_of)]
#![feature(never_type)] #![feature(never_type)]
mod bgworker; mod bgworker;

2
src/prelude.rs
View File

@@ -4,7 +4,7 @@ pub trait FromSys<T> {
fn from_sys(sys: T) -> Self; fn from_sys(sys: T) -> Self;
} }
impl FromSys<pgrx::pg_sys::Oid> for Id { impl FromSys<pgrx::pg_sys::Oid> for Handle {
fn from_sys(sys: pgrx::pg_sys::Oid) -> Self { fn from_sys(sys: pgrx::pg_sys::Oid) -> Self {
Self { Self {
newtype: sys.as_u32(), newtype: sys.as_u32(),