diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62eb0c6..6e0f5e7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,7 +87,7 @@ jobs: sudo apt-get update sudo apt-get -y install libpq-dev postgresql-${{ matrix.version }} postgresql-server-dev-${{ matrix.version }} sudo apt-get -y install clang-16 - cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) + cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2) cargo pgrx init --pg${{ matrix.version }}=/usr/lib/postgresql/${{ matrix.version }}/bin/pg_config if [[ "${{ matrix.arch }}" == "aarch64" ]]; then sudo apt-get -y install crossbuild-essential-arm64 diff --git a/Cargo.lock b/Cargo.lock index 03fbedd..a52b978 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,21 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstyle" version = "1.0.4" @@ -471,9 +456,9 @@ dependencies = [ [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -531,19 +516,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "windows-targets 0.48.5", -] - [[package]] name = "clang-sys" version = "1.6.1" @@ -557,9 +529,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -577,9 +549,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstyle", "clap_lex", @@ -779,41 +751,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "darling" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.43", -] - -[[package]] -name = "darling_macro" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.43", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -821,22 +758,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown", "lock_api", "once_cell", "parking_lot_core", ] -[[package]] -name = "deranged" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" -dependencies = [ - "powerfmt", - "serde", -] - [[package]] name = "detect" version = "0.0.0" @@ -852,12 +779,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" @@ -1087,15 +1008,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1286,12 +1198,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.3" @@ -1323,12 +1229,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "hmac" version = "0.12.1" @@ -1429,35 +1329,6 @@ dependencies = [ "want", ] -[[package]] -name = "iana-time-zone" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "0.4.0" @@ -1490,17 +1361,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - [[package]] name = "indexmap" version = "2.1.0" @@ -1508,8 +1368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.3", - "serde", + "hashbrown", ] [[package]] @@ -1534,13 +1393,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", "rustix 0.38.28", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1579,6 +1438,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1614,7 +1482,7 @@ dependencies = [ "diff", "ena", "is-terminal", - "itertools", + "itertools 0.10.5", "lalrpop-util", "petgraph", "regex", @@ -1750,9 +1618,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -1806,9 +1674,9 @@ dependencies = [ [[package]] name = "mockall" -version = "0.11.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" dependencies = [ "cfg-if", "downcast", @@ -1821,14 +1689,14 @@ dependencies = [ [[package]] name = "mockall_derive" -version = "0.11.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.43", ] [[package]] @@ -1869,12 +1737,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "ntapi" version = "0.4.1" @@ -2031,13 +1893,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap", ] [[package]] name = "pgrx" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb44171122605250e719ca2ae49afb357bdb2fce4b3c876fcf2225165237328a" dependencies = [ "atomic-traits", "bitflags 2.4.1", @@ -2060,8 +1923,9 @@ dependencies = [ [[package]] name = "pgrx-macros" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18ac8628b7de2f29a93d0abdbdcaee95a0e0ef4b59fd4de99cc117e166e843b" dependencies = [ "pgrx-sql-entity-graph", "proc-macro2", @@ -2071,8 +1935,9 @@ dependencies = [ [[package]] name = "pgrx-pg-config" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd45ac6eb1142c5690df63c4e0bdfb74f27c9f93a7af84f064dc2c0a2c2d6f7" dependencies = [ "cargo_toml", "dirs", @@ -2088,8 +1953,9 @@ dependencies = [ [[package]] name = "pgrx-pg-sys" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81c6207939582934fc26fceb651cb5338e363c06ddc6b2d50ca71867f7c70ffe" dependencies = [ "bindgen", "clang-sys", @@ -2111,8 +1977,9 @@ dependencies = [ [[package]] name = "pgrx-sql-entity-graph" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50083de83b1fac2484e8f2c2a7da5fed0193904e2578fa6c4ce02262c455c2b" dependencies = [ "convert_case", "eyre", @@ -2125,8 +1992,9 @@ dependencies = [ [[package]] name = "pgrx-tests" -version = "0.11.0" -source = "git+https://github.com/tensorchord/pgrx.git?rev=7c30e2023876c1efce613756f5ec81f3ab05696b#7c30e2023876c1efce613756f5ec81f3ab05696b" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0115cd80d9e3ca1d5d2a8ab8b7320d6ed614a53d025b86152696a8b3caa75" dependencies = [ "clap-cargo", "eyre", @@ -2295,12 +2163,6 @@ dependencies = [ "postgres-protocol", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2315,16 +2177,13 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "predicates" -version = "2.1.5" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" dependencies = [ - "difflib", - "float-cmp", - "itertools", - "normalize-line-endings", + "anstyle", + "itertools 0.11.0", "predicates-core", - "regex", ] [[package]] @@ -2788,35 +2647,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" -dependencies = [ - "base64", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.1.0", - "serde", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.43", -] - [[package]] name = "service" version = "0.0.0" @@ -2844,8 +2674,6 @@ dependencies = [ "rustix 0.38.28", "serde", "serde_json", - "serde_with", - "tempfile", "thiserror", "ulock-sys", "uuid", @@ -2990,12 +2818,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "subtle" version = "2.5.0" @@ -3110,35 +2932,6 @@ dependencies = [ "syn 2.0.43", ] -[[package]] -name = "time" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" -dependencies = [ - "deranged", - "itoa", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" -dependencies = [ - "time-core", -] - [[package]] name = "tiny-keccak" version = "2.0.2" @@ -3259,7 +3052,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.1.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -3490,6 +3283,7 @@ name = "vectors" version = "0.0.0" dependencies = [ "bincode", + "bytemuck", "byteorder", "detect", "env_logger", @@ -3681,15 +3475,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-core" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 2328769..0b90db5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,19 +27,19 @@ byteorder.workspace = true bincode.workspace = true half.workspace = true num-traits.workspace = true +rand.workspace = true +bytemuck.workspace = true service = { path = "crates/service" } detect = { path = "crates/detect" } -pgrx = { git = "https://github.com/tensorchord/pgrx.git", rev = "7c30e2023876c1efce613756f5ec81f3ab05696b", default-features = false, features = [ -] } +pgrx = { version = "0.11.2", default-features = false, features = [] } openai_api_rust = { git = "https://github.com/tensorchord/openai-api.git", rev = "228d54b6002e98257b3c81501a054942342f585f" } env_logger = "0.10.0" toml = "0.8.8" -rand = "0.8.5" [dev-dependencies] -pgrx-tests = { git = "https://github.com/tensorchord/pgrx.git", rev = "7c30e2023876c1efce613756f5ec81f3ab05696b" } +pgrx-tests = "0.11.2" httpmock = "0.6" -mockall = "0.11.4" +mockall = "0.12" [lints] clippy.too_many_arguments = "allow" @@ -62,7 +62,8 @@ serde = "~1.0" serde_json = "1" thiserror = "~1.0" bincode = "~1.3" -byteorder = "~1.4" +byteorder = "~1.5" +bytemuck = { version = "~1.14", features = ["extern_crate_alloc"] } half = { version = "~2.3", features = [ "bytemuck", "num-traits", @@ -72,6 +73,7 @@ half = { version = "~2.3", features = [ num-traits = "~0.2" validator = { version = "~0.16", features = ["derive"] } rustix = { version = "~0.38", features = ["fs", "net", "mm"] } +rand = "~0.8" [profile.dev] panic = "unwind" diff --git a/crates/service/Cargo.toml b/crates/service/Cargo.toml index 150619d..c8cc2ae 100644 --- a/crates/service/Cargo.toml +++ b/crates/service/Cargo.toml @@ -15,22 +15,20 @@ byteorder.workspace = true bincode.workspace = true half.workspace = true num-traits.workspace = true +rand.workspace = true +bytemuck.workspace = true c = { path = "../c" } detect = { path = "../detect" } -rand = "0.8.5" crc32fast = "1.3.2" crossbeam = "0.8.2" dashmap = "5.4.0" parking_lot = "0.12.1" memoffset = "0.9.0" -tempfile = "3.6.0" arrayvec = { version = "0.7.3", features = ["serde"] } memmap2 = "0.9.0" rayon = "1.6.1" uuid = { version = "1.6.1", features = ["serde"] } arc-swap = "1.6.0" -bytemuck = { version = "1.14.0", features = ["extern_crate_alloc"] } -serde_with = "3.4.0" multiversion = "0.7.3" [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/service/src/index/mod.rs b/crates/service/src/index/mod.rs index f41df99..dafd195 100644 --- a/crates/service/src/index/mod.rs +++ b/crates/service/src/index/mod.rs @@ -64,7 +64,7 @@ pub struct IndexOptions { #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct SegmentSizeInfo { +pub struct SegmentStat { pub id: Uuid, #[serde(rename = "type")] pub typ: String, @@ -73,13 +73,13 @@ pub struct SegmentSizeInfo { } #[derive(Debug, Serialize, Deserialize)] -pub struct IndexStat { - pub indexing: bool, - pub sealed: Vec, - pub growing: Vec, - pub write: u32, - pub options: IndexOptions, - pub sizes: Vec, +pub enum IndexStat { + Normal { + indexing: bool, + segments: Vec, + options: IndexOptions, + }, + Upgrade, } pub struct Index { @@ -97,6 +97,11 @@ impl Index { pub fn create(path: PathBuf, options: IndexOptions) -> Arc { assert!(options.validate().is_ok()); std::fs::create_dir(&path).unwrap(); + std::fs::write( + path.join("options"), + serde_json::to_string::(&options).unwrap(), + ) + .unwrap(); std::fs::create_dir(path.join("segments")).unwrap(); let startup = FileAtomic::create( path.join("startup"), @@ -132,7 +137,10 @@ impl Index { OptimizerSealing::new(index.clone()).spawn(); index } - pub fn open(path: PathBuf, options: IndexOptions) -> Arc { + pub fn open(path: PathBuf) -> Arc { + let options = + serde_json::from_slice::(&std::fs::read(path.join("options")).unwrap()) + .unwrap(); let tracker = Arc::new(IndexTracker { path: path.clone() }); let startup = FileAtomic::::open(path.join("startup")); clean( @@ -244,18 +252,22 @@ impl Index { } pub fn stat(&self) -> IndexStat { let view = self.view(); - IndexStat { + IndexStat::Normal { indexing: self.instant_index.load() < self.instant_write.load(), - sealed: view.sealed.values().map(|x| x.len()).collect(), - growing: view.growing.values().map(|x| x.len()).collect(), - write: view.write.as_ref().map(|(_, x)| x.len()).unwrap_or(0), options: self.options().clone(), - sizes: view - .sealed - .values() - .map(|x| x.size()) - .chain(view.growing.values().map(|x| x.size())) - .collect(), + segments: { + let mut segments = Vec::new(); + for sealed in view.sealed.values() { + segments.push(sealed.stat_sealed()); + } + for growing in view.growing.values() { + segments.push(growing.stat_growing()); + } + if let Some(write) = view.write.as_ref().map(|(_, x)| x) { + segments.push(write.stat_write()); + } + segments + }, } } } diff --git a/crates/service/src/index/segments/growing.rs b/crates/service/src/index/segments/growing.rs index 5a7ff7b..453e128 100644 --- a/crates/service/src/index/segments/growing.rs +++ b/crates/service/src/index/segments/growing.rs @@ -1,9 +1,7 @@ -#![allow(clippy::all)] // Clippy bug. - use super::SegmentTracker; use crate::index::IndexOptions; use crate::index::IndexTracker; -use crate::index::SegmentSizeInfo; +use crate::index::SegmentStat; use crate::prelude::*; use crate::utils::dir_ops::sync_dir; use crate::utils::file_wal::FileWal; @@ -12,8 +10,7 @@ use serde::{Deserialize, Serialize}; use std::cell::UnsafeCell; use std::mem::MaybeUninit; use std::path::PathBuf; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use thiserror::Error; use uuid::Uuid; @@ -44,6 +41,7 @@ impl GrowingSegment { sync_dir(&path); Arc::new(Self { uuid, + #[allow(clippy::uninit_vec)] vec: unsafe { let mut vec = Vec::with_capacity(capacity as usize); vec.set_len(capacity as usize); @@ -141,14 +139,22 @@ impl GrowingSegment { pub fn len(&self) -> u32 { self.len.load(Ordering::Acquire) as u32 } - pub fn size(&self) -> SegmentSizeInfo { - SegmentSizeInfo { + pub fn stat_growing(&self) -> SegmentStat { + SegmentStat { id: self.uuid, typ: "growing".to_string(), length: self.len() as usize, size: (self.len() as u64) * (std::mem::size_of::>() as u64), } } + pub fn stat_write(&self) -> SegmentStat { + SegmentStat { + id: self.uuid, + typ: "write".to_string(), + length: self.len() as usize, + size: (self.len() as u64) * (std::mem::size_of::>() as u64), + } + } pub fn vector(&self, i: u32) -> &[S::Scalar] { let i = i as usize; if i >= self.len.load(Ordering::Acquire) { diff --git a/crates/service/src/index/segments/sealed.rs b/crates/service/src/index/segments/sealed.rs index b32c7d1..53326e4 100644 --- a/crates/service/src/index/segments/sealed.rs +++ b/crates/service/src/index/segments/sealed.rs @@ -1,7 +1,7 @@ use super::growing::GrowingSegment; use super::SegmentTracker; use crate::index::indexing::{DynamicIndexIter, DynamicIndexing}; -use crate::index::{IndexOptions, IndexTracker, SegmentSizeInfo}; +use crate::index::{IndexOptions, IndexTracker, SegmentStat}; use crate::prelude::*; use crate::utils::dir_ops::{dir_size, sync_dir}; use serde::{Deserialize, Serialize}; @@ -58,21 +58,14 @@ impl SealedSegment { pub fn len(&self) -> u32 { self.indexing.len() } - pub fn size(&self) -> SegmentSizeInfo { - let mut info = SegmentSizeInfo { + pub fn stat_sealed(&self) -> SegmentStat { + let path = self._tracker.path.join("indexing"); + SegmentStat { id: self.uuid, typ: "sealed".to_string(), length: self.len() as usize, - size: 0, - }; - let path = self._tracker.path.join("indexing"); - match dir_size(&path) { - Ok(size) => info.size = size as u64, - Err(e) => { - panic!("Failed to get size of {:?}: {}", path, e); - } + size: dir_size(&path).unwrap(), } - info } pub fn vector(&self, i: u32) -> &[S::Scalar] { self.indexing.vector(i) diff --git a/crates/service/src/instance/metadata.rs b/crates/service/src/instance/metadata.rs new file mode 100644 index 0000000..34a995b --- /dev/null +++ b/crates/service/src/instance/metadata.rs @@ -0,0 +1,46 @@ +use serde::{Deserialize, Serialize}; +use std::error::Error; +use std::path::Path; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum MetadataError { + #[error("Invalid version.")] + InvalidVersion, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Metadata { + #[serde(default)] + pub version: Option, + #[serde(default)] + pub soft_version: Option, +} + +impl Metadata { + const VERSION: u64 = 2; + const SOFT_VERSION: u64 = 1; +} + +impl Metadata { + pub fn write(path: impl AsRef) { + let metadata = Metadata { + version: Some(Self::VERSION), + soft_version: Some(Self::SOFT_VERSION), + }; + let contents = serde_json::to_string(&metadata).unwrap(); + std::fs::write(path, contents).unwrap(); + } + pub fn read(path: impl AsRef) -> Result<(), Box> { + use MetadataError::*; + let contents = std::fs::read_to_string(path)?; + let metadata = serde_json::from_str::(&contents)?; + if Self::VERSION != metadata.version.ok_or(InvalidVersion)? { + return Err(Box::new(InvalidVersion)); + } + if Self::SOFT_VERSION < metadata.soft_version.ok_or(InvalidVersion)? { + return Err(Box::new(InvalidVersion)); + } + Ok(()) + } +} diff --git a/crates/service/src/worker/instance.rs b/crates/service/src/instance/mod.rs similarity index 67% rename from crates/service/src/worker/instance.rs rename to crates/service/src/instance/mod.rs index 4c72bf4..fc04450 100644 --- a/crates/service/src/worker/instance.rs +++ b/crates/service/src/instance/mod.rs @@ -1,3 +1,5 @@ +pub mod metadata; + use crate::index::segments::SearchGucs; use crate::index::Index; use crate::index::IndexOptions; @@ -16,67 +18,120 @@ pub enum Instance { F16Cos(Arc>), F16Dot(Arc>), F16L2(Arc>), + Upgrade, } impl Instance { pub fn create(path: PathBuf, options: IndexOptions) -> Self { match (options.vector.d, options.vector.k) { - (Distance::Cos, Kind::F32) => Self::F32Cos(Index::create(path, options)), - (Distance::Dot, Kind::F32) => Self::F32Dot(Index::create(path, options)), - (Distance::L2, Kind::F32) => Self::F32L2(Index::create(path, options)), - (Distance::Cos, Kind::F16) => Self::F16Cos(Index::create(path, options)), - (Distance::Dot, Kind::F16) => Self::F16Dot(Index::create(path, options)), - (Distance::L2, Kind::F16) => Self::F16L2(Index::create(path, options)), + (Distance::Cos, Kind::F32) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F32Cos(index) + } + (Distance::Dot, Kind::F32) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F32Dot(index) + } + (Distance::L2, Kind::F32) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F32L2(index) + } + (Distance::Cos, Kind::F16) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F16Cos(index) + } + (Distance::Dot, Kind::F16) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F16Dot(index) + } + (Distance::L2, Kind::F16) => { + let index = Index::create(path.clone(), options); + self::metadata::Metadata::write(path.join("metadata")); + Self::F16L2(index) + } } } - pub fn open(path: PathBuf, options: IndexOptions) -> Self { + pub fn open(path: PathBuf) -> Self { + if self::metadata::Metadata::read(path.join("metadata")).is_err() { + return Self::Upgrade; + } + let options = + serde_json::from_slice::(&std::fs::read(path.join("options")).unwrap()) + .unwrap(); match (options.vector.d, options.vector.k) { - (Distance::Cos, Kind::F32) => Self::F32Cos(Index::open(path, options)), - (Distance::Dot, Kind::F32) => Self::F32Dot(Index::open(path, options)), - (Distance::L2, Kind::F32) => Self::F32L2(Index::open(path, options)), - (Distance::Cos, Kind::F16) => Self::F16Cos(Index::open(path, options)), - (Distance::Dot, Kind::F16) => Self::F16Dot(Index::open(path, options)), - (Distance::L2, Kind::F16) => Self::F16L2(Index::open(path, options)), + (Distance::Cos, Kind::F32) => Self::F32Cos(Index::open(path)), + (Distance::Dot, Kind::F32) => Self::F32Dot(Index::open(path)), + (Distance::L2, Kind::F32) => Self::F32L2(Index::open(path)), + (Distance::Cos, Kind::F16) => Self::F16Cos(Index::open(path)), + (Distance::Dot, Kind::F16) => Self::F16Dot(Index::open(path)), + (Distance::L2, Kind::F16) => Self::F16L2(Index::open(path)), } } - pub fn options(&self) -> &IndexOptions { + pub fn options(&self) -> Result<&IndexOptions, FriendlyError> { match self { - Instance::F32Cos(x) => x.options(), - Instance::F32Dot(x) => x.options(), - Instance::F32L2(x) => x.options(), - Instance::F16Cos(x) => x.options(), - Instance::F16Dot(x) => x.options(), - Instance::F16L2(x) => x.options(), + Instance::F32Cos(x) => Ok(x.options()), + Instance::F32Dot(x) => Ok(x.options()), + Instance::F32L2(x) => Ok(x.options()), + Instance::F16Cos(x) => Ok(x.options()), + Instance::F16Dot(x) => Ok(x.options()), + Instance::F16L2(x) => Ok(x.options()), + Instance::Upgrade => Err(FriendlyError::Upgrade2), } } - pub fn refresh(&self) { + pub fn refresh(&self) -> Result<(), FriendlyError> { match self { - Instance::F32Cos(x) => x.refresh(), - Instance::F32Dot(x) => x.refresh(), - Instance::F32L2(x) => x.refresh(), - Instance::F16Cos(x) => x.refresh(), - Instance::F16Dot(x) => x.refresh(), - Instance::F16L2(x) => x.refresh(), + Instance::F32Cos(x) => { + x.refresh(); + Ok(()) + } + Instance::F32Dot(x) => { + x.refresh(); + Ok(()) + } + Instance::F32L2(x) => { + x.refresh(); + Ok(()) + } + Instance::F16Cos(x) => { + x.refresh(); + Ok(()) + } + Instance::F16Dot(x) => { + x.refresh(); + Ok(()) + } + Instance::F16L2(x) => { + x.refresh(); + Ok(()) + } + Instance::Upgrade => Err(FriendlyError::Upgrade2), } } - pub fn view(&self) -> InstanceView { + pub fn view(&self) -> Result { match self { - Instance::F32Cos(x) => InstanceView::F32Cos(x.view()), - Instance::F32Dot(x) => InstanceView::F32Dot(x.view()), - Instance::F32L2(x) => InstanceView::F32L2(x.view()), - Instance::F16Cos(x) => InstanceView::F16Cos(x.view()), - Instance::F16Dot(x) => InstanceView::F16Dot(x.view()), - Instance::F16L2(x) => InstanceView::F16L2(x.view()), + Instance::F32Cos(x) => Ok(InstanceView::F32Cos(x.view())), + Instance::F32Dot(x) => Ok(InstanceView::F32Dot(x.view())), + Instance::F32L2(x) => Ok(InstanceView::F32L2(x.view())), + Instance::F16Cos(x) => Ok(InstanceView::F16Cos(x.view())), + Instance::F16Dot(x) => Ok(InstanceView::F16Dot(x.view())), + Instance::F16L2(x) => Ok(InstanceView::F16L2(x.view())), + Instance::Upgrade => Err(FriendlyError::Upgrade2), } } - pub fn stat(&self) -> IndexStat { + pub fn stat(&self) -> Result { match self { - Instance::F32Cos(x) => x.stat(), - Instance::F32Dot(x) => x.stat(), - Instance::F32L2(x) => x.stat(), - Instance::F16Cos(x) => x.stat(), - Instance::F16Dot(x) => x.stat(), - Instance::F16L2(x) => x.stat(), + Instance::F32Cos(x) => Ok(x.stat()), + Instance::F32Dot(x) => Ok(x.stat()), + Instance::F32L2(x) => Ok(x.stat()), + Instance::F16Cos(x) => Ok(x.stat()), + Instance::F16Dot(x) => Ok(x.stat()), + Instance::F16L2(x) => Ok(x.stat()), + Instance::Upgrade => Ok(IndexStat::Upgrade), } } } diff --git a/crates/service/src/lib.rs b/crates/service/src/lib.rs index b534589..1ae36c6 100644 --- a/crates/service/src/lib.rs +++ b/crates/service/src/lib.rs @@ -3,6 +3,7 @@ pub mod algorithms; pub mod index; +pub mod instance; pub mod prelude; pub mod worker; diff --git a/crates/service/src/prelude/error.rs b/crates/service/src/prelude/error.rs index 19ccf09..b9ceff7 100644 --- a/crates/service/src/prelude/error.rs +++ b/crates/service/src/prelude/error.rs @@ -69,10 +69,15 @@ Please check the full PostgreSQL log to get more information.\ ")] Ipc, #[error("\ -The extension is upgraded. However, the index files is outdated. -ADVICE: Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\ +The extension is upgraded so all index files are outdated. +ADVICE: Delete all index files. Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\ ")] Upgrade, + #[error("\ +The extension is upgraded so this index is outdated. +ADVICE: Rebuild the index. Please read `https://github.com/tensorchord/pgvecto.rs/blob/main/docs/upgrade.md`.\ +")] + Upgrade2, } pub trait FriendlyErrorLike: Sized { diff --git a/crates/service/src/prelude/mod.rs b/crates/service/src/prelude/mod.rs index 5b66102..6bf6cd3 100644 --- a/crates/service/src/prelude/mod.rs +++ b/crates/service/src/prelude/mod.rs @@ -11,6 +11,6 @@ pub use self::scalar::{F16, F32}; pub use self::filter::{Filter, Payload}; pub use self::heap::{Heap, HeapElement}; -pub use self::sys::{Id, Pointer}; +pub use self::sys::{Handle, Pointer}; pub use num_traits::{Float, Zero}; diff --git a/crates/service/src/prelude/sys.rs b/crates/service/src/prelude/sys.rs index 640ff7a..5229bfd 100644 --- a/crates/service/src/prelude/sys.rs +++ b/crates/service/src/prelude/sys.rs @@ -2,27 +2,27 @@ use serde::{Deserialize, Serialize}; use std::{fmt::Display, num::ParseIntError, str::FromStr}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct Id { +pub struct Handle { pub newtype: u32, } -impl Id { +impl Handle { pub fn as_u32(self) -> u32 { self.newtype } } -impl Display for Id { +impl Display for Handle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_u32()) } } -impl FromStr for Id { +impl FromStr for Handle { type Err = ParseIntError; fn from_str(s: &str) -> Result { - Ok(Id { + Ok(Handle { newtype: u32::from_str(s)?, }) } diff --git a/crates/service/src/utils/dir_ops.rs b/crates/service/src/utils/dir_ops.rs index 222bae1..a815ec6 100644 --- a/crates/service/src/utils/dir_ops.rs +++ b/crates/service/src/utils/dir_ops.rs @@ -7,7 +7,7 @@ pub fn sync_dir(path: impl AsRef) { file.sync_all().expect("Failed to sync dir."); } -pub fn dir_size(dir: &Path) -> io::Result { +pub fn dir_size(dir: &Path) -> io::Result { let mut size = 0; if dir.is_dir() { for entry in read_dir(dir)? { @@ -21,7 +21,7 @@ pub fn dir_size(dir: &Path) -> io::Result { if path.is_dir() { size += dir_size(&path)?; } else { - size += entry.metadata()?.len() as usize; + size += entry.metadata()?.len(); } } } diff --git a/crates/service/src/worker/metadata.rs b/crates/service/src/worker/metadata.rs index e79f6cb..34a995b 100644 --- a/crates/service/src/worker/metadata.rs +++ b/crates/service/src/worker/metadata.rs @@ -18,7 +18,7 @@ pub struct Metadata { } impl Metadata { - const VERSION: u64 = 1; + const VERSION: u64 = 2; const SOFT_VERSION: u64 = 1; } @@ -38,7 +38,7 @@ impl Metadata { if Self::VERSION != metadata.version.ok_or(InvalidVersion)? { return Err(Box::new(InvalidVersion)); } - if Self::SOFT_VERSION <= metadata.soft_version.ok_or(InvalidVersion)? { + if Self::SOFT_VERSION < metadata.soft_version.ok_or(InvalidVersion)? { return Err(Box::new(InvalidVersion)); } Ok(()) diff --git a/crates/service/src/worker/mod.rs b/crates/service/src/worker/mod.rs index fcc3ee8..0876278 100644 --- a/crates/service/src/worker/mod.rs +++ b/crates/service/src/worker/mod.rs @@ -1,11 +1,10 @@ -pub mod instance; pub mod metadata; -use self::instance::Instance; use crate::index::segments::SearchGucs; use crate::index::IndexOptions; use crate::index::IndexStat; use crate::index::OutdatedError; +use crate::instance::Instance; use crate::prelude::*; use crate::utils::clean::clean; use crate::utils::dir_ops::sync_dir; @@ -13,8 +12,7 @@ use crate::utils::file_atomic::FileAtomic; use arc_swap::ArcSwap; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; -use serde_with::DisplayFromStr; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; @@ -49,12 +47,12 @@ impl Worker { let startup = FileAtomic::::open(path.join("startup")); clean( path.join("indexes"), - startup.get().indexes.keys().map(|s| s.to_string()), + startup.get().indexes.iter().map(|s| s.to_string()), ); let mut indexes = HashMap::new(); - for (&id, options) in startup.get().indexes.iter() { + for &id in startup.get().indexes.iter() { let path = path.join("indexes").join(id.to_string()); - let index = Instance::open(path, options.clone()); + let index = Instance::open(path); indexes.insert(id, index); } let view = Arc::new(WorkerView { @@ -67,17 +65,17 @@ impl Worker { view: ArcSwap::new(view), }) } - pub fn call_create(&self, id: Id, options: IndexOptions) { + pub fn call_create(&self, handle: Handle, options: IndexOptions) { let mut protect = self.protect.lock(); - let index = Instance::create(self.path.join("indexes").join(id.to_string()), options); - if protect.indexes.insert(id, index).is_some() { - panic!("index {} already exists", id) + let index = Instance::create(self.path.join("indexes").join(handle.to_string()), options); + if protect.indexes.insert(handle, index).is_some() { + panic!("index {} already exists", handle) } protect.maintain(&self.view); } pub fn call_search( &self, - id: Id, + handle: Handle, search: (DynamicVector, usize), gucs: SearchGucs, filter: F, @@ -86,80 +84,90 @@ impl Worker { F: FnMut(Pointer) -> bool, { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; - let view = index.view(); + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; + let view = index.view()?; view.search(search.1, search.0, gucs, filter) } pub fn call_insert( &self, - id: Id, + handle: Handle, insert: (DynamicVector, Pointer), ) -> Result<(), FriendlyError> { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; loop { - let view = index.view(); + let view = index.view()?; match view.insert(insert.0.clone(), insert.1)? { Ok(()) => break Ok(()), - Err(OutdatedError(_)) => index.refresh(), + Err(OutdatedError(_)) => index.refresh()?, } } } - pub fn call_delete(&self, id: Id, f: F) -> Result<(), FriendlyError> + pub fn call_delete(&self, handle: Handle, f: F) -> Result<(), FriendlyError> where F: FnMut(Pointer) -> bool, { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; - let view = index.view(); + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; + let view = index.view()?; view.delete(f); Ok(()) } - pub fn call_flush(&self, id: Id) -> Result<(), FriendlyError> { + pub fn call_flush(&self, handle: Handle) -> Result<(), FriendlyError> { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; - let view = index.view(); + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; + let view = index.view()?; view.flush(); Ok(()) } - pub fn call_destory(&self, ids: Vec) { - let mut updated = false; + pub fn call_destory(&self, handle: Handle) { let mut protect = self.protect.lock(); - for id in ids { - updated |= protect.indexes.remove(&id).is_some(); - } - if updated { + if protect.indexes.remove(&handle).is_some() { protect.maintain(&self.view); } } - pub fn call_stat(&self, id: Id) -> Result { + pub fn call_stat(&self, handle: Handle) -> Result { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; - Ok(index.stat()) + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; + index.stat() } - pub fn get_instance(&self, id: Id) -> Result { + pub fn get_instance(&self, handle: Handle) -> Result { let view = self.view.load_full(); - let index = view.indexes.get(&id).ok_or(FriendlyError::UnknownIndex)?; + let index = view + .indexes + .get(&handle) + .ok_or(FriendlyError::UnknownIndex)?; Ok(index.clone()) } } struct WorkerView { - indexes: HashMap, + indexes: HashMap, } struct WorkerProtect { startup: FileAtomic, - indexes: HashMap, + indexes: HashMap, } impl WorkerProtect { fn maintain(&mut self, swap: &ArcSwap) { - let indexes = self - .indexes - .iter() - .map(|(&k, v)| (k, v.options().clone())) - .collect(); + let indexes = self.indexes.keys().copied().collect(); self.startup.set(WorkerStartup { indexes }); swap.swap(Arc::new(WorkerView { indexes: self.indexes.clone(), @@ -167,17 +175,15 @@ impl WorkerProtect { } } -#[serde_with::serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] struct WorkerStartup { - #[serde_as(as = "HashMap")] - indexes: HashMap, + indexes: HashSet, } impl WorkerStartup { pub fn new() -> Self { Self { - indexes: HashMap::new(), + indexes: HashSet::new(), } } } diff --git a/docs/indexing.md b/docs/indexing.md index c3641f7..85e6936 100644 --- a/docs/indexing.md +++ b/docs/indexing.md @@ -108,19 +108,20 @@ Options for table `product`. We also provide a view `pg_vector_index_info` to monitor the progress of indexing. -| Column | Type | Description | -| ------------ | ------ | --------------------------------------------- | -| tablerelid | oid | The oid of the table. | -| indexrelid | oid | The oid of the index. | -| tablename | name | The name of the table. | -| indexname | name | The name of the index. | -| idx_indexing | bool | Whether the background thread is indexing. | -| idx_tuples | int8 | The number of tuples. | -| idx_sealed | int8[] | The number of tuples in each sealed segment. | -| idx_growing | int8[] | The number of tuples in each growing segment. | -| idx_write | int8 | The number of tuples in write buffer. | -| idx_size | int8 | The byte size for all the segments. | -| idx_config | text | The configuration of the index. | +| Column | Type | Description | +| ------------ | ------ | ----------------------------------------------------------------------------------- | +| tablerelid | oid | The oid of the table. | +| indexrelid | oid | The oid of the index. | +| tablename | name | The name of the table. | +| indexname | name | The name of the index. | +| idx_status | text | Its value is `NORMAL` or `UPGRADE`. Whether this index is normal or needs upgrade. | +| idx_indexing | bool | Not null if `idx_status` is `NORMAL`. Whether the background thread is indexing. | +| idx_tuples | int8 | Not null if `idx_status` is `NORMAL`. The number of tuples. | +| idx_sealed | int8[] | Not null if `idx_status` is `NORMAL`. The number of tuples in each sealed segment. | +| idx_growing | int8[] | Not null if `idx_status` is `NORMAL`. The number of tuples in each growing segment. | +| idx_write | int8 | Not null if `idx_status` is `NORMAL`. The number of tuples in write buffer. | +| idx_size | int8 | Not null if `idx_status` is `NORMAL`. The byte size for all the segments. | +| idx_config | text | Not null if `idx_status` is `NORMAL`. The configuration of the index. | ## Examples diff --git a/docs/installation.md b/docs/installation.md index c71f8a3..b38729f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -77,7 +77,7 @@ cd pgvecto.rs Install cargo-pgrx. ```sh -cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) +cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2) cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config ``` diff --git a/docs/upgrade.md b/docs/upgrade.md index be42023..cd94fb3 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1,6 +1,6 @@ # Upgrade -## `The extension is upgraded. However, the index files is outdated.` +## `The extension is upgraded so all index files are outdated.` You may see this error if you upgrade the extension. On this condition, you should follow these steps: @@ -49,3 +49,41 @@ REINDEX INDEX t_val_idx1; REINDEX INDEX t_val_idx2; REINDEX INDEX t_val_idx3; ``` + +## `The extension is upgraded so this index is outdated.` + +You may see this error if you upgrade the extension. On this condition, you should follow these steps: + +* Reindex. + +You can list all indexes that needed to be reindexed with this command: + +```sql +SELECT + I.oid AS indexrelid, + I.relname AS indexname +FROM pg_index X + JOIN pg_class I ON I.oid = X.indexrelid + JOIN pg_am A ON A.oid = I.relam +WHERE A.amname = 'vectors'; +``` + +If you get the result like this: + +``` + indexrelid | indexname +------------+------------ + 17988 | t_val_idx + 17989 | t_val_idx1 + 17990 | t_val_idx2 + 17991 | t_val_idx3 +``` + +You will reindex them with this SQL: + +```sql +REINDEX INDEX t_val_idx; +REINDEX INDEX t_val_idx1; +REINDEX INDEX t_val_idx2; +REINDEX INDEX t_val_idx3; +``` diff --git a/scripts/ci_setup.sh b/scripts/ci_setup.sh index f260500..b24b9ac 100755 --- a/scripts/ci_setup.sh +++ b/scripts/ci_setup.sh @@ -41,5 +41,5 @@ sudo chmod -R 777 `pg_config --sharedir`/extension curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash cargo binstall sqllogictest-bin -y --force -cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) --debug +cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2) --debug cargo pgrx init --pg$VERSION=$(which pg_config) diff --git a/scripts/envd.sh b/scripts/envd.sh index 8847d8d..49482a1 100755 --- a/scripts/envd.sh +++ b/scripts/envd.sh @@ -9,5 +9,5 @@ sudo chmod 777 /usr/share/postgresql/15/extension/ sudo chmod 777 /usr/lib/postgresql/15/lib/ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none -cargo install cargo-pgrx --git https://github.com/tensorchord/pgrx.git --rev $(cat Cargo.toml | grep "pgrx =" | awk -F'rev = "' '{print $2}' | cut -d'"' -f1) +cargo install cargo-pgrx@$(grep 'pgrx = {' Cargo.toml | cut -d '"' -f 2) cargo pgrx init --pg15=/usr/lib/postgresql/15/bin/pg_config diff --git a/src/bgworker/normal.rs b/src/bgworker/normal.rs index 8293190..30165fb 100644 --- a/src/bgworker/normal.rs +++ b/src/bgworker/normal.rs @@ -60,58 +60,62 @@ fn session(worker: Arc, mut handler: RpcHandler) -> Result<(), IpcError> use crate::ipc::server::RpcHandle; loop { match handler.handle()? { - RpcHandle::Create { id, options, x } => { - worker.call_create(id, options); + RpcHandle::Create { handle, options, x } => { + worker.call_create(handle, options); handler = x.leave()?; } - RpcHandle::Insert { id, insert, x } => match worker.call_insert(id, insert) { - Ok(()) => handler = x.leave()?, - Err(res) => x.reset(res)?, - }, - RpcHandle::Delete { id, mut x } => match worker.call_delete(id, |p| x.next(p).unwrap()) - { + RpcHandle::Insert { handle, insert, x } => match worker.call_insert(handle, insert) { Ok(()) => handler = x.leave()?, Err(res) => x.reset(res)?, }, + RpcHandle::Delete { handle, mut x } => { + match worker.call_delete(handle, |p| x.next(p).unwrap()) { + Ok(()) => handler = x.leave()?, + Err(res) => x.reset(res)?, + } + } RpcHandle::Search { - id, + handle, search, prefilter: true, gucs, mut x, - } => match worker.call_search(id, search, gucs, |p| x.check(p).unwrap()) { + } => match worker.call_search(handle, search, gucs, |p| x.check(p).unwrap()) { Ok(res) => handler = x.leave(res)?, Err(e) => x.reset(e)?, }, RpcHandle::Search { - id, + handle, search, prefilter: false, gucs, x, - } => match worker.call_search(id, search, gucs, |_| true) { + } => match worker.call_search(handle, search, gucs, |_| true) { Ok(res) => handler = x.leave(res)?, Err(e) => x.reset(e)?, }, - RpcHandle::Flush { id, x } => match worker.call_flush(id) { + RpcHandle::Flush { handle, x } => match worker.call_flush(handle) { Ok(()) => handler = x.leave()?, Err(e) => x.reset(e)?, }, - RpcHandle::Destory { ids, x } => { - worker.call_destory(ids); + RpcHandle::Destory { handle, x } => { + worker.call_destory(handle); handler = x.leave()?; } - RpcHandle::Stat { id, x } => match worker.call_stat(id) { + RpcHandle::Stat { handle, x } => match worker.call_stat(handle) { Ok(res) => handler = x.leave(res)?, Err(e) => x.reset(e)?, }, - RpcHandle::Vbase { id, vbase, x } => { + RpcHandle::Vbase { handle, vbase, x } => { use crate::ipc::server::VbaseHandle::*; - let instance = match worker.get_instance(id) { + let instance = match worker.get_instance(handle) { + Ok(x) => x, + Err(e) => x.reset(e)?, + }; + let view = match instance.view() { Ok(x) => x, Err(e) => x.reset(e)?, }; - let view = instance.view(); let mut it = match view.vbase(vbase.0, vbase.1) { Ok(x) => x, Err(e) => x.reset(e)?, diff --git a/src/index/am.rs b/src/index/am.rs index 519bfc3..c744a5c 100644 --- a/src/index/am.rs +++ b/src/index/am.rs @@ -205,7 +205,7 @@ pub unsafe extern "C" fn aminsert( _index_info: *mut pgrx::pg_sys::IndexInfo, ) -> bool { let oid = (*index_relation).rd_node.relNode; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); let vector = from_datum(*values.add(0)); am_update::update_insert(id, vector, *heap_tid); true @@ -227,7 +227,7 @@ pub unsafe extern "C" fn aminsert( let oid = (*index_relation).rd_node.relNode; #[cfg(feature = "pg16")] let oid = (*index_relation).rd_locator.relNumber; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); let vector = from_datum(*values.add(0)); am_update::update_insert(id, vector, *heap_tid); true @@ -284,7 +284,7 @@ pub unsafe extern "C" fn ambulkdelete( let oid = (*(*info).index).rd_node.relNode; #[cfg(feature = "pg16")] let oid = (*(*info).index).rd_locator.relNumber; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); if let Some(callback) = callback { am_update::update_delete(id, |pointer| { callback( diff --git a/src/index/am_build.rs b/src/index/am_build.rs index 603d254..c7b1007 100644 --- a/src/index/am_build.rs +++ b/src/index/am_build.rs @@ -21,7 +21,7 @@ pub unsafe fn build( let oid = (*index).rd_node.relNode; #[cfg(feature = "pg16")] let oid = (*index).rd_locator.relNumber; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); flush_if_commit(id); let options = options(index); let mut rpc = crate::ipc::client::borrow_mut(); @@ -55,7 +55,7 @@ unsafe extern "C" fn callback( ) { let ctid = &(*htup).t_self; let oid = (*index_relation).rd_node.relNode; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); let state = &mut *(state as *mut Builder); let vector = from_datum(*values.add(0)); let data = (vector, Pointer::from_sys(*ctid)); @@ -78,7 +78,7 @@ unsafe extern "C" fn callback( let oid = (*index_relation).rd_node.relNode; #[cfg(feature = "pg16")] let oid = (*index_relation).rd_locator.relNumber; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); let state = &mut *(state as *mut Builder); let vector = from_datum(*values.add(0)); let data = (vector, Pointer::from_sys(*ctid)); diff --git a/src/index/am_scan.rs b/src/index/am_scan.rs index 806ed05..1447785 100644 --- a/src/index/am_scan.rs +++ b/src/index/am_scan.rs @@ -95,7 +95,7 @@ pub unsafe fn next_scan(scan: pgrx::pg_sys::IndexScanDesc) -> bool { let oid = (*(*scan).indexRelation).rd_node.relNode; #[cfg(feature = "pg16")] let oid = (*(*scan).indexRelation).rd_locator.relNumber; - let id = Id::from_sys(oid); + let id = Handle::from_sys(oid); let mut rpc = crate::ipc::client::borrow_mut(); diff --git a/src/index/am_setup.rs b/src/index/am_setup.rs index dda296d..c55f73d 100644 --- a/src/index/am_setup.rs +++ b/src/index/am_setup.rs @@ -9,7 +9,7 @@ use std::ffi::CStr; use validator::Validate; pub fn helper_offset() -> usize { - std::mem::offset_of!(Helper, offset) + bytemuck::offset_of!(Helper, offset) } pub fn helper_size() -> usize { @@ -101,7 +101,7 @@ pub unsafe fn options(index_relation: pgrx::pg_sys::Relation) -> IndexOptions { options } -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, Default)] #[repr(C)] struct Helper { pub vl_len_: i32, diff --git a/src/index/am_update.rs b/src/index/am_update.rs index e81205f..f14c201 100644 --- a/src/index/am_update.rs +++ b/src/index/am_update.rs @@ -2,14 +2,14 @@ use crate::index::hook_transaction::flush_if_commit; use crate::prelude::*; use service::prelude::*; -pub fn update_insert(id: Id, vector: DynamicVector, tid: pgrx::pg_sys::ItemPointerData) { - flush_if_commit(id); +pub fn update_insert(handle: Handle, vector: DynamicVector, tid: pgrx::pg_sys::ItemPointerData) { + flush_if_commit(handle); let p = Pointer::from_sys(tid); let mut rpc = crate::ipc::client::borrow_mut(); - rpc.insert(id, (vector, p)); + rpc.insert(handle, (vector, p)); } -pub fn update_delete(id: Id, hook: impl Fn(Pointer) -> bool) { +pub fn update_delete(handle: Handle, hook: impl Fn(Pointer) -> bool) { struct Delete { hook: H, } @@ -25,7 +25,7 @@ pub fn update_delete(id: Id, hook: impl Fn(Pointer) -> bool) { let client_delete = Delete { hook }; - flush_if_commit(id); + flush_if_commit(handle); let mut rpc = crate::ipc::client::borrow_mut(); - rpc.delete(id, client_delete); + rpc.delete(handle, client_delete); } diff --git a/src/index/hook_transaction.rs b/src/index/hook_transaction.rs index c9017a3..f25c385 100644 --- a/src/index/hook_transaction.rs +++ b/src/index/hook_transaction.rs @@ -2,7 +2,7 @@ use crate::utils::cells::PgRefCell; use service::prelude::*; use std::collections::BTreeSet; -static FLUSH_IF_COMMIT: PgRefCell> = unsafe { PgRefCell::new(BTreeSet::new()) }; +static FLUSH_IF_COMMIT: PgRefCell> = unsafe { PgRefCell::new(BTreeSet::new()) }; pub fn aborting() { *FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new(); @@ -21,6 +21,6 @@ pub fn committing() { *FLUSH_IF_COMMIT.borrow_mut() = BTreeSet::new(); } -pub fn flush_if_commit(id: Id) { - FLUSH_IF_COMMIT.borrow_mut().insert(id); +pub fn flush_if_commit(handle: Handle) { + FLUSH_IF_COMMIT.borrow_mut().insert(handle); } diff --git a/src/index/hooks.rs b/src/index/hooks.rs index a3fda3a..bcd9085 100644 --- a/src/index/hooks.rs +++ b/src/index/hooks.rs @@ -42,12 +42,14 @@ unsafe fn xact_delete() { let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _); if n > 0 { let nodes = std::slice::from_raw_parts(ptr, n as usize); - let ids = nodes + let handles = nodes .iter() - .map(|node| Id::from_sys(node.relNode)) + .map(|node| Handle::from_sys(node.relNode)) .collect::>(); let mut rpc = crate::ipc::client::borrow_mut(); - rpc.destory(ids); + for handle in handles { + rpc.destory(handle); + } } } @@ -57,11 +59,13 @@ unsafe fn xact_delete() { let n = pgrx::pg_sys::smgrGetPendingDeletes(true, &mut ptr as *mut _); if n > 0 { let nodes = std::slice::from_raw_parts(ptr, n as usize); - let ids = nodes + let handles = nodes .iter() - .map(|node| Id::from_sys(node.relNumber)) + .map(|node| Handle::from_sys(node.relNumber)) .collect::>(); let mut rpc = crate::ipc::client::borrow_mut(); - rpc.destory(ids); + for handle in handles { + rpc.destory(handle); + } } } diff --git a/src/index/views.rs b/src/index/views.rs index 0465d88..995816d 100644 --- a/src/index/views.rs +++ b/src/index/views.rs @@ -4,6 +4,7 @@ use service::prelude::*; pgrx::extension_sql!( "\ CREATE TYPE VectorIndexStat AS ( + idx_status TEXT, idx_indexing BOOL, idx_tuples BIGINT, idx_sealed BIGINT[], @@ -17,36 +18,63 @@ CREATE TYPE VectorIndexStat AS ( #[pgrx::pg_extern(volatile, strict)] fn vector_stat(oid: pgrx::pg_sys::Oid) -> pgrx::composite_type!("VectorIndexStat") { - let id = Id::from_sys(oid); + use service::index::IndexStat; + let id = Handle::from_sys(oid); let mut res = pgrx::prelude::PgHeapTuple::new_composite_type("VectorIndexStat").unwrap(); let mut rpc = crate::ipc::client::borrow_mut(); let stat = rpc.stat(id); - res.set_by_name("idx_indexing", stat.indexing).unwrap(); - res.set_by_name("idx_tuples", { - let mut tuples = 0; - tuples += stat.sealed.iter().map(|x| *x as i64).sum::(); - tuples += stat.growing.iter().map(|x| *x as i64).sum::(); - tuples += stat.write as i64; - tuples - }) - .unwrap(); - res.set_by_name("idx_sealed", { - let sealed = stat.sealed; - sealed.into_iter().map(|x| x as i64).collect::>() - }) - .unwrap(); - res.set_by_name("idx_growing", { - let growing = stat.growing; - growing.into_iter().map(|x| x as i64).collect::>() - }) - .unwrap(); - res.set_by_name("idx_write", stat.write as i64).unwrap(); - res.set_by_name( - "idx_size", - stat.sizes.iter().map(|x| x.size as i64).sum::(), - ) - .unwrap(); - res.set_by_name("idx_options", serde_json::to_string(&stat.options)) - .unwrap(); - res + match stat { + IndexStat::Normal { + indexing, + options, + segments, + } => { + res.set_by_name("idx_status", "NORMAL").unwrap(); + res.set_by_name("idx_indexing", indexing).unwrap(); + res.set_by_name( + "idx_tuples", + segments.iter().map(|x| x.length as i64).sum::(), + ) + .unwrap(); + res.set_by_name( + "idx_sealed", + segments + .iter() + .filter(|x| x.typ == "sealed") + .map(|x| x.length as i64) + .collect::>(), + ) + .unwrap(); + res.set_by_name( + "idx_growing", + segments + .iter() + .filter(|x| x.typ == "growing") + .map(|x| x.length as i64) + .collect::>(), + ) + .unwrap(); + res.set_by_name( + "idx_write", + segments + .iter() + .filter(|x| x.typ == "write") + .map(|x| x.length as i64) + .sum::(), + ) + .unwrap(); + res.set_by_name( + "idx_size", + segments.iter().map(|x| x.size as i64).sum::(), + ) + .unwrap(); + res.set_by_name("idx_options", serde_json::to_string(&options)) + .unwrap(); + res + } + IndexStat::Upgrade => { + res.set_by_name("idx_status", "UPGRADE").unwrap(); + res + } + } } diff --git a/src/ipc/client/mod.rs b/src/ipc/client/mod.rs index 8147a7c..04c4d7a 100644 --- a/src/ipc/client/mod.rs +++ b/src/ipc/client/mod.rs @@ -54,21 +54,21 @@ impl Rpc { } impl ClientGuard { - pub fn create(&mut self, id: Id, options: IndexOptions) { - let packet = RpcPacket::Create { id, options }; + pub fn create(&mut self, handle: Handle, options: IndexOptions) { + let packet = RpcPacket::Create { handle, options }; self.socket.send(packet).friendly(); let create::CreatePacket::Leave {} = self.socket.recv().friendly(); } pub fn search( &mut self, - id: Id, + handle: Handle, search: (DynamicVector, usize), prefilter: bool, gucs: SearchGucs, mut t: impl Search, ) -> Vec { let packet = RpcPacket::Search { - id, + handle, search, prefilter, gucs, @@ -87,8 +87,8 @@ impl ClientGuard { } } } - pub fn delete(&mut self, id: Id, mut t: impl Delete) { - let packet = RpcPacket::Delete { id }; + pub fn delete(&mut self, handle: Handle, mut t: impl Delete) { + let packet = RpcPacket::Delete { handle }; self.socket.send(packet).friendly(); loop { match self.socket.recv().friendly() { @@ -103,29 +103,29 @@ impl ClientGuard { } } } - pub fn insert(&mut self, id: Id, insert: (DynamicVector, Pointer)) { - let packet = RpcPacket::Insert { id, insert }; + pub fn insert(&mut self, handle: Handle, insert: (DynamicVector, Pointer)) { + let packet = RpcPacket::Insert { handle, insert }; self.socket.send(packet).friendly(); let insert::InsertPacket::Leave {} = self.socket.recv().friendly(); } - pub fn flush(&mut self, id: Id) { - let packet = RpcPacket::Flush { id }; + pub fn flush(&mut self, handle: Handle) { + let packet = RpcPacket::Flush { handle }; self.socket.send(packet).friendly(); let flush::FlushPacket::Leave {} = self.socket.recv().friendly(); } - pub fn destory(&mut self, ids: Vec) { - let packet = RpcPacket::Destory { ids }; + pub fn destory(&mut self, handle: Handle) { + let packet = RpcPacket::Destory { handle }; self.socket.send(packet).friendly(); let destory::DestoryPacket::Leave {} = self.socket.recv().friendly(); } - pub fn stat(&mut self, id: Id) -> IndexStat { - let packet = RpcPacket::Stat { id }; + pub fn stat(&mut self, handle: Handle) -> IndexStat { + let packet = RpcPacket::Stat { handle }; self.socket.send(packet).friendly(); let stat::StatPacket::Leave { result } = self.socket.recv().friendly(); result } - pub fn vbase(mut self, id: Id, vbase: (DynamicVector, usize)) -> ClientGuard { - let packet = RpcPacket::Vbase { id, vbase }; + pub fn vbase(mut self, handle: Handle, vbase: (DynamicVector, usize)) -> ClientGuard { + let packet = RpcPacket::Vbase { handle, vbase }; self.socket.send(packet).friendly(); let vbase::VbaseErrorPacket {} = self.socket.recv().friendly(); ClientGuard::map(self) diff --git a/src/ipc/packet/mod.rs b/src/ipc/packet/mod.rs index 055f7da..8797a8d 100644 --- a/src/ipc/packet/mod.rs +++ b/src/ipc/packet/mod.rs @@ -15,33 +15,33 @@ use service::prelude::*; #[derive(Debug, Serialize, Deserialize)] pub enum RpcPacket { Create { - id: Id, + handle: Handle, options: IndexOptions, }, Delete { - id: Id, + handle: Handle, }, Destory { - ids: Vec, + handle: Handle, }, Flush { - id: Id, + handle: Handle, }, Insert { - id: Id, + handle: Handle, insert: (DynamicVector, Pointer), }, Search { - id: Id, + handle: Handle, search: (DynamicVector, usize), prefilter: bool, gucs: SearchGucs, }, Stat { - id: Id, + handle: Handle, }, Vbase { - id: Id, + handle: Handle, vbase: (DynamicVector, usize), }, } diff --git a/src/ipc/server/mod.rs b/src/ipc/server/mod.rs index 115c007..0d12780 100644 --- a/src/ipc/server/mod.rs +++ b/src/ipc/server/mod.rs @@ -16,33 +16,33 @@ impl RpcHandler { } pub fn handle(mut self) -> Result { Ok(match self.socket.recv::()? { - RpcPacket::Create { id, options } => RpcHandle::Create { - id, + RpcPacket::Create { handle, options } => RpcHandle::Create { + handle, options, x: Create { socket: self.socket, }, }, - RpcPacket::Insert { id, insert } => RpcHandle::Insert { - id, + RpcPacket::Insert { handle, insert } => RpcHandle::Insert { + handle, insert, x: Insert { socket: self.socket, }, }, - RpcPacket::Delete { id } => RpcHandle::Delete { - id, + RpcPacket::Delete { handle } => RpcHandle::Delete { + handle, x: Delete { socket: self.socket, }, }, RpcPacket::Search { - id, + handle, search, prefilter, gucs, } => RpcHandle::Search { - id, + handle, search, prefilter, gucs, @@ -50,26 +50,26 @@ impl RpcHandler { socket: self.socket, }, }, - RpcPacket::Flush { id } => RpcHandle::Flush { - id, + RpcPacket::Flush { handle } => RpcHandle::Flush { + handle, x: Flush { socket: self.socket, }, }, - RpcPacket::Destory { ids } => RpcHandle::Destory { - ids, + RpcPacket::Destory { handle } => RpcHandle::Destory { + handle, x: Destory { socket: self.socket, }, }, - RpcPacket::Stat { id } => RpcHandle::Stat { - id, + RpcPacket::Stat { handle } => RpcHandle::Stat { + handle, x: Stat { socket: self.socket, }, }, - RpcPacket::Vbase { id, vbase } => RpcHandle::Vbase { - id, + RpcPacket::Vbase { handle, vbase } => RpcHandle::Vbase { + handle, vbase, x: Vbase { socket: self.socket, @@ -81,40 +81,40 @@ impl RpcHandler { pub enum RpcHandle { Create { - id: Id, + handle: Handle, options: IndexOptions, x: Create, }, Search { - id: Id, + handle: Handle, search: (DynamicVector, usize), prefilter: bool, gucs: SearchGucs, x: Search, }, Insert { - id: Id, + handle: Handle, insert: (DynamicVector, Pointer), x: Insert, }, Delete { - id: Id, + handle: Handle, x: Delete, }, Flush { - id: Id, + handle: Handle, x: Flush, }, Destory { - ids: Vec, + handle: Handle, x: Destory, }, Stat { - id: Id, + handle: Handle, x: Stat, }, Vbase { - id: Id, + handle: Handle, vbase: (DynamicVector, usize), x: Vbase, }, diff --git a/src/lib.rs b/src/lib.rs index 0f3faa1..1491097 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,6 @@ //! Postgres vector extension. //! //! Provides an easy-to-use extension for vector similarity search. -#![feature(offset_of)] #![feature(never_type)] mod bgworker; diff --git a/src/prelude.rs b/src/prelude.rs index 7a18d29..0c56d5d 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -4,7 +4,7 @@ pub trait FromSys { fn from_sys(sys: T) -> Self; } -impl FromSys for Id { +impl FromSys for Handle { fn from_sys(sys: pgrx::pg_sys::Oid) -> Self { Self { newtype: sys.as_u32(),