diff --git a/crates/service/src/algorithms/quantization/mod.rs b/crates/service/src/algorithms/quantization/mod.rs index 5e23c28..d5a01f8 100644 --- a/crates/service/src/algorithms/quantization/mod.rs +++ b/crates/service/src/algorithms/quantization/mod.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub enum QuantizationOptions { Trivial(TrivialQuantizationOptions), diff --git a/crates/service/src/algorithms/quantization/product.rs b/crates/service/src/algorithms/quantization/product.rs index 9685563..bbc1905 100644 --- a/crates/service/src/algorithms/quantization/product.rs +++ b/crates/service/src/algorithms/quantization/product.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct ProductQuantizationOptions { #[serde(default = "ProductQuantizationOptions::default_sample")] pub sample: u32, @@ -40,6 +41,7 @@ impl Default for ProductQuantizationOptions { #[repr(u16)] #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub enum ProductQuantizationOptionsRatio { X4 = 1, diff --git a/crates/service/src/algorithms/quantization/scalar.rs b/crates/service/src/algorithms/quantization/scalar.rs index f6effab..3758fcd 100644 --- a/crates/service/src/algorithms/quantization/scalar.rs +++ b/crates/service/src/algorithms/quantization/scalar.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct ScalarQuantizationOptions {} impl Default for ScalarQuantizationOptions { diff --git a/crates/service/src/algorithms/quantization/trivial.rs b/crates/service/src/algorithms/quantization/trivial.rs index 416e398..6ae3330 100644 --- a/crates/service/src/algorithms/quantization/trivial.rs +++ b/crates/service/src/algorithms/quantization/trivial.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct TrivialQuantizationOptions {} impl Default for TrivialQuantizationOptions { diff --git a/crates/service/src/index/delete.rs b/crates/service/src/index/delete.rs index 2fbd8d1..6c14ba4 100644 --- a/crates/service/src/index/delete.rs +++ b/crates/service/src/index/delete.rs @@ -3,6 +3,7 @@ use crate::utils::file_wal::FileWal; use dashmap::mapref::entry::Entry; use dashmap::DashMap; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::sync::Arc; @@ -79,7 +80,7 @@ impl Delete { } } -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct Log { key: Pointer, } diff --git a/crates/service/src/index/indexing/flat.rs b/crates/service/src/index/indexing/flat.rs index d288216..d8032bf 100644 --- a/crates/service/src/index/indexing/flat.rs +++ b/crates/service/src/index/indexing/flat.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct FlatIndexingOptions { #[serde(default)] #[validate] diff --git a/crates/service/src/index/indexing/hnsw.rs b/crates/service/src/index/indexing/hnsw.rs index d1bc8ee..fe30da5 100644 --- a/crates/service/src/index/indexing/hnsw.rs +++ b/crates/service/src/index/indexing/hnsw.rs @@ -10,6 +10,7 @@ use std::{path::PathBuf, sync::Arc}; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct HnswIndexingOptions { #[serde(default = "HnswIndexingOptions::default_m")] #[validate(range(min = 4, max = 128))] diff --git a/crates/service/src/index/indexing/ivf.rs b/crates/service/src/index/indexing/ivf.rs index 9d07486..f658cab 100644 --- a/crates/service/src/index/indexing/ivf.rs +++ b/crates/service/src/index/indexing/ivf.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct IvfIndexingOptions { #[serde(default = "IvfIndexingOptions::default_least_iterations")] #[validate(range(min = 1, max = 1_000_000))] diff --git a/crates/service/src/index/indexing/mod.rs b/crates/service/src/index/indexing/mod.rs index ec88392..d222a84 100644 --- a/crates/service/src/index/indexing/mod.rs +++ b/crates/service/src/index/indexing/mod.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub enum IndexingOptions { Flat(FlatIndexingOptions), diff --git a/crates/service/src/index/mod.rs b/crates/service/src/index/mod.rs index 2d30902..f9b049e 100644 --- a/crates/service/src/index/mod.rs +++ b/crates/service/src/index/mod.rs @@ -37,6 +37,7 @@ use validator::Validate; pub struct OutdatedError(#[from] pub Option); #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct VectorOptions { #[validate(range(min = 1, max = 65535))] #[serde(rename = "dimensions")] @@ -48,6 +49,7 @@ pub struct VectorOptions { } #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct IndexOptions { #[validate] pub vector: VectorOptions, @@ -467,7 +469,7 @@ impl IndexView { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct IndexStartup { sealeds: HashSet, growings: HashSet, diff --git a/crates/service/src/index/optimizing/mod.rs b/crates/service/src/index/optimizing/mod.rs index d0c4b2a..92c4afc 100644 --- a/crates/service/src/index/optimizing/mod.rs +++ b/crates/service/src/index/optimizing/mod.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use validator::Validate; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct OptimizingOptions { #[serde(default = "OptimizingOptions::default_sealing_secs")] #[validate(range(min = 0, max = 60))] diff --git a/crates/service/src/index/segments/growing.rs b/crates/service/src/index/segments/growing.rs index cd393b4..0132446 100644 --- a/crates/service/src/index/segments/growing.rs +++ b/crates/service/src/index/segments/growing.rs @@ -7,6 +7,7 @@ use crate::prelude::*; use crate::utils::dir_ops::sync_dir; use crate::utils::file_wal::FileWal; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use std::cell::UnsafeCell; use std::mem::MaybeUninit; use std::path::PathBuf; @@ -199,7 +200,7 @@ impl Drop for GrowingSegment { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct Log { vector: Vec, payload: Payload, diff --git a/crates/service/src/index/segments/mod.rs b/crates/service/src/index/segments/mod.rs index b09c1f9..85822b9 100644 --- a/crates/service/src/index/segments/mod.rs +++ b/crates/service/src/index/segments/mod.rs @@ -9,6 +9,7 @@ use validator::Validate; use validator::ValidationError; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] +#[serde(deny_unknown_fields)] #[validate(schema(function = "Self::validate_0"))] pub struct SegmentsOptions { #[serde(default = "SegmentsOptions::default_max_growing_segment_size")] diff --git a/crates/service/src/prelude/global/mod.rs b/crates/service/src/prelude/global/mod.rs index 2eedaf9..3f5894d 100644 --- a/crates/service/src/prelude/global/mod.rs +++ b/crates/service/src/prelude/global/mod.rs @@ -87,7 +87,7 @@ pub trait FloatCast: Sized { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum DynamicVector { F32(Vec), F16(Vec), diff --git a/crates/service/src/utils/vec2.rs b/crates/service/src/utils/vec2.rs index 2640f51..318383b 100644 --- a/crates/service/src/utils/vec2.rs +++ b/crates/service/src/utils/vec2.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use std::ops::{Deref, DerefMut, Index, IndexMut}; -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone)] pub struct Vec2 { dims: u16, v: Vec, diff --git a/crates/service/src/worker/mod.rs b/crates/service/src/worker/mod.rs index 3e7c974..908a284 100644 --- a/crates/service/src/worker/mod.rs +++ b/crates/service/src/worker/mod.rs @@ -10,6 +10,7 @@ use crate::utils::dir_ops::sync_dir; use crate::utils::file_atomic::FileAtomic; use arc_swap::ArcSwap; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; use std::collections::HashMap; use std::path::PathBuf; @@ -172,7 +173,7 @@ impl WorkerProtect { } #[serde_with::serde_as] -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct WorkerStartup { #[serde_as(as = "HashMap")] indexes: HashMap, diff --git a/src/index/am_setup.rs b/src/index/am_setup.rs index fe452f0..dda296d 100644 --- a/src/index/am_setup.rs +++ b/src/index/am_setup.rs @@ -1,5 +1,5 @@ use crate::datatype::typmod::Typmod; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use service::index::indexing::IndexingOptions; use service::index::optimizing::OptimizingOptions; use service::index::segments::SegmentsOptions; @@ -118,7 +118,8 @@ unsafe fn get_parsed_from_varlena(helper: *const pgrx::pg_sys::varlena) -> Parse toml::from_str::(cstr.to_str().unwrap()).unwrap() } -#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Parsed { #[serde(default)] segment: SegmentsOptions, diff --git a/src/ipc/packet/create.rs b/src/ipc/packet/create.rs index 14a1996..edb1afd 100644 --- a/src/ipc/packet/create.rs +++ b/src/ipc/packet/create.rs @@ -1,5 +1,4 @@ -use serde::Deserialize; -use serde::Serialize; +use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub enum CreatePacket { diff --git a/tests/sqllogictest/index.slt b/tests/sqllogictest/index.slt new file mode 100644 index 0000000..c04f46c --- /dev/null +++ b/tests/sqllogictest/index.slt @@ -0,0 +1,12 @@ +statement ok +DROP TABLE IF EXISTS t; + +statement ok +CREATE TABLE t (val vector(3)); + +statement ok +INSERT INTO t (val) SELECT ARRAY[random(), random(), random()]::real[] FROM generate_series(1, 1000); + +statement error unknown +CREATE INDEX ON t USING vectors (val vector_l2_ops) +WITH (options = "unknown_field = 1");