feat: deny unknown field in options (#184)

* fix: deny unknown options

Signed-off-by: usamoi <usamoi@outlook.com>

* test: deny unknown options

Signed-off-by: usamoi <usamoi@outlook.com>

---------

Signed-off-by: usamoi <usamoi@outlook.com>

crates/service/src/algorithms/quantization/mod.rs

@@ -15,6 +15,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum QuantizationOptions {
     Trivial(TrivialQuantizationOptions),

crates/service/src/algorithms/quantization/product.rs

@@ -16,6 +16,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct ProductQuantizationOptions {
     #[serde(default = "ProductQuantizationOptions::default_sample")]
     pub sample: u32,
@@ -40,6 +41,7 @@ impl Default for ProductQuantizationOptions {
 
 #[repr(u16)]
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum ProductQuantizationOptionsRatio {
     X4 = 1,

crates/service/src/algorithms/quantization/scalar.rs

@@ -11,6 +11,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct ScalarQuantizationOptions {}
 
 impl Default for ScalarQuantizationOptions {

crates/service/src/algorithms/quantization/trivial.rs

@@ -9,6 +9,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct TrivialQuantizationOptions {}
 
 impl Default for TrivialQuantizationOptions {

crates/service/src/index/delete.rs

@@ -3,6 +3,7 @@ use crate::utils::file_wal::FileWal;
 use dashmap::mapref::entry::Entry;
 use dashmap::DashMap;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -79,7 +80,7 @@ impl Delete {
     }
 }
 
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 struct Log {
     key: Pointer,
 }

crates/service/src/index/indexing/flat.rs

@@ -10,6 +10,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct FlatIndexingOptions {
     #[serde(default)]
     #[validate]

crates/service/src/index/indexing/hnsw.rs

@@ -10,6 +10,7 @@ use std::{path::PathBuf, sync::Arc};
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct HnswIndexingOptions {
     #[serde(default = "HnswIndexingOptions::default_m")]
     #[validate(range(min = 4, max = 128))]

crates/service/src/index/indexing/ivf.rs

@@ -11,6 +11,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct IvfIndexingOptions {
     #[serde(default = "IvfIndexingOptions::default_least_iterations")]
     #[validate(range(min = 1, max = 1_000_000))]

crates/service/src/index/indexing/mod.rs

@@ -16,6 +16,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum IndexingOptions {
     Flat(FlatIndexingOptions),

crates/service/src/index/mod.rs

@@ -37,6 +37,7 @@ use validator::Validate;
 pub struct OutdatedError(#[from] pub Option<GrowingSegmentInsertError>);
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct VectorOptions {
     #[validate(range(min = 1, max = 65535))]
     #[serde(rename = "dimensions")]
@@ -48,6 +49,7 @@ pub struct VectorOptions {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct IndexOptions {
     #[validate]
     pub vector: VectorOptions,
@@ -467,7 +469,7 @@ impl<S: G> IndexView<S> {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct IndexStartup {
     sealeds: HashSet<Uuid>,
     growings: HashSet<Uuid>,

crates/service/src/index/optimizing/mod.rs

@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct OptimizingOptions {
     #[serde(default = "OptimizingOptions::default_sealing_secs")]
     #[validate(range(min = 0, max = 60))]

crates/service/src/index/segments/growing.rs

@@ -7,6 +7,7 @@ use crate::prelude::*;
 use crate::utils::dir_ops::sync_dir;
 use crate::utils::file_wal::FileWal;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use std::cell::UnsafeCell;
 use std::mem::MaybeUninit;
 use std::path::PathBuf;
@@ -199,7 +200,7 @@ impl<S: G> Drop for GrowingSegment<S> {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct Log<S: G> {
     vector: Vec<S::Scalar>,
     payload: Payload,

crates/service/src/index/segments/mod.rs

@@ -9,6 +9,7 @@ use validator::Validate;
 use validator::ValidationError;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 #[validate(schema(function = "Self::validate_0"))]
 pub struct SegmentsOptions {
     #[serde(default = "SegmentsOptions::default_max_growing_segment_size")]

crates/service/src/prelude/global/mod.rs

@@ -87,7 +87,7 @@ pub trait FloatCast: Sized {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum DynamicVector {
     F32(Vec<F32>),
     F16(Vec<F16>),

crates/service/src/utils/vec2.rs

@@ -1,7 +1,7 @@
 use crate::prelude::*;
 use std::ops::{Deref, DerefMut, Index, IndexMut};
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone)]
 pub struct Vec2<S: G> {
     dims: u16,
     v: Vec<S::Scalar>,

crates/service/src/worker/mod.rs

@@ -10,6 +10,7 @@ use crate::utils::dir_ops::sync_dir;
 use crate::utils::file_atomic::FileAtomic;
 use arc_swap::ArcSwap;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use serde_with::DisplayFromStr;
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -172,7 +173,7 @@ impl WorkerProtect {
 }
 
 #[serde_with::serde_as]
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct WorkerStartup {
     #[serde_as(as = "HashMap<DisplayFromStr, _>")]
     indexes: HashMap<Id, IndexOptions>,

src/index/am_setup.rs

@@ -1,5 +1,5 @@
 use crate::datatype::typmod::Typmod;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use service::index::indexing::IndexingOptions;
 use service::index::optimizing::OptimizingOptions;
 use service::index::segments::SegmentsOptions;
@@ -118,7 +118,8 @@ unsafe fn get_parsed_from_varlena(helper: *const pgrx::pg_sys::varlena) -> Parse
     toml::from_str::<Parsed>(cstr.to_str().unwrap()).unwrap()
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Deserialize, Default)]
+#[serde(deny_unknown_fields)]
 struct Parsed {
     #[serde(default)]
     segment: SegmentsOptions,

src/ipc/packet/create.rs

@@ -1,5 +1,4 @@
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
-use serde::Serialize;
 
 #[derive(Debug, Serialize, Deserialize)]
 pub enum CreatePacket {

tests/sqllogictest/index.slt

@@ -0,0 +1,12 @@
+statement ok
+DROP TABLE IF EXISTS t;
+
+statement ok
+CREATE TABLE t (val vector(3));
+
+statement ok
+INSERT INTO t (val) SELECT ARRAY[random(), random(), random()]::real[] FROM generate_series(1, 1000);
+
+statement error unknown
+CREATE INDEX ON t USING vectors (val vector_l2_ops)
+WITH (options = "unknown_field = 1");