refactor: add crate "base" (#367)

Signed-off-by: usamoi <usamoi@outlook.com>

Cargo.lock

@@ -143,6 +143,36 @@ dependencies = [
  "rustc-demangle",
 ]
 
+[[package]]
+name = "base"
+version = "0.0.0"
+dependencies = [
+ "arc-swap",
+ "bincode",
+ "bytemuck",
+ "byteorder",
+ "c",
+ "crc32fast",
+ "crossbeam",
+ "dashmap",
+ "detect",
+ "half 2.3.1",
+ "libc",
+ "log",
+ "memmap2",
+ "multiversion",
+ "num-traits",
+ "parking_lot",
+ "rand",
+ "rayon",
+ "rustix",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "uuid",
+ "validator",
+]
+
 [[package]]
 name = "base64"
 version = "0.21.7"
@@ -1739,6 +1769,7 @@ name = "service"
 version = "0.0.0"
 dependencies = [
  "arc-swap",
+ "base",
  "bincode",
  "bytemuck",
  "byteorder",
@@ -2197,6 +2228,7 @@ name = "vectors"
 version = "0.0.0"
 dependencies = [
  "arrayvec",
+ "base",
  "bincode",
  "bytemuck",
  "byteorder",

Cargo.toml

@@ -30,6 +30,7 @@ serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 validator.workspace = true
+base = { path = "crates/base" }
 detect = { path = "crates/detect" }
 send_fd = { path = "crates/send_fd" }
 service = { path = "crates/service" }

crates/base/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "base"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+bincode.workspace = true
+bytemuck.workspace = true
+byteorder.workspace = true
+half.workspace = true
+libc.workspace = true
+log.workspace = true
+memmap2.workspace = true
+num-traits.workspace = true
+rand.workspace = true
+rustix.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+uuid.workspace = true
+validator.workspace = true
+c = { path = "../c" }
+detect = { path = "../detect" }
+crc32fast = "1.4.0"
+crossbeam = "0.8.4"
+dashmap = "5.5.3"
+parking_lot = "0.12.1"
+rayon = "1.8.1"
+arc-swap = "1.6.0"
+multiversion = "0.7.3"
+
+[lints]
+clippy.derivable_impls = "allow"
+clippy.len_without_is_empty = "allow"
+clippy.needless_range_loop = "allow"
+clippy.too_many_arguments = "allow"
+rust.internal_features = "allow"
+rust.unsafe_op_in_unsafe_fn = "forbid"
+rust.unused_lifetimes = "warn"
+rust.unused_qualifications = "warn"

crates/base/src/lib.rs

@@ -0,0 +1,7 @@
+#![feature(core_intrinsics)]
+
+pub mod error;
+pub mod scalar;
+pub mod search;
+pub mod sys;
+pub mod vector;

crates/base/src/scalar/f16.rs

@@ -1,4 +1,4 @@
-use crate::prelude::global::FloatCast;
+use super::FloatCast;
 use half::f16;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;

crates/base/src/scalar/f32.rs

@@ -1,4 +1,4 @@
-use crate::prelude::global::FloatCast;
+use super::FloatCast;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::fmt::{Debug, Display};

crates/base/src/scalar/mod.rs

@@ -0,0 +1,16 @@
+mod f16;
+mod f32;
+
+pub use f16::F16;
+pub use f32::F32;
+
+pub trait FloatCast: Sized {
+    fn from_f32(x: f32) -> Self;
+    fn to_f32(self) -> f32;
+    fn from_f(x: F32) -> Self {
+        Self::from_f32(x.0)
+    }
+    fn to_f(self) -> F32 {
+        F32(Self::to_f32(self))
+    }
+}

crates/base/src/search.rs

@@ -1,4 +1,4 @@
-use crate::prelude::F32;
+use crate::scalar::F32;
 
 pub type Payload = u64;
 

crates/base/src/vector/mod.rs

@@ -0,0 +1,19 @@
+mod sparse_f32;
+
+pub use sparse_f32::{SparseF32, SparseF32Ref};
+
+pub trait Vector {
+    fn dims(&self) -> u16;
+}
+
+impl<T> Vector for Vec<T> {
+    fn dims(&self) -> u16 {
+        self.len().try_into().unwrap()
+    }
+}
+
+impl<'a, T> Vector for &'a [T] {
+    fn dims(&self) -> u16 {
+        self.len().try_into().unwrap()
+    }
+}

crates/base/src/vector/sparse_f32.rs

@@ -1,4 +1,6 @@
-use crate::prelude::*;
+use super::Vector;
+use crate::scalar::F32;
+use num_traits::Zero;
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/service/Cargo.toml

@@ -19,6 +19,7 @@ serde_json.workspace = true
 thiserror.workspace = true
 uuid.workspace = true
 validator.workspace = true
+base = { path = "../base" }
 c = { path = "../c" }
 detect = { path = "../detect" }
 crc32fast = "1.4.0"

crates/service/src/algorithms/clustering/elkan_k_means.rs

@@ -1,5 +1,6 @@
 use crate::prelude::*;
 use crate::utils::vec2::Vec2;
+use base::scalar::FloatCast;
 use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
 use std::ops::{Index, IndexMut};

crates/service/src/algorithms/quantization/scalar.rs

@@ -5,6 +5,7 @@ use crate::index::IndexOptions;
 use crate::prelude::*;
 use crate::utils::dir_ops::sync_dir;
 use crate::utils::mmap_array::MmapArray;
+use base::scalar::FloatCast;
 use serde::{Deserialize, Serialize};
 use std::path::Path;
 use std::sync::Arc;

crates/service/src/prelude/global/f16.rs

@@ -1,4 +1,5 @@
 use crate::prelude::*;
+use base::scalar::FloatCast;
 
 pub fn cosine(lhs: &[F16], rhs: &[F16]) -> F32 {
     #[inline(always)]

crates/service/src/prelude/global/f16_cos.rs

@@ -1,6 +1,6 @@
-use std::borrow::Cow;
-
 use crate::prelude::*;
+use base::scalar::FloatCast;
+use std::borrow::Cow;
 
 #[derive(Debug, Clone, Copy)]
 pub enum F16Cos {}

crates/service/src/prelude/global/f16_dot.rs

@@ -1,6 +1,6 @@
-use std::borrow::Cow;
-
 use crate::prelude::*;
+use base::scalar::FloatCast;
+use std::borrow::Cow;
 
 #[derive(Debug, Clone, Copy)]
 pub enum F16Dot {}

crates/service/src/prelude/global/f16_l2.rs

@@ -1,6 +1,6 @@
-use std::borrow::Cow;
-
 use crate::prelude::*;
+use base::scalar::FloatCast;
+use std::borrow::Cow;
 
 #[derive(Debug, Clone, Copy)]
 pub enum F16L2 {}

crates/service/src/prelude/global/f32_l2.rs

@@ -1,6 +1,5 @@
-use std::borrow::Cow;
-
 use crate::prelude::*;
+use std::borrow::Cow;
 
 #[derive(Debug, Clone, Copy)]
 pub enum F32L2 {}

crates/service/src/prelude/global/mod.rs

@@ -43,7 +43,7 @@ pub trait G: Copy + Debug + 'static {
         + Zero
         + num_traits::NumOps
         + num_traits::NumAssignOps
-        + FloatCast;
+        + base::scalar::FloatCast;
     type Storage: for<'a> Storage<VectorRef<'a> = Self::VectorRef<'a>>;
     type L2: for<'a> G<Scalar = Self::Scalar, VectorRef<'a> = &'a [Self::Scalar]>;
     type VectorOwned: Vector + Clone + Serialize + for<'a> Deserialize<'a>;
@@ -103,33 +103,6 @@ pub trait G: Copy + Debug + 'static {
     ) -> F32;
 }
 
-pub trait FloatCast: Sized {
-    fn from_f32(x: f32) -> Self;
-    fn to_f32(self) -> f32;
-    fn from_f(x: F32) -> Self {
-        Self::from_f32(x.0)
-    }
-    fn to_f(self) -> F32 {
-        F32(Self::to_f32(self))
-    }
-}
-
-pub trait Vector {
-    fn dims(&self) -> u16;
-}
-
-impl<T> Vector for Vec<T> {
-    fn dims(&self) -> u16 {
-        self.len().try_into().unwrap()
-    }
-}
-
-impl<'a, T> Vector for &'a [T] {
-    fn dims(&self) -> u16 {
-        self.len().try_into().unwrap()
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum DynamicVector {
     F32(Vec<F32>),

crates/service/src/prelude/mod.rs

@@ -1,15 +1,13 @@
-mod error;
 mod global;
-mod scalar;
-mod search;
 mod storage;
-mod sys;
 
-pub use self::error::*;
 pub use self::global::*;
-pub use self::scalar::{SparseF32, SparseF32Ref, F16, F32};
-pub use self::search::{Element, Filter, Payload};
 pub use self::storage::{DenseMmap, SparseMmap, Storage};
-pub use self::sys::{Handle, Pointer};
+
+pub use base::error::*;
+pub use base::scalar::{F16, F32};
+pub use base::search::{Element, Filter, Payload};
+pub use base::sys::{Handle, Pointer};
+pub use base::vector::{SparseF32, SparseF32Ref, Vector};
 
 pub use num_traits::{Float, Zero};

crates/service/src/prelude/scalar/mod.rs

@@ -1,7 +0,0 @@
-mod f16;
-mod f32;
-mod sparse_f32;
-
-pub use f16::F16;
-pub use f32::F32;
-pub use sparse_f32::{SparseF32, SparseF32Ref};

src/datatype/casts_f32.rs

@@ -2,6 +2,7 @@ use crate::datatype::svecf32::{SVecf32, SVecf32Input, SVecf32Output};
 use crate::datatype::vecf16::{Vecf16, Vecf16Input, Vecf16Output};
 use crate::datatype::vecf32::{Vecf32, Vecf32Input, Vecf32Output};
 use crate::prelude::check_value_dimensions;
+use base::scalar::FloatCast;
 use service::prelude::*;
 
 #[pgrx::pg_extern(immutable, parallel_safe, strict)]

src/datatype/operators_svecf32.rs

@@ -1,5 +1,6 @@
 use crate::datatype::svecf32::{SVecf32, SVecf32Input, SVecf32Output};
 use crate::prelude::*;
+use base::scalar::FloatCast;
 use service::prelude::*;
 use std::ops::Deref;
 

src/datatype/operators_vecf16.rs

@@ -1,5 +1,6 @@
 use crate::datatype::vecf16::{Vecf16, Vecf16Input, Vecf16Output};
 use crate::prelude::*;
+use base::scalar::FloatCast;
 use service::prelude::*;
 use std::ops::Deref;
 

src/datatype/operators_vecf32.rs

@@ -1,5 +1,6 @@
 use crate::datatype::vecf32::{Vecf32, Vecf32Input, Vecf32Output};
 use crate::prelude::*;
+use base::scalar::FloatCast;
 use service::prelude::*;
 use std::ops::Deref;