Add compatibility tests for spherical flatbuffer serialization

diskann-quantization/Cargo.toml

@@ -29,6 +29,8 @@ uninlined_format_args = "allow"
 
 [dev-dependencies]
 cfg-if = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
 trybuild = "1.0.101"
 rand_distr = { workspace = true }
 

diskann-quantization/src/algorithms/transforms/double_hadamard.rs

@@ -392,6 +392,7 @@ mod tests {
     use crate::{
         algorithms::transforms::{Transform, TransformKind, test_utils},
         alloc::GlobalAllocator,
+        test_util::Check,
     };
 
     test_utils::delegate_transformer!(DoubleHadamard<GlobalAllocator>);
@@ -404,9 +405,9 @@ mod tests {
         // These error bounds are for when we set the output dimenion to a power of 2 that
         // is higher than input dimension.
         let natural_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::ulp(5),
-            l2: test_utils::Check::ulp(5),
-            ip: test_utils::Check::absrel(2.5e-5, 2e-4),
+            norm: Check::ulp(5),
+            l2: Check::ulp(5),
+            ip: Check::absrel(2.5e-5, 2e-4),
         };
 
         // NOTE: Subsampling introduces high variance in the norm and L2, so our error
@@ -415,9 +416,9 @@ mod tests {
         // Subsampling results in poor preservation of inner products, so we skip it
         // altogether.
         let subsampled_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::absrel(0.0, 2e-2),
-            l2: test_utils::Check::absrel(0.0, 2e-2),
-            ip: test_utils::Check::skip(),
+            norm: Check::absrel(0.0, 2e-2),
+            l2: Check::absrel(0.0, 2e-2),
+            ip: Check::skip(),
         };
 
         let target_dim = |v| TargetDim::Override(NonZeroUsize::new(v).unwrap());

diskann-quantization/src/algorithms/transforms/padding_hadamard.rs

@@ -369,7 +369,7 @@ mod tests {
     use rand::{SeedableRng, rngs::StdRng};
 
     use super::*;
-    use crate::alloc::GlobalAllocator;
+    use crate::{alloc::GlobalAllocator, test_util::Check};
 
     // Since we use a slightly non-obvious strategy for applying the +/-1 permutation, we
     // test its behavior explicitly.
@@ -451,9 +451,9 @@ mod tests {
         // These error bounds are for when we set the output dimenion to a power of 2 that
         // is higher than input dimension.
         let natural_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::ulp(4),
-            l2: test_utils::Check::ulp(4),
-            ip: test_utils::Check::absrel(5.0e-6, 2e-4),
+            norm: Check::ulp(4),
+            l2: Check::ulp(4),
+            ip: Check::absrel(5.0e-6, 2e-4),
         };
 
         // NOTE: Subsampling introduces high variance in the norm and L2, so our error
@@ -462,9 +462,9 @@ mod tests {
         // Subsampling results in poor preservation of inner products, so we skip it
         // altogether.
         let subsampled_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::absrel(0.0, 1e-1),
-            l2: test_utils::Check::absrel(0.0, 1e-1),
-            ip: test_utils::Check::skip(),
+            norm: Check::absrel(0.0, 1e-1),
+            l2: Check::absrel(0.0, 1e-1),
+            ip: Check::skip(),
         };
 
         let target_dim = |v| TargetDim::Override(NonZeroUsize::new(v).unwrap());

diskann-quantization/src/algorithms/transforms/random_rotation.rs

@@ -240,6 +240,7 @@ mod tests {
     use crate::{
         algorithms::transforms::{test_utils, Transform, TransformFailed, TransformKind},
         alloc::GlobalAllocator,
+        test_util::Check,
     };
 
     impl test_utils::Transformer for RandomRotation {
@@ -257,18 +258,18 @@ mod tests {
     #[test]
     fn test_transform_matrix() {
         let nonsubsampled_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::ulp(10),
-            l2: test_utils::Check::ulp(10),
-            ip: test_utils::Check::absrel(2e-5, 1e-4),
+            norm: Check::ulp(10),
+            l2: Check::ulp(10),
+            ip: Check::absrel(2e-5, 1e-4),
         };
 
         // Because we're using relatively low dimensions, subsampling yields pretty large
         // variances. We can't use higher dimensionality, though, because then the tests
         // would never complete.
         let subsampled_errors = test_utils::ErrorSetup {
-            norm: test_utils::Check::absrel(0.0, 0.18),
-            l2: test_utils::Check::absrel(0.0, 0.18),
-            ip: test_utils::Check::skip(),
+            norm: Check::absrel(0.0, 0.18),
+            l2: Check::absrel(0.0, 0.18),
+            ip: Check::skip(),
         };
 
         let target_dim = |v| TargetDim::Override(NonZeroUsize::new(v).unwrap());

diskann-quantization/src/algorithms/transforms/test_utils.rs

@@ -3,17 +3,18 @@
  * Licensed under the MIT license.
  */
 
+use diskann_utils::lazy_format;
 use diskann_vector::{
     Norm, PureDistanceFunction,
     distance::{InnerProduct, SquaredL2},
     norm::FastL2NormSquared,
 };
 use rand::{distr::Distribution, rngs::StdRng};
 use rand_distr::StandardNormal;
-use thiserror::Error;
 
 use super::TransformFailed;
-use diskann_utils::lazy_format;
+
+use crate::test_util::Check;
 
 pub(super) trait Transformer {
     fn input_dim_(&self) -> usize;
@@ -147,107 +148,6 @@ pub(super) fn test_transform(
     }
 }
 
-fn within_ulp(mut got: f32, expected: f32, ulp: usize) -> bool {
-    if got == expected {
-        true
-    } else if got < expected {
-        for _ in 0..ulp {
-            got = got.next_up();
-            if got >= expected {
-                return true;
-            }
-        }
-        false
-    } else {
-        for _ in 0..ulp {
-            got = got.next_down();
-            if got <= expected {
-                return true;
-            }
-        }
-        false
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub(super) enum Check {
-    Ulp(usize),
-    AbsRel {
-        abs: f32,
-        rel: f32,
-    },
-    #[cfg(not(miri))]
-    Skip,
-}
-
-impl Check {
-    pub(super) fn ulp(ulp: usize) -> Self {
-        Self::Ulp(ulp)
-    }
-
-    pub(super) fn absrel(abs: f32, rel: f32) -> Self {
-        Self::AbsRel { abs, rel }
-    }
-
-    #[cfg(not(miri))]
-    pub(super) fn skip() -> Self {
-        Self::Skip
-    }
-
-    pub(super) fn check(&self, got: f32, expected: f32) -> Result<(), CheckFailed> {
-        match self {
-            Self::Ulp(ulp) => {
-                if within_ulp(got, expected, *ulp) {
-                    Ok(())
-                } else {
-                    Err(CheckFailed::Ulp {
-                        ulp: *ulp,
-                        got,
-                        expected,
-                    })
-                }
-            }
-            Self::AbsRel { abs, rel } => {
-                let abs_got = (got - expected).abs();
-                let rel_got = abs_got / (got.abs().max(expected.abs()));
-
-                if abs_got <= *abs || rel_got <= *rel {
-                    Ok(())
-                } else {
-                    Err(CheckFailed::AbsRel {
-                        abs_limit: *abs,
-                        rel_limit: *rel,
-                        abs_got,
-                        rel_got,
-                        got,
-                        expected,
-                    })
-                }
-            }
-            #[cfg(not(miri))]
-            Self::Skip => Ok(()),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, Error)]
-pub(super) enum CheckFailed {
-    #[error("not within {ulp} ulp - got {got}, expected {expected}")]
-    Ulp { ulp: usize, got: f32, expected: f32 },
-    #[error(
-        "not within {abs_limit}/{rel_limit} - errors {abs_got}/{rel_got} - \
-            got {got}, expected {expected}"
-    )]
-    AbsRel {
-        abs_limit: f32,
-        rel_limit: f32,
-        abs_got: f32,
-        rel_got: f32,
-        got: f32,
-        expected: f32,
-    },
-}
-
 #[derive(Debug, Clone, Copy)]
 pub(super) struct ErrorSetup {
     /// The error bound for the norm.