Merge branch 'main' into EXC-1404-memory-manager-support-freeing-a-memory

Author: dragoljub-duric

src/btreemap.rs

@@ -36,9 +36,9 @@ use allocator::Allocator;
 pub use iter::Iter;
 use iter::{Cursor, Index};
 use node::{DerivedPageSize, Entry, Node, NodeType, PageSize, Version};
+use std::borrow::Cow;
 use std::marker::PhantomData;
 use std::ops::{Bound, RangeBounds};
-use std::{borrow::Cow, fmt};
 
 #[cfg(test)]
 mod proptests;
@@ -1187,32 +1187,6 @@ where
     }
 }
 
-/// An error returned when inserting entries into the map.
-#[derive(Debug, PartialEq, Eq)]
-pub enum InsertError {
-    KeyTooLarge { given: usize, max: usize },
-    ValueTooLarge { given: usize, max: usize },
-}
-
-impl fmt::Display for InsertError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::KeyTooLarge { given, max } => {
-                write!(
-                    f,
-                    "InsertError::KeyTooLarge Expected key to be <= {max} bytes but received key with {given} bytes."
-                )
-            }
-            Self::ValueTooLarge { given, max } => {
-                write!(
-                    f,
-                    "InsertError::ValueTooLarge Expected value to be <= {max} bytes but received value with {given} bytes."
-                )
-            }
-        }
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::*;
@@ -1223,7 +1197,7 @@ mod test {
     use std::cell::RefCell;
     use std::rc::Rc;
 
-    fn make_memory() -> Rc<RefCell<Vec<u8>>> {
+    pub(crate) fn make_memory() -> Rc<RefCell<Vec<u8>>> {
         Rc::new(RefCell::new(Vec::new()))
     }
 

src/btreemap/proptests.rs

@@ -1,17 +1,92 @@
 use crate::{
-    btreemap::test::{b, btree_test},
+    btreemap::{
+        test::{b, btree_test, make_memory},
+        BTreeMap,
+    },
     storable::Blob,
+    Memory,
 };
 use proptest::collection::btree_set as pset;
 use proptest::collection::vec as pvec;
 use proptest::prelude::*;
-use std::collections::BTreeSet;
+use std::collections::{BTreeMap as StdBTreeMap, BTreeSet};
 use test_strategy::proptest;
 
+#[derive(Debug, Clone)]
+enum Operation {
+    Insert { key: Vec<u8>, value: Vec<u8> },
+    Iter { from: usize, len: usize },
+    Get(usize),
+    Remove(usize),
+}
+
+// A custom strategy that gives unequal weights to the different operations.
+// Note that `Insert` has a higher weight than `Remove` so that, on average, BTreeMaps
+// are growing in size the more operations are executed.
+fn operation_strategy() -> impl Strategy<Value = Operation> {
+    prop_oneof![
+        3 => (any::<Vec<u8>>(), any::<Vec<u8>>())
+            .prop_map(|(key, value)| Operation::Insert { key, value }),
+        1 => (any::<usize>(), any::<usize>())
+            .prop_map(|(from, len)| Operation::Iter { from, len }),
+        2 => (any::<usize>()).prop_map(Operation::Get),
+        1 => (any::<usize>()).prop_map(Operation::Remove),
+    ]
+}
+
 fn arb_blob() -> impl Strategy<Value = Blob<10>> {
     pvec(0..u8::MAX, 0..10).prop_map(|v| Blob::<10>::try_from(v.as_slice()).unwrap())
 }
 
+// Runs a comprehensive test for the major stable BTreeMap operations.
+// Results are validated against a standard BTreeMap.
+#[proptest(cases = 10)]
+fn comprehensive(#[strategy(pvec(operation_strategy(), 100..5_000))] ops: Vec<Operation>) {
+    let mem = make_memory();
+    let mut btree = BTreeMap::new(mem);
+    let mut std_btree = StdBTreeMap::new();
+
+    // Execute all the operations, validating that the stable btreemap behaves similarly to a std
+    // btreemap.
+    for op in ops.into_iter() {
+        execute_operation(&mut std_btree, &mut btree, op);
+    }
+}
+
+// A comprehensive fuzz test that runs until it's explicitly terminated. To run:
+//
+// ```
+// cargo t comprehensive_fuzz -- --ignored --nocapture 2> comprehensive_fuzz.log
+// ```
+//
+// comprehensive_fuzz.log contains all the operations to help triage a failure.
+#[test]
+#[ignore]
+fn comprehensive_fuzz() {
+    use proptest::strategy::ValueTree;
+    use proptest::test_runner::TestRunner;
+    let mut runner = TestRunner::default();
+
+    let mem = make_memory();
+    let mut btree = BTreeMap::new(mem);
+    let mut std_btree = StdBTreeMap::new();
+
+    let mut i = 0;
+
+    loop {
+        let operation = operation_strategy()
+            .new_tree(&mut runner)
+            .unwrap()
+            .current();
+        execute_operation(&mut std_btree, &mut btree, operation);
+        i += 1;
+        if i % 1000 == 0 {
+            println!("=== Step {i} ===");
+            println!("=== BTree Size: {}", btree.len());
+        }
+    }
+}
+
 #[proptest(cases = 10)]
 fn insert(#[strategy(pset(arb_blob(), 1000..10_000))] keys: BTreeSet<Blob<10>>) {
     btree_test(|mut btree| {
@@ -61,3 +136,65 @@ fn map_upper_bound_iter(#[strategy(pvec(0u64..u64::MAX -1 , 10..100))] keys: Vec
         Ok(())
     });
 }
+
+// Given an operation, executes it on the given stable btreemap and standard btreemap, verifying
+// that the result of the operation is equal in both btrees.
+fn execute_operation<M: Memory>(
+    std_btree: &mut StdBTreeMap<Vec<u8>, Vec<u8>>,
+    btree: &mut BTreeMap<Vec<u8>, Vec<u8>, M>,
+    op: Operation,
+) {
+    match op {
+        Operation::Insert { key, value } => {
+            let std_res = std_btree.insert(key.clone(), value.clone());
+
+            eprintln!("Insert({}, {})", hex::encode(&key), hex::encode(&value));
+            let res = btree.insert(key, value);
+            assert_eq!(std_res, res);
+        }
+        Operation::Iter { from, len } => {
+            assert_eq!(std_btree.len(), btree.len() as usize);
+            if std_btree.is_empty() {
+                return;
+            }
+
+            let from = from % std_btree.len();
+            let len = len % std_btree.len();
+
+            eprintln!("Iterate({}, {})", from, len);
+            let std_iter = std_btree.iter().skip(from).take(len);
+            let stable_iter = btree.iter().skip(from).take(len);
+            for ((k1, v1), (k2, v2)) in std_iter.zip(stable_iter) {
+                assert_eq!(k1, &k2);
+                assert_eq!(v1, &v2);
+            }
+        }
+        Operation::Get(idx) => {
+            assert_eq!(std_btree.len(), btree.len() as usize);
+            if std_btree.is_empty() {
+                return;
+            }
+            let idx = idx % std_btree.len();
+
+            if let Some((k, v)) = btree.iter().skip(idx).take(1).next() {
+                eprintln!("Get({})", hex::encode(&k));
+                assert_eq!(std_btree.get(&k), Some(&v));
+                assert_eq!(btree.get(&k), Some(v));
+            }
+        }
+        Operation::Remove(idx) => {
+            assert_eq!(std_btree.len(), btree.len() as usize);
+            if std_btree.is_empty() {
+                return;
+            }
+
+            let idx = idx % std_btree.len();
+
+            if let Some((k, v)) = btree.iter().skip(idx).take(1).next() {
+                eprintln!("Remove({})", hex::encode(&k));
+                assert_eq!(std_btree.remove(&k), Some(v.clone()));
+                assert_eq!(btree.remove(&k), Some(v));
+            }
+        }
+    };
+}

src/storable.rs

@@ -70,6 +70,26 @@ pub enum Bound {
     },
 }
 
+impl Bound {
+    /// Returns the maximum size of the type if bounded, panics if unbounded.
+    pub const fn max_size(&self) -> u32 {
+        if let Bound::Bounded { max_size, .. } = self {
+            *max_size
+        } else {
+            panic!("Cannot get max size of unbounded type.");
+        }
+    }
+
+    /// Returns true if the type is fixed in size, false otherwise.
+    pub const fn is_fixed_size(&self) -> bool {
+        if let Bound::Bounded { is_fixed_size, .. } = self {
+            *is_fixed_size
+        } else {
+            false
+        }
+    }
+}
+
 /// Variable-size, but limited in capacity byte array.
 #[derive(Eq, Copy, Clone)]
 pub struct Blob<const N: usize> {
@@ -450,6 +470,46 @@ where
     };
 }
 
+impl<T: Storable> Storable for Option<T> {
+    fn to_bytes(&self) -> Cow<[u8]> {
+        match self {
+            Some(t) => {
+                let mut bytes = t.to_bytes().into_owned();
+                bytes.push(1);
+                Cow::Owned(bytes)
+            }
+            None => Cow::Borrowed(&[0]),
+        }
+    }
+
+    fn from_bytes(bytes: Cow<[u8]>) -> Self {
+        match bytes.split_last() {
+            Some((last, rest)) => match last {
+                0 => {
+                    assert!(rest.is_empty(), "Invalid Option encoding: unexpected prefix before the None marker: {rest:?}");
+                    None
+                }
+                1 => Some(T::from_bytes(Cow::Borrowed(rest))),
+                _ => panic!("Invalid Option encoding: unexpected variant marker {last}"),
+            },
+            None => panic!("Invalid Option encoding: expected at least one byte"),
+        }
+    }
+
+    const BOUND: Bound = {
+        match T::BOUND {
+            Bound::Bounded {
+                max_size,
+                is_fixed_size,
+            } => Bound::Bounded {
+                max_size: max_size + 1,
+                is_fixed_size,
+            },
+            Bound::Unbounded => Bound::Unbounded,
+        }
+    };
+}
+
 pub(crate) struct Bounds {
     pub max_size: u32,
     pub is_fixed_size: bool,

src/storable/tests.rs

@@ -35,6 +35,27 @@ proptest! {
     fn f32_roundtrip(v in any::<f32>()) {
         prop_assert_eq!(v, Storable::from_bytes(v.to_bytes()));
     }
+
+    #[test]
+    fn optional_f64_roundtrip(v in proptest::option::of(any::<f64>())) {
+        prop_assert_eq!(v, Storable::from_bytes(v.to_bytes()));
+    }
+
+    #[test]
+    fn optional_string_roundtrip(v in proptest::option::of(any::<String>())) {
+        prop_assert_eq!(v.clone(), Storable::from_bytes(v.to_bytes()));
+    }
+
+    #[test]
+    fn optional_tuple_roundtrip(v in proptest::option::of((any::<u64>(), uniform20(any::<u8>())))) {
+        prop_assert_eq!(v, Storable::from_bytes(v.to_bytes()));
+    }
+
+    #[test]
+    fn optional_tuple_variable_width_u8_roundtrip(v in proptest::option::of((any::<u64>(), pvec(any::<u8>(), 0..40)))) {
+        let v = v.map(|(n, bytes)| (n, Blob::<48>::try_from(&bytes[..]).unwrap()));
+        prop_assert_eq!(v, Storable::from_bytes(v.to_bytes()));
+    }
 }
 
 #[test]