From ef6377a64b04b896af09c1872746f273cbb5946a Mon Sep 17 00:00:00 2001 From: marvin-j97 Date: Thu, 27 Feb 2025 19:22:20 +0100 Subject: [PATCH 1/5] experiment with new Iter api --- src/blob_tree/mod.rs | 34 ++++++++++++++++++++++++++++++++++ src/iter_guard.rs | 6 ++++++ src/lib.rs | 2 ++ src/tree/mod.rs | 21 +++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 src/iter_guard.rs diff --git a/src/blob_tree/mod.rs b/src/blob_tree/mod.rs index 10734b6..352d95a 100644 --- a/src/blob_tree/mod.rs +++ b/src/blob_tree/mod.rs @@ -11,6 +11,7 @@ use crate::{ coding::{Decode, Encode}, compaction::stream::CompactionStream, file::BLOBS_FOLDER, + iter_guard::IterGuard, r#abstract::{AbstractTree, RangeItem}, tree::inner::MemtableId, value::InternalValue, @@ -74,7 +75,40 @@ pub struct BlobTree { pub pending_segments: Arc, } +struct Guard<'a>( + &'a ValueLog, + crate::Result<(UserKey, UserValue)>, +); + +impl IterGuard for Guard<'_> { + fn key(self) -> crate::Result { + Ok(self.1?.0) + } + + // TODO: size() -> does not need to resolve vHandle + + fn with_value(self) -> crate::Result<(UserKey, UserValue)> { + resolve_value_handle(self.0, self.1) + } +} + impl BlobTree { + fn new_iter( + &self, + seqno: Option, + index: Option>, + ) -> impl Iterator> + use<'_> { + self.iter(seqno, index).map(|kv| Guard(&self.blobs, kv)) + } + + /* fn keys( + &self, + seqno: Option, + index: Option>, + ) -> impl Iterator> + use<'_> { + self.new_iter(seqno, index).map(IterGuard::key) + } */ + pub(crate) fn open(config: Config) -> crate::Result { let path = &config.path; diff --git a/src/iter_guard.rs b/src/iter_guard.rs new file mode 100644 index 0000000..8fd0d04 --- /dev/null +++ b/src/iter_guard.rs @@ -0,0 +1,6 @@ +use crate::{UserKey, UserValue}; + +pub trait IterGuard { + fn key(self) -> crate::Result; + fn with_value(self) -> crate::Result<(UserKey, UserValue)>; +} diff --git a/src/lib.rs b/src/lib.rs index 3a7920c..9920db7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,6 +147,8 @@ mod error; #[doc(hidden)] pub mod file; +mod iter_guard; + mod key; mod key_range; diff --git a/src/tree/mod.rs b/src/tree/mod.rs index 6dc030f..c39dccd 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -9,6 +9,7 @@ use crate::{ compaction::{stream::CompactionStream, CompactionStrategy}, config::Config, descriptor_table::FileDescriptorTable, + iter_guard::IterGuard, level_manifest::LevelManifest, manifest::Manifest, memtable::Memtable, @@ -379,7 +380,27 @@ impl AbstractTree for Tree { } } +struct Guard(crate::Result<(UserKey, UserValue)>); + +impl IterGuard for Guard { + fn key(self) -> crate::Result { + Ok(self.0?.0) + } + + fn with_value(self) -> crate::Result<(UserKey, UserValue)> { + self.0 + } +} + impl Tree { + fn new_iter( + &self, + seqno: Option, + index: Option>, + ) -> impl Iterator { + self.iter(seqno, index).map(Guard) + } + /// Opens an LSM-tree in the given directory. /// /// Will recover previous state if the folder was previously From fa8b3cc063e6c2969e002faf320b7fa12eaf0a36 Mon Sep 17 00:00:00 2001 From: marvin-j97 Date: Fri, 28 Feb 2025 15:54:27 +0100 Subject: [PATCH 2/5] wip --- src/blob_tree/mod.rs | 7 +++++-- src/iter_guard.rs | 3 ++- src/tree/mod.rs | 8 ++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/blob_tree/mod.rs b/src/blob_tree/mod.rs index 352d95a..abf1e0b 100644 --- a/src/blob_tree/mod.rs +++ b/src/blob_tree/mod.rs @@ -82,12 +82,15 @@ struct Guard<'a>( impl IterGuard for Guard<'_> { fn key(self) -> crate::Result { - Ok(self.1?.0) + self.1.map(|(k, _)| k) } // TODO: size() -> does not need to resolve vHandle + fn size(self) -> crate::Result { + unimplemented!() + } - fn with_value(self) -> crate::Result<(UserKey, UserValue)> { + fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { resolve_value_handle(self.0, self.1) } } diff --git a/src/iter_guard.rs b/src/iter_guard.rs index 8fd0d04..9347b26 100644 --- a/src/iter_guard.rs +++ b/src/iter_guard.rs @@ -2,5 +2,6 @@ use crate::{UserKey, UserValue}; pub trait IterGuard { fn key(self) -> crate::Result; - fn with_value(self) -> crate::Result<(UserKey, UserValue)>; + fn size(self) -> crate::Result; + fn into_inner(self) -> crate::Result<(UserKey, UserValue)>; } diff --git a/src/tree/mod.rs b/src/tree/mod.rs index c39dccd..c02f95e 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -384,10 +384,14 @@ struct Guard(crate::Result<(UserKey, UserValue)>); impl IterGuard for Guard { fn key(self) -> crate::Result { - Ok(self.0?.0) + self.0.map(|(k, _)| k) } - fn with_value(self) -> crate::Result<(UserKey, UserValue)> { + fn size(self) -> crate::Result { + self.into_inner().map(|(_, v)| v.len() as u32) + } + + fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { self.0 } } From 3f46cb8ec32692a98f7c85a3404cbeed9bb99805 Mon Sep 17 00:00:00 2001 From: marvin-j97 Date: Thu, 13 Mar 2025 20:44:10 +0100 Subject: [PATCH 3/5] add #110 as experimental hidden API --- src/abstract.rs | 30 +++++++++- src/blob_tree/mod.rs | 61 ++++++++++++------- src/iter_guard.rs | 44 +++++++++++++- src/lib.rs | 1 + src/tree/mod.rs | 60 +++++++++++++------ tests/experimental_blob_tree_guarded_size.rs | 19 ++++++ tests/experimental_tree_guarded_range.rs | 62 ++++++++++++++++++++ 7 files changed, 236 insertions(+), 41 deletions(-) create mode 100644 tests/experimental_blob_tree_guarded_size.rs create mode 100644 tests/experimental_tree_guarded_range.rs diff --git a/src/abstract.rs b/src/abstract.rs index 711bf8d..cde0da7 100644 --- a/src/abstract.rs +++ b/src/abstract.rs @@ -3,8 +3,9 @@ // (found in the LICENSE-* files in the repository) use crate::{ - compaction::CompactionStrategy, config::TreeType, tree::inner::MemtableId, AnyTree, BlobTree, - Config, KvPair, Memtable, Segment, SegmentId, SeqNo, Snapshot, Tree, UserKey, UserValue, + compaction::CompactionStrategy, config::TreeType, iter_guard::IterGuardImpl, + tree::inner::MemtableId, AnyTree, BlobTree, Config, KvPair, Memtable, Segment, SegmentId, + SeqNo, Snapshot, Tree, UserKey, UserValue, }; use enum_dispatch::enum_dispatch; use std::{ @@ -18,6 +19,31 @@ pub type RangeItem = crate::Result; #[allow(clippy::module_name_repetitions)] #[enum_dispatch] pub trait AbstractTree { + #[doc(hidden)] + fn guarded_iter( + &self, + seqno: Option, + index: Option>, + ) -> Box + '_> { + self.guarded_range::<&[u8], _>(.., seqno, index) + } + + #[doc(hidden)] + fn guarded_prefix>( + &self, + prefix: K, + seqno: Option, + index: Option>, + ) -> Box + '_>; + + #[doc(hidden)] + fn guarded_range, R: RangeBounds>( + &self, + range: R, + seqno: Option, + index: Option>, + ) -> Box + '_>; + /// Gets the memory usage of all bloom filters in the tree. fn bloom_filter_size(&self) -> usize; diff --git a/src/blob_tree/mod.rs b/src/blob_tree/mod.rs index abf1e0b..cd82bed 100644 --- a/src/blob_tree/mod.rs +++ b/src/blob_tree/mod.rs @@ -11,7 +11,7 @@ use crate::{ coding::{Decode, Encode}, compaction::stream::CompactionStream, file::BLOBS_FOLDER, - iter_guard::IterGuard, + iter_guard::{IterGuard, IterGuardImpl}, r#abstract::{AbstractTree, RangeItem}, tree::inner::MemtableId, value::InternalValue, @@ -75,7 +75,7 @@ pub struct BlobTree { pub pending_segments: Arc, } -struct Guard<'a>( +pub struct Guard<'a>( &'a ValueLog, crate::Result<(UserKey, UserValue)>, ); @@ -85,9 +85,18 @@ impl IterGuard for Guard<'_> { self.1.map(|(k, _)| k) } - // TODO: size() -> does not need to resolve vHandle fn size(self) -> crate::Result { - unimplemented!() + use MaybeInlineValue::{Indirect, Inline}; + + let value = self.1?.1; + let mut cursor = Cursor::new(value); + + Ok(match MaybeInlineValue::decode_from(&mut cursor)? { + Inline(bytes) => bytes.len() as u32, + + // NOTE: No need to resolve vHandle, because the size is already stored + Indirect { size, .. } => size, + }) } fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { @@ -96,22 +105,6 @@ impl IterGuard for Guard<'_> { } impl BlobTree { - fn new_iter( - &self, - seqno: Option, - index: Option>, - ) -> impl Iterator> + use<'_> { - self.iter(seqno, index).map(|kv| Guard(&self.blobs, kv)) - } - - /* fn keys( - &self, - seqno: Option, - index: Option>, - ) -> impl Iterator> + use<'_> { - self.new_iter(seqno, index).map(IterGuard::key) - } */ - pub(crate) fn open(config: Config) -> crate::Result { let path = &config.path; @@ -267,6 +260,34 @@ impl BlobTree { } impl AbstractTree for BlobTree { + fn guarded_prefix>( + &self, + prefix: K, + seqno: Option, + index: Option>, + ) -> Box + '_> { + Box::new( + self.index + .0 + .create_prefix(&prefix, seqno, index) + .map(move |kv| IterGuardImpl::Blob(Guard(&self.blobs, kv))), + ) + } + + fn guarded_range, R: RangeBounds>( + &self, + range: R, + seqno: Option, + index: Option>, + ) -> Box + '_> { + Box::new( + self.index + .0 + .create_range(&range, seqno, index) + .map(move |kv| IterGuardImpl::Blob(Guard(&self.blobs, kv))), + ) + } + fn blob_file_count(&self) -> usize { self.blobs.segment_count() } diff --git a/src/iter_guard.rs b/src/iter_guard.rs index 9347b26..5655dbd 100644 --- a/src/iter_guard.rs +++ b/src/iter_guard.rs @@ -1,7 +1,47 @@ -use crate::{UserKey, UserValue}; +use crate::{ + blob_tree::Guard as BlobGuard, tree::Guard as StandardGuard, KvPair, UserKey, UserValue, +}; +use enum_dispatch::enum_dispatch; +/// An iterator item +#[enum_dispatch] pub trait IterGuard { + /// Accesses the key-value tuple. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn into_inner(self) -> crate::Result; + + /// Accesses the key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. fn key(self) -> crate::Result; + + /// Returns the value size. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. fn size(self) -> crate::Result; - fn into_inner(self) -> crate::Result<(UserKey, UserValue)>; + + /// Accesses the value. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn value(self) -> crate::Result + where + Self: Sized, + { + self.into_inner().map(|(_, v)| v) + } +} + +#[enum_dispatch(IterGuard)] +pub enum IterGuardImpl<'a> { + Standard(StandardGuard), + Blob(BlobGuard<'a>), } diff --git a/src/lib.rs b/src/lib.rs index 9920db7..0706b0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -194,6 +194,7 @@ pub type KvPair = (UserKey, UserValue); #[doc(hidden)] pub use { + iter_guard::IterGuard as Guard, merge::BoxedIterator, segment::{block::checksum::Checksum, id::GlobalSegmentId, meta::SegmentId}, tree::inner::TreeId, diff --git a/src/tree/mod.rs b/src/tree/mod.rs index 0a69773..6df4f37 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -9,7 +9,7 @@ use crate::{ compaction::{stream::CompactionStream, CompactionStrategy}, config::Config, descriptor_table::FileDescriptorTable, - iter_guard::IterGuard, + iter_guard::{IterGuard, IterGuardImpl}, level_manifest::LevelManifest, manifest::Manifest, memtable::Memtable, @@ -52,7 +52,49 @@ impl std::ops::Deref for Tree { } } +pub struct Guard(crate::Result<(UserKey, UserValue)>); + +impl IterGuard for Guard { + fn key(self) -> crate::Result { + self.0.map(|(k, _)| k) + } + + fn size(self) -> crate::Result { + // NOTE: We know LSM-tree values are 32 bits in length max + #[allow(clippy::cast_possible_truncation)] + self.into_inner().map(|(_, v)| v.len() as u32) + } + + fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { + self.0 + } +} + impl AbstractTree for Tree { + fn guarded_prefix>( + &self, + prefix: K, + seqno: Option, + index: Option>, + ) -> Box + '_> { + Box::new( + self.create_prefix(&prefix, seqno, index) + .map(|kv| IterGuardImpl::Standard(Guard(kv))), + ) + } + + fn guarded_range, R: RangeBounds>( + &self, + range: R, + seqno: Option, + index: Option>, + ) -> Box + '_> { + Box::new( + self.create_range(&range, seqno, index) + .map(|kv| IterGuardImpl::Standard(Guard(kv))), + ) + } + fn size_of>(&self, key: K, seqno: Option) -> crate::Result> { Ok(self.get(key, seqno)?.map(|x| x.len() as u32)) } @@ -382,22 +424,6 @@ impl AbstractTree for Tree { } } -struct Guard(crate::Result<(UserKey, UserValue)>); - -impl IterGuard for Guard { - fn key(self) -> crate::Result { - self.0.map(|(k, _)| k) - } - - fn size(self) -> crate::Result { - self.into_inner().map(|(_, v)| v.len() as u32) - } - - fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { - self.0 - } -} - impl Tree { fn new_iter( &self, diff --git a/tests/experimental_blob_tree_guarded_size.rs b/tests/experimental_blob_tree_guarded_size.rs new file mode 100644 index 0000000..6647d82 --- /dev/null +++ b/tests/experimental_blob_tree_guarded_size.rs @@ -0,0 +1,19 @@ +use lsm_tree::{AbstractTree, Config, Guard}; +use test_log::test; + +#[test] +fn experimental_blob_tree_guarded_size() -> lsm_tree::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = Config::new(folder).open_as_blob_tree()?; + + tree.insert("a".as_bytes(), "abc", 0); + tree.insert("b".as_bytes(), "a".repeat(10_000), 0); + + assert_eq!( + 10_003u32, + tree.guarded_iter(None, None).flat_map(Guard::size).sum() + ); + + Ok(()) +} diff --git a/tests/experimental_tree_guarded_range.rs b/tests/experimental_tree_guarded_range.rs new file mode 100644 index 0000000..c4c8977 --- /dev/null +++ b/tests/experimental_tree_guarded_range.rs @@ -0,0 +1,62 @@ +use lsm_tree::{AbstractTree, Config, Guard}; +use test_log::test; + +#[test] +fn experimental_tree_guarded_range() -> lsm_tree::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = Config::new(folder).open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!( + 2, + tree.guarded_range("a"..="f", None, None) + .flat_map(Guard::key) + .count() + ); + assert_eq!( + 2, + tree.guarded_range("f"..="g", None, None) + .flat_map(Guard::key) + .count() + ); + + Ok(()) +} + +#[test] +fn experimental_blob_tree_guarded_range() -> lsm_tree::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = Config::new(folder).open_as_blob_tree()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!( + 2, + tree.guarded_range("a"..="f", None, None) + .flat_map(Guard::key) + .count() + ); + assert_eq!( + 2, + tree.guarded_range("f"..="g", None, None) + .flat_map(Guard::key) + .count() + ); + + Ok(()) +} From 92fcaaa0377bd3b1a30d26a09b76a0c825576b7d Mon Sep 17 00:00:00 2001 From: marvin-j97 Date: Thu, 13 Mar 2025 20:46:12 +0100 Subject: [PATCH 4/5] doc --- src/abstract.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/abstract.rs b/src/abstract.rs index cde0da7..176fd0e 100644 --- a/src/abstract.rs +++ b/src/abstract.rs @@ -19,6 +19,15 @@ pub type RangeItem = crate::Result; #[allow(clippy::module_name_repetitions)] #[enum_dispatch] pub trait AbstractTree { + /// Returns an iterator that scans through the entire tree. + /// + /// Avoid using this function, or limit it as otherwise it may scan a lot of items. + /// + /// # Experimental + /// + /// This API is experimental and will 100% be renamed. + /// + /// https://github.com/fjall-rs/lsm-tree/issues/110 #[doc(hidden)] fn guarded_iter( &self, @@ -28,6 +37,15 @@ pub trait AbstractTree { self.guarded_range::<&[u8], _>(.., seqno, index) } + /// Returns an iterator over a prefixed set of items. + /// + /// Avoid using an empty prefix as it may scan a lot of items (unless limited). + /// + /// # Experimental + /// + /// This API is experimental and will 100% be renamed. + /// + /// https://github.com/fjall-rs/lsm-tree/issues/110 #[doc(hidden)] fn guarded_prefix>( &self, @@ -36,6 +54,15 @@ pub trait AbstractTree { index: Option>, ) -> Box + '_>; + /// Returns an iterator over a range of items. + /// + /// Avoid using full or unbounded ranges as they may scan a lot of items (unless limited). + /// + /// # Experimental + /// + /// This API is experimental and will 100% be renamed. + /// + /// https://github.com/fjall-rs/lsm-tree/issues/110 #[doc(hidden)] fn guarded_range, R: RangeBounds>( &self, From 5c6c8cc82bdaa7a02c95af8cdb61f41dd4ee5803 Mon Sep 17 00:00:00 2001 From: marvin-j97 Date: Thu, 13 Mar 2025 20:49:02 +0100 Subject: [PATCH 5/5] wip --- src/blob_tree/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/blob_tree/mod.rs b/src/blob_tree/mod.rs index cd82bed..2550163 100644 --- a/src/blob_tree/mod.rs +++ b/src/blob_tree/mod.rs @@ -92,6 +92,8 @@ impl IterGuard for Guard<'_> { let mut cursor = Cursor::new(value); Ok(match MaybeInlineValue::decode_from(&mut cursor)? { + // NOTE: We know LSM-tree values are 32 bits in length max + #[allow(clippy::cast_possible_truncation)] Inline(bytes) => bytes.len() as u32, // NOTE: No need to resolve vHandle, because the size is already stored