From 7e78e612cf60194bf1ba17620d5e0506d8f3e390 Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Sun, 3 Nov 2019 14:41:22 +0900 Subject: [PATCH 1/7] Implement recycle_vec --- src/liballoc/lib.rs | 1 + src/liballoc/vec.rs | 87 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index 94379afc2bd45..448ad5909e0b2 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -123,6 +123,7 @@ #![feature(alloc_layout_extra)] #![feature(try_trait)] #![feature(associated_type_bounds)] +#![feature(recycle_vec)] // Allow testing this library diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 5b53a6a289958..b6e5f40abd27d 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -603,6 +603,93 @@ impl Vec { self.buf.try_reserve_exact(self.len, additional) } + /// Allows re-interpreting the type of a Vec to reuse the allocation. + /// The vector is emptied and any values contained in it will be dropped. + /// The target type must have the same size and alignment as the source type. + /// This API doesn't transmute any values of T to U, because it makes sure + /// to empty the vector before any unsafe operations. + /// + /// # Example + /// + /// This API is useful especially when using `Vec` as a + /// temporary storage for data with short lifetimes. + /// By recycling the allocation, the `Vec` is able to safely + /// outlive the lifetime of the type that was stored in it. + /// ``` + /// # use std::error::Error; + /// # + /// # struct Stream; + /// # + /// # impl Stream { + /// # fn new() -> Self { + /// # Stream + /// # } + /// # + /// # fn next(&mut self) -> Option<&[u8]> { + /// # Some(&b"foo"[..]) + /// # } + /// # } + /// # + /// # struct DbConnection; + /// # + /// # impl DbConnection { + /// # fn new() -> Self { + /// # DbConnection + /// # } + /// # + /// # fn insert(&mut self, _objects: &[Object<'_>]) -> Result<(), Box> { + /// # Ok(()) + /// # } + /// # } + /// # + /// # struct Object<'a> { + /// # reference: &'a [u8], + /// # } + /// # + /// # fn deserialize<'a>(input: &'a [u8], output: &mut Vec>) -> Result<(), Box> { + /// # output.push(Object { reference: input }); + /// # Ok(()) + /// # } + /// # + /// # fn processor() -> Result<(), Box> { + /// # let mut stream = Stream::new(); + /// # let mut db_connection = DbConnection::new(); + /// let mut objects: Vec> = Vec::new(); + /// + /// while let Some(byte_chunk) = stream.next() { // byte_chunk only lives this scope + /// let mut objects_temp: Vec> = objects.recycle(); + /// + /// // Zero-copy parsing; Object has references to chunk + /// deserialize(byte_chunk, &mut objects_temp)?; + /// db_connection.insert(&objects_temp)?; + /// + /// objects = objects_temp.recycle(); + /// } // byte_chunk lifetime ends + /// # + /// # Ok(()) + /// # } + /// ``` + /// + /// # Panics + /// Panics if the size or alignment of the source and target types don't match. + /// + /// # Note about stabilization + /// The size and alignment contract is enforceable at compile-time, + /// so we will want to wait until compile-time asserts become stable and + /// modify this API to cause a compile error instead of panicking + /// before stabilizing it. + #[unstable(feature = "recycle_vec", reason = "new API", issue = "0")] + pub fn recycle(mut self) -> Vec { + self.truncate(0); + // TODO make these const asserts once it becomes possible + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + assert_eq!(core::mem::align_of::(), core::mem::align_of::()); + let capacity = self.capacity(); + let ptr = self.as_mut_ptr() as *mut U; + core::mem::forget(self); + unsafe { Vec::from_raw_parts(ptr, 0, capacity) } + } + /// Shrinks the capacity of the vector as much as possible. /// /// It will drop down as close as possible to the length but the allocator From 6ebdc5d9d1a47f1e6d04a1003e12bd0825c1e4c1 Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 07:13:12 +0900 Subject: [PATCH 2/7] Implement tests for recycle_vec --- src/liballoc/tests/lib.rs | 1 + src/liballoc/tests/vec.rs | 70 +++++++++++++++++++++++++++++++++++++++ src/liballoc/vec.rs | 49 ++++++++++++++------------- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src/liballoc/tests/lib.rs b/src/liballoc/tests/lib.rs index 3273feb7b5dd4..55ac639cc6441 100644 --- a/src/liballoc/tests/lib.rs +++ b/src/liballoc/tests/lib.rs @@ -10,6 +10,7 @@ #![feature(associated_type_bounds)] #![feature(binary_heap_into_iter_sorted)] #![feature(binary_heap_drain_sorted)] +#![feature(recycle_vec)] use std::hash::{Hash, Hasher}; use std::collections::hash_map::DefaultHasher; diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 80537217697ad..8790b01a6ab7f 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -1264,6 +1264,76 @@ fn test_try_reserve_exact() { } +/// Tests that `recycle` successfully re-interprets the type to have different lifetime from the original +#[test] +fn test_recycle_lifetime() { + let s_1 = "foo".to_string(); + let mut buf = Vec::with_capacity(100); + { + let mut buf2 = buf; + let s_2 = "foo".to_string(); + buf2.push(s_2.as_str()); + + assert_eq!(buf2.len(), 1); + assert_eq!(buf2.capacity(), 100); + + buf = buf2.recycle(); + } + buf.push(s_1.as_str()); +} + +/// Tests that `recycle` successfully re-interprets the type itself +#[test] +fn test_recycle_type() { + let s = "foo".to_string(); + let mut buf = Vec::with_capacity(100); + { + let mut buf2 = buf.recycle(); + + let mut i = Vec::new(); + i.push(1); + i.push(2); + i.push(3); + + buf2.push(i.as_slice()); + + assert_eq!(buf2.len(), 1); + assert_eq!(buf2.capacity(), 100); + + buf = buf2.recycle(); + } + buf.push(s.as_str()); +} + +/// Tests that `recycle` successfully panics with incompatible sizes +#[test] +#[should_panic] +fn test_recycle_incompatible_size() { + let mut buf = Vec::with_capacity(100); + buf.push(1_u16); + { + let mut buf2 = buf.recycle(); + buf2.push(1_u32); + buf = buf2.recycle(); + } + buf.push(1_u16); +} + + +/// Tests that `recycle` successfully panics with incompatible alignments +#[test] +#[should_panic] +fn test_recycle_incompatible_alignment() { + let mut buf = Vec::with_capacity(100); + buf.push([0_u16, 1_u16]); + { + let mut buf2 = buf.recycle(); + buf2.push(1_u32); + buf = buf2.recycle(); + } + buf.push([0_u16, 1_u16]); +} + #[test] fn test_stable_push_pop() { // Test that, if we reserved enough space, adding and removing elements does not diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index b6e5f40abd27d..d91ee74605e97 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -616,30 +616,31 @@ impl Vec { /// By recycling the allocation, the `Vec` is able to safely /// outlive the lifetime of the type that was stored in it. /// ``` + /// # #![feature(recycle_vec)] /// # use std::error::Error; /// # - /// # struct Stream; + /// # struct Stream(bool); /// # /// # impl Stream { /// # fn new() -> Self { - /// # Stream + /// # Stream(false) /// # } /// # /// # fn next(&mut self) -> Option<&[u8]> { - /// # Some(&b"foo"[..]) + /// # if self.0 { + /// # None + /// # } else { + /// # self.0 = true; + /// # Some(&b"foo"[..]) + /// # } /// # } /// # } /// # - /// # struct DbConnection; - /// # - /// # impl DbConnection { - /// # fn new() -> Self { - /// # DbConnection - /// # } - /// # - /// # fn insert(&mut self, _objects: &[Object<'_>]) -> Result<(), Box> { - /// # Ok(()) + /// # fn process(input: &[Object<'_>]) -> Result<(), Box> { + /// # for obj in input { + /// # let _ = obj.reference; /// # } + /// # Ok(()) /// # } /// # /// # struct Object<'a> { @@ -651,22 +652,20 @@ impl Vec { /// # Ok(()) /// # } /// # - /// # fn processor() -> Result<(), Box> { - /// # let mut stream = Stream::new(); - /// # let mut db_connection = DbConnection::new(); - /// let mut objects: Vec> = Vec::new(); + /// # fn main() -> Result<(), Box> { + /// # let mut stream = Stream::new(); + /// let mut objects: Vec> = Vec::new(); // Any lifetime goes here /// - /// while let Some(byte_chunk) = stream.next() { // byte_chunk only lives this scope - /// let mut objects_temp: Vec> = objects.recycle(); + /// while let Some(byte_chunk) = stream.next() { // `byte_chunk` lifetime starts + /// let mut temp: Vec> = objects.recycle(); // `temp` lifetime starts /// - /// // Zero-copy parsing; Object has references to chunk - /// deserialize(byte_chunk, &mut objects_temp)?; - /// db_connection.insert(&objects_temp)?; + /// // Zero-copy parsing; deserialized `Object`s have references to `byte_chunk` + /// deserialize(byte_chunk, &mut temp)?; + /// process(&temp)?; /// - /// objects = objects_temp.recycle(); - /// } // byte_chunk lifetime ends - /// # - /// # Ok(()) + /// objects = temp.recycle(); // `temp` lifetime ends + /// } // `byte_chunk` lifetime ends + /// # Ok(()) /// # } /// ``` /// From 2d214d59ad602b06bda10b040b4c865e910c234d Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 15:00:10 +0900 Subject: [PATCH 3/7] Address review comments. --- src/liballoc/tests/vec.rs | 17 +++++++++++++++-- src/liballoc/vec.rs | 34 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 8790b01a6ab7f..3191be6f908bf 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -1264,22 +1264,31 @@ fn test_try_reserve_exact() { } -/// Tests that `recycle` successfully re-interprets the type to have different lifetime from the original +/// Tests that `recycle` successfully re-interprets the type +/// to have a different lifetime from the original #[test] fn test_recycle_lifetime() { let s_1 = "foo".to_string(); + + let val_size; + let val_align; + let mut buf = Vec::with_capacity(100); { let mut buf2 = buf; - let s_2 = "foo".to_string(); + let s_2 = "bar".to_string(); buf2.push(s_2.as_str()); assert_eq!(buf2.len(), 1); assert_eq!(buf2.capacity(), 100); + val_size = core::mem::size_of_val(&buf[0]); + val_align = core::mem::align_of_val(&buf[0]); buf = buf2.recycle(); } buf.push(s_1.as_str()); + assert_eq!(val_size, core::mem::size_of_val(&buf[0])); + assert_eq!(val_align, core::mem::align_of_val(&buf[0])); } /// Tests that `recycle` successfully re-interprets the type itself @@ -1299,10 +1308,14 @@ fn test_recycle_type() { assert_eq!(buf2.len(), 1); assert_eq!(buf2.capacity(), 100); + val_size = core::mem::size_of_val(&buf[0]); + val_align = core::mem::align_of_val(&buf[0]); buf = buf2.recycle(); } buf.push(s.as_str()); + assert_eq!(val_size, core::mem::size_of_val(&buf[0])); + assert_eq!(val_align, core::mem::align_of_val(&buf[0])); } /// Tests that `recycle` successfully panics with incompatible sizes diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index d91ee74605e97..f9814168681e6 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -603,11 +603,15 @@ impl Vec { self.buf.try_reserve_exact(self.len, additional) } - /// Allows re-interpreting the type of a Vec to reuse the allocation. + /// Allows reusing the allocation of the `Vec` at a different, + /// but compatible, type `Vec`. /// The vector is emptied and any values contained in it will be dropped. + /// As a result, no elements of type `T` are transmuted to `U` + /// and so this operation is safe. /// The target type must have the same size and alignment as the source type. - /// This API doesn't transmute any values of T to U, because it makes sure - /// to empty the vector before any unsafe operations. + /// + /// # Panics + /// Panics if the size or alignment of the source and target types don't match. /// /// # Example /// @@ -616,7 +620,7 @@ impl Vec { /// By recycling the allocation, the `Vec` is able to safely /// outlive the lifetime of the type that was stored in it. /// ``` - /// # #![feature(recycle_vec)] + /// #![feature(recycle_vec)] /// # use std::error::Error; /// # /// # struct Stream(bool); @@ -637,17 +641,18 @@ impl Vec { /// # } /// # /// # fn process(input: &[Object<'_>]) -> Result<(), Box> { - /// # for obj in input { - /// # let _ = obj.reference; - /// # } /// # Ok(()) /// # } /// # /// # struct Object<'a> { + /// # #[allow(dead_code)] /// # reference: &'a [u8], /// # } /// # - /// # fn deserialize<'a>(input: &'a [u8], output: &mut Vec>) -> Result<(), Box> { + /// # fn deserialize<'a>( + /// # input: &'a [u8], + /// # output: &mut Vec>, + /// # ) -> Result<(), Box> { /// # output.push(Object { reference: input }); /// # Ok(()) /// # } @@ -659,7 +664,7 @@ impl Vec { /// while let Some(byte_chunk) = stream.next() { // `byte_chunk` lifetime starts /// let mut temp: Vec> = objects.recycle(); // `temp` lifetime starts /// - /// // Zero-copy parsing; deserialized `Object`s have references to `byte_chunk` + /// // Zero-copy parsing; deserialized `Object`s have references to `byte_chunk`. /// deserialize(byte_chunk, &mut temp)?; /// process(&temp)?; /// @@ -669,9 +674,6 @@ impl Vec { /// # } /// ``` /// - /// # Panics - /// Panics if the size or alignment of the source and target types don't match. - /// /// # Note about stabilization /// The size and alignment contract is enforceable at compile-time, /// so we will want to wait until compile-time asserts become stable and @@ -679,13 +681,11 @@ impl Vec { /// before stabilizing it. #[unstable(feature = "recycle_vec", reason = "new API", issue = "0")] pub fn recycle(mut self) -> Vec { - self.truncate(0); - // TODO make these const asserts once it becomes possible assert_eq!(core::mem::size_of::(), core::mem::size_of::()); assert_eq!(core::mem::align_of::(), core::mem::align_of::()); - let capacity = self.capacity(); - let ptr = self.as_mut_ptr() as *mut U; - core::mem::forget(self); + self.clear(); + let (ptr, len, capacity) = self.into_raw_parts(); + let ptr = ptr as *mut U; unsafe { Vec::from_raw_parts(ptr, 0, capacity) } } From 161cab28dfa56c42853be3734a3f6268c54b0bf4 Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 15:19:27 +0900 Subject: [PATCH 4/7] Ignore len of raw parts --- src/liballoc/vec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index f9814168681e6..c126ec582d2a6 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -684,7 +684,7 @@ impl Vec { assert_eq!(core::mem::size_of::(), core::mem::size_of::()); assert_eq!(core::mem::align_of::(), core::mem::align_of::()); self.clear(); - let (ptr, len, capacity) = self.into_raw_parts(); + let (ptr, _, capacity) = self.into_raw_parts(); let ptr = ptr as *mut U; unsafe { Vec::from_raw_parts(ptr, 0, capacity) } } From 8e246b425465d3ae1891c990cd61924359f47d4d Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 15:36:20 +0900 Subject: [PATCH 5/7] ignore-tidy-filelength --- src/liballoc/vec.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index c126ec582d2a6..cd1320a537999 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -1,3 +1,4 @@ +// ignore-tidy-filelength //! A contiguous growable array type with heap-allocated contents, written //! `Vec`. //! From e7ffde22e9883569ac3cd8d3cd571df79268bb74 Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 17:25:04 +0900 Subject: [PATCH 6/7] Add forgotten val_size and val_align --- src/liballoc/tests/vec.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 3191be6f908bf..0c00d9a8bc1e4 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -1295,6 +1295,10 @@ fn test_recycle_lifetime() { #[test] fn test_recycle_type() { let s = "foo".to_string(); + + let val_size; + let val_align; + let mut buf = Vec::with_capacity(100); { let mut buf2 = buf.recycle(); From 35e4d9b49d9712f669d7a79841741cf8a09ba2a5 Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Mon, 4 Nov 2019 19:02:54 +0900 Subject: [PATCH 7/7] Fix test --- src/liballoc/tests/vec.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/liballoc/tests/vec.rs b/src/liballoc/tests/vec.rs index 0c00d9a8bc1e4..aea7fc6a4ab1c 100644 --- a/src/liballoc/tests/vec.rs +++ b/src/liballoc/tests/vec.rs @@ -1281,8 +1281,8 @@ fn test_recycle_lifetime() { assert_eq!(buf2.len(), 1); assert_eq!(buf2.capacity(), 100); - val_size = core::mem::size_of_val(&buf[0]); - val_align = core::mem::align_of_val(&buf[0]); + val_size = core::mem::size_of_val(&buf2[0]); + val_align = core::mem::align_of_val(&buf2[0]); buf = buf2.recycle(); } @@ -1312,8 +1312,8 @@ fn test_recycle_type() { assert_eq!(buf2.len(), 1); assert_eq!(buf2.capacity(), 100); - val_size = core::mem::size_of_val(&buf[0]); - val_align = core::mem::align_of_val(&buf[0]); + val_size = core::mem::size_of_val(&buf2[0]); + val_align = core::mem::align_of_val(&buf2[0]); buf = buf2.recycle(); }