Subject: [PATCH] Trace Vec buffers for the same-size SmallVec count For Rust 1.99.0 (b940084d7) on x86-64 Linux with glibc. Set UV_VEC_TRACE to write 40-byte records. From a uv checkout with the Rust 1.99.0 rust-src component installed, run the companion script to apply the patch, run the workload, and count eligible buffers: uv run /absolute/path/to/blog/data/benchmarks/vec-layout/same-size-smallvec.py Pass --trace /path/to/events.bin to analyze an existing trace. --- diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index c0d7d8b..2c1342b 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -174,2 +174,3 @@ pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { unsafe fn dealloc_nonnull(ptr: NonNull, layout: Layout) { + crate::uv_vec_trace::record(b'f', ptr.as_ptr(), layout.size(), 0); unsafe { __rust_dealloc(ptr, layout.size(), layout.alignment()) } diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 5cf0507..7ddf783 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -276 +276,3 @@ pub mod __export { } + +mod uv_vec_trace; diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index 5d4ad3a..fce81e3 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -474,2 +474,3 @@ const impl RawVecInner { + crate::uv_vec_trace::record(b'a', ptr.cast::().as_ptr(), layout.size(), 0); // Allocators currently return a `NonNull<[u8]>` whose length @@ -528,3 +529,3 @@ const impl RawVecInner { // - other preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, len, elem_layout)? }; @@ -546,2 +547,3 @@ const impl RawVecInner { cap: usize, + len: usize, elem_layout: Layout, @@ -550,2 +552,7 @@ const impl RawVecInner { + let old_capacity = self.cap.as_inner() * elem_layout.size(); + let old_pointer = self.ptr.as_ptr(); + let ticket = crate::uv_vec_trace::runtime( + b'b', old_pointer, old_capacity, len * elem_layout.size(), 0 + ); let memory = if let Some((ptr, old_layout)) = unsafe { self.current_memory(elem_layout) } { @@ -562,2 +569,7 @@ const impl RawVecInner { + let (pointer, capacity) = match memory { + Ok(pointer) => (pointer.cast::().as_ptr(), new_layout.size()), + Err(_) => (old_pointer, old_capacity), + }; + crate::uv_vec_trace::runtime(b'e', pointer, capacity, 0, ticket); memory.map_err(const |_| AllocError { layout: new_layout, non_exhaustive: () }.into()) @@ -796,3 +808,3 @@ impl RawVecInner { // SAFETY: preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, len, elem_layout)? }; @@ -839,2 +851,3 @@ impl RawVecInner { if cap == 0 { + crate::uv_vec_trace::record(b'f', ptr.as_ptr(), layout.size(), 0); unsafe { self.alloc.deallocate(ptr, layout) }; @@ -844,3 +857,4 @@ impl RawVecInner { } else { - let ptr = unsafe { + let ticket = crate::uv_vec_trace::runtime(b'b', ptr.as_ptr(), layout.size(), usize::MAX, 0); + let result = unsafe { // Layout cannot overflow here because it would have @@ -851,4 +865,10 @@ impl RawVecInner { .shrink(ptr, layout, new_layout) - .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? + .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }) }; + let (pointer, size) = match result { + Ok(pointer) => (pointer.cast::().as_ptr(), cap * elem_layout.size()), + Err(_) => (ptr.as_ptr(), layout.size()), + }; + crate::uv_vec_trace::runtime(b'e', pointer, size, 0, ticket); + let ptr = result?; // SAFETY: if the allocation is valid, then the capacity is too @@ -874,2 +894,3 @@ const impl RawVecInner { if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { + crate::uv_vec_trace::record(b'f', ptr.as_ptr(), layout.size(), 0); unsafe { diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 256efc6..dafce3d 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -493,3 +493,3 @@ impl String { pub fn with_capacity(capacity: usize) -> String { - String { vec: Vec::with_capacity(capacity) } + String { vec: Vec::with_capacity(capacity) }.trace_new() } @@ -506,3 +506,3 @@ impl String { pub fn try_with_capacity(capacity: usize) -> Result { - Ok(String { vec: Vec::try_with_capacity(capacity)? }) + Ok(String { vec: Vec::try_with_capacity(capacity)? }.trace_new()) } @@ -570,3 +570,3 @@ impl String { match str::from_utf8(&vec) { - Ok(..) => Ok(String { vec }), + Ok(..) => Ok(String { vec }.trace_new()), Err(e) => Err(FromUtf8Error { bytes: vec, error: e }), @@ -983,3 +983,3 @@ impl String { pub unsafe fn from_raw_parts(buf: *mut u8, length: usize, capacity: usize) -> String { - unsafe { String { vec: Vec::from_raw_parts(buf, length, capacity) } } + unsafe { String { vec: Vec::from_raw_parts(buf, length, capacity) } }.trace_new() } @@ -1016,3 +1016,3 @@ impl String { pub unsafe fn from_utf8_unchecked(bytes: Vec) -> String { - String { vec: bytes } + String { vec: bytes }.trace_new() } @@ -1037,2 +1037,3 @@ impl String { pub const fn into_bytes(self) -> Vec { + self.vec.trace_size(b's'); self.vec @@ -1104,3 +1105,4 @@ impl String { pub fn push_str(&mut self, string: &str) { - self.vec.extend_from_slice(string.as_bytes()) + self.vec.extend_from_slice(string.as_bytes()); + self.vec.trace_size(b's'); } @@ -1117,3 +1119,5 @@ impl String { pub(crate) fn try_push_str(&mut self, string: &str) -> Result<(), TryReserveError> { - self.vec.try_extend_from_slice_of_bytes(string.as_bytes()) + self.vec.try_extend_from_slice_of_bytes(string.as_bytes())?; + self.vec.trace_size(b's'); + Ok(()) } @@ -1235,3 +1239,4 @@ impl String { pub fn reserve(&mut self, additional: usize) { - self.vec.reserve(additional) + self.vec.reserve(additional); + self.vec.trace_size(b's'); } @@ -1285,3 +1290,4 @@ impl String { pub fn reserve_exact(&mut self, additional: usize) { - self.vec.reserve_exact(additional) + self.vec.reserve_exact(additional); + self.vec.trace_size(b's'); } @@ -1320,3 +1326,5 @@ impl String { pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.vec.try_reserve(additional) + self.vec.try_reserve(additional)?; + self.vec.trace_size(b's'); + Ok(()) } @@ -1361,3 +1369,5 @@ impl String { pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.vec.try_reserve_exact(additional) + self.vec.try_reserve_exact(additional)?; + self.vec.trace_size(b's'); + Ok(()) } @@ -2107,2 +2117,3 @@ impl String { unsafe { self.as_mut_vec() }.splice(checked_range, replace_with.bytes()); + self.vec.trace_size(b's'); } @@ -2388,3 +2399,3 @@ impl Clone for String { fn clone(&self) -> Self { - String { vec: self.vec.clone() } + String { vec: self.vec.clone() }.trace_new() } @@ -2397,2 +2408,3 @@ impl Clone for String { self.vec.clone_from(&source.vec); + self.vec.trace_size(b's'); } @@ -2619,2 +2631,3 @@ impl Extend for String { self.vec.extend(iter.into_iter().map(|c| c.to_u8())); + self.vec.trace_size(b's'); } @@ -2624,2 +2637,3 @@ impl Extend for String { self.vec.push(c.to_u8()); + self.vec.trace_size(b's'); } @@ -2638,2 +2652,3 @@ impl<'a> Extend<&'a core::ascii::Char> for String { self.vec.push(c.to_u8()); + self.vec.trace_size(b's'); } @@ -3643 +3658,9 @@ impl From for String { } + +impl String { + #[inline] + fn trace_new(self) -> Self { + self.vec.trace_size(b's'); + self + } +} diff --git a/library/alloc/src/uv_vec_trace.rs b/library/alloc/src/uv_vec_trace.rs new file mode 100644 index 0000000..9275aca --- /dev/null +++ b/library/alloc/src/uv_vec_trace.rs @@ -0,0 +1,103 @@ +use core::cell::UnsafeCell; +use core::ffi::{c_char, c_int, c_void}; +use core::ptr; +use core::sync::atomic::{AtomicPtr, AtomicU64, Ordering::Relaxed}; + +struct Once(UnsafeCell); + +// SAFETY: Only pthread_once accesses the control word. +unsafe impl Sync for Once {} + +static INIT: Once = Once(UnsafeCell::new(0)); +static TRACE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); +static TICKET: AtomicU64 = AtomicU64::new(1); + +unsafe extern "C" { + fn pthread_once(control: *mut c_int, init: extern "C" fn()) -> c_int; + fn getenv(name: *const c_char) -> *const c_char; + fn fopen(path: *const c_char, mode: *const c_char) -> *mut c_void; + fn fwrite(data: *const c_void, size: usize, count: usize, file: *mut c_void) -> usize; + fn fflush(file: *mut c_void) -> c_int; + fn atexit(callback: extern "C" fn()) -> c_int; + fn _exit(status: c_int) -> !; +} + +extern "C" fn open_trace() { + // SAFETY: The name and mode are NUL-terminated. The launcher sets the path + // before starting the process and leaves it in the environment. + unsafe { + let path = getenv(c"UV_VEC_TRACE".as_ptr()); + if !path.is_null() { + let file = fopen(path, c"wb".as_ptr()); + if file.is_null() { + _exit(90); + } + TRACE.store(file, Relaxed); + if atexit(flush_trace) != 0 { + _exit(90); + } + } + } +} + +extern "C" fn flush_trace() { + // SAFETY: Only a successfully opened stream registers this callback. + unsafe { + if fflush(TRACE.load(Relaxed)) != 0 { + _exit(90); + } + } +} + +fn emit(operation: u8, pointer: *const u8, capacity: usize, value: usize, mut ticket: u64) -> u64 { + // SAFETY: INIT uses the x86-64 glibc pthread_once_t layout and initializer. + unsafe { pthread_once(INIT.0.get(), open_trace) }; + let file = TRACE.load(Relaxed); + if file.is_null() { + return 0; + } + if operation == b'b' { + ticket = TICKET.fetch_add(1, Relaxed); + } + let event = [pointer.addr() as u64, capacity as u64, value as u64, ticket, operation as u64]; + // SAFETY: The stream stays open. fwrite serializes the complete record + // against concurrent writers, and the stack array is valid for this call. + unsafe { + if fwrite(event.as_ptr().cast(), size_of_val(&event), 1, file) != 1 { + _exit(90); + } + } + ticket +} + +const fn skip(_op: u8, _pointer: *const u8, _capacity: usize, _value: usize, _ticket: u64) -> u64 { + 0 +} + +#[inline] +#[rustc_allow_const_fn_unstable(const_eval_select)] +pub(crate) const fn runtime(op: u8, pointer: *const u8, capacity: usize, value: usize, ticket: u64) -> u64 { + core::intrinsics::const_eval_select((op, pointer, capacity, value, ticket), skip, emit) +} + +#[inline] +pub(crate) const fn record(op: u8, pointer: *const u8, capacity: usize, value: usize) { + if capacity != 0 { + runtime(op, pointer, capacity, value, 0); + } +} + +#[inline] +pub(crate) const fn observe( + op: u8, + pointer: *const u8, + capacity: usize, + length: usize, + element_size: usize, + element_alignment: usize, +) { + if capacity != 0 { + let wide_alignment = ((element_alignment > 8) as u64) << 63; + runtime(op, pointer, capacity, length, element_size as u64 | wide_alignment); + } +} diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 21d88f9..7dec32a 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -837,2 +837,3 @@ impl Vec { let mut me = ManuallyDrop::new(self); + me.trace_size(b'x'); (me.as_mut_ptr(), me.len(), me.capacity()) @@ -968,3 +969,3 @@ const impl Vec { pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { - Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 } + Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 }.trace_new() } @@ -1037,3 +1038,3 @@ const impl Vec { ptr::write(end, value); - self.len = len + 1; + self.set_len(len + 1); // SAFETY: We just wrote a value to the pointer that will live the lifetime of the reference. @@ -1079,3 +1080,3 @@ impl Vec { pub fn try_with_capacity_in(capacity: usize, alloc: A) -> Result { - Ok(Vec { buf: RawVec::try_with_capacity_in(capacity, alloc)?, len: 0 }) + Ok(Vec { buf: RawVec::try_with_capacity_in(capacity, alloc)?, len: 0 }.trace_new()) } @@ -1195,3 +1196,3 @@ impl Vec { ); - unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } } + unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } }.trace_new() } @@ -1310,3 +1311,3 @@ impl Vec { ); - unsafe { Vec { buf: RawVec::from_nonnull_in(ptr, capacity, alloc), len: length } } + unsafe { Vec { buf: RawVec::from_nonnull_in(ptr, capacity, alloc), len: length } }.trace_new() } @@ -1359,2 +1360,3 @@ impl Vec { let alloc = unsafe { ptr::read(me.allocator()) }; + me.trace_size(b'x'); (ptr, len, capacity, alloc) @@ -1460,3 +1462,7 @@ impl Vec { pub fn reserve(&mut self, additional: usize) { + let old_capacity = self.capacity(); self.buf.reserve(self.len, additional); + if self.capacity() != old_capacity { + self.trace_size(b'v'); + } } @@ -1490,3 +1496,7 @@ impl Vec { pub fn reserve_exact(&mut self, additional: usize) { + let old_capacity = self.capacity(); self.buf.reserve_exact(self.len, additional); + if self.capacity() != old_capacity { + self.trace_size(b'v'); + } } @@ -1527,3 +1537,8 @@ impl Vec { pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve(self.len, additional) + let old_capacity = self.capacity(); + self.buf.try_reserve(self.len, additional)?; + if self.capacity() != old_capacity { + self.trace_size(b'v'); + } + Ok(()) } @@ -1570,3 +1585,8 @@ impl Vec { pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve_exact(self.len, additional) + let old_capacity = self.capacity(); + self.buf.try_reserve_exact(self.len, additional)?; + if self.capacity() != old_capacity { + self.trace_size(b'v'); + } + Ok(()) } @@ -1725,2 +1745,3 @@ impl Vec { self.shrink_to_fit(); + self.trace_size(b'x'); let me = ManuallyDrop::new(self); @@ -1827,3 +1848,3 @@ impl Vec { let s = self.as_mut_ptr().add(len).cast_slice(remaining_len); - self.len = len; + self.set_len(len); ptr::drop_in_place(s); @@ -2224,2 +2245,3 @@ impl Vec { self.len = new_len; + self.trace_size(b'v'); } @@ -2842,3 +2864,3 @@ impl Vec { ptr::write(end, value); - self.len += 1; + self.set_len(self.len + 1); @@ -2876,3 +2898,3 @@ impl Vec { unsafe { - self.len -= 1; + self.set_len(self.len - 1); core::hint::assert_unchecked(self.len < self.capacity()); @@ -2985,3 +3007,3 @@ impl Vec { } - self.len += count; + unsafe { self.set_len(self.len + count) }; } @@ -3081,3 +3103,3 @@ impl Vec { unsafe { - self.len = 0; + unsafe { self.set_len(0) }; ptr::drop_in_place(elems); @@ -3265,2 +3287,3 @@ impl Vec { let mut me = ManuallyDrop::new(self); + me.trace_size(b'x'); unsafe { slice::from_raw_parts_mut(me.as_mut_ptr(), me.len) } @@ -3720,3 +3743,3 @@ impl Vec { // don't alias. - let mut local_len = SetLenOnDrop::new(&mut self.len); + let mut local_len = SetLenOnDrop::new(self); @@ -3836,3 +3859,3 @@ impl ExtendFromWithinSpec for Vec { // - The elements were just initialized by `copy_nonoverlapping` - self.len += count; + unsafe { self.set_len(self.len + count) }; } @@ -4023,2 +4046,3 @@ impl IntoIterator for Vec { unsafe { + self.trace_size(b'x'); let me = ManuallyDrop::new(self); @@ -4130,3 +4154,3 @@ impl Vec { let ptr = self.as_mut_ptr(); - let mut local_len = SetLenOnDrop::new(&mut self.len); + let mut local_len = SetLenOnDrop::new(self); iterator.for_each(move |element| { @@ -4353,2 +4377,3 @@ const unsafe impl<#[may_dangle] T: [const] Destruct, A: [const] Allocator + [con fn drop(&mut self) { + self.trace_size(b'x'); unsafe { @@ -4619 +4644,23 @@ const impl Vec { + #[inline] + pub(crate) const fn trace_size(&self, operation: u8) { + if !T::IS_ZST { + crate::uv_vec_trace::observe( + operation, + self.buf.ptr().cast(), + self.buf.capacity() * size_of::(), + self.len * size_of::(), + size_of::(), + align_of::(), + ); + } + } + + #[inline] + const fn trace_new(self) -> Self { + self.trace_size(b'v'); + self + } +} diff --git a/library/alloc/src/vec/set_len_on_drop.rs b/library/alloc/src/vec/set_len_on_drop.rs index 6ce5a3a..2905aaf 100644 --- a/library/alloc/src/vec/set_len_on_drop.rs +++ b/library/alloc/src/vec/set_len_on_drop.rs @@ -8,2 +8,6 @@ pub(super) struct SetLenOnDrop<'a> { local_len: usize, + pointer: *const u8, + capacity: usize, + element_size: usize, + element_alignment: usize, } @@ -12,4 +16,6 @@ impl<'a> SetLenOnDrop<'a> { #[inline] - pub(super) fn new(len: &'a mut usize) -> Self { - SetLenOnDrop { local_len: *len, len } + pub(super) fn new(vec: &'a mut super::Vec) -> Self { + let capacity = if size_of::() == 0 { 0 } else { vec.capacity() * size_of::() }; + SetLenOnDrop { local_len: vec.len, len: &mut vec.len, pointer: vec.buf.ptr().cast(), + capacity, element_size: size_of::(), element_alignment: align_of::() } } @@ -31,2 +37,6 @@ impl Drop for SetLenOnDrop<'_> { *self.len = self.local_len; + crate::uv_vec_trace::observe( + b'v', self.pointer, self.capacity, self.local_len * self.element_size, + self.element_size, self.element_alignment, + ); } diff --git a/library/alloc/src/vec/spec_from_elem.rs b/library/alloc/src/vec/spec_from_elem.rs index 96d701e..173fb13 100644 --- a/library/alloc/src/vec/spec_from_elem.rs +++ b/library/alloc/src/vec/spec_from_elem.rs @@ -23,3 +23,3 @@ impl SpecFromElem for T { if elem.is_zero() { - return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }.trace_new(); } @@ -35,3 +35,3 @@ impl SpecFromElem for i8 { if elem == 0 { - return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }.trace_new(); } @@ -50,3 +50,3 @@ impl SpecFromElem for u8 { if elem == 0 { - return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }.trace_new(); } diff --git a/library/alloc/src/vec/splice.rs b/library/alloc/src/vec/splice.rs index 99ebcb4..5557c30 100644 --- a/library/alloc/src/vec/splice.rs +++ b/library/alloc/src/vec/splice.rs @@ -116,3 +116,3 @@ impl Drain<'_, T, A> { unsafe { vec.as_mut_ptr().add(idx).write(new_item) }; - vec.len += 1; + unsafe { vec.set_len(vec.len + 1) }; }