From 317fb075ae9478b13590907ac8757b169c430208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 02:09:59 +0000 Subject: [PATCH 1/6] gc: back GC arena blocks with OS regions Linux arena blocks now come straight from mmap in aligned regions, pooled per agent, instead of from the Rust global allocator. mimalloc keeps native allocations only. Other platforms keep std::alloc. --- crates/perry-runtime/src/arena/block.rs | 88 +++------ .../src/arena/block/reuse_window_tests.rs | 6 +- crates/perry-runtime/src/arena/mod.rs | 2 +- crates/perry-runtime/src/arena/region.rs | 185 ++++++++++++++++++ crates/perry-runtime/src/arena/tests.rs | 18 +- .../src/ffi/perry_memory_profile.c | 26 ++- .../src/gc/tests/census_block_windows.rs | 15 +- .../src/startup_memory_profile.rs | 26 +-- 8 files changed, 264 insertions(+), 102 deletions(-) create mode 100644 crates/perry-runtime/src/arena/region.rs diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 00f51668f3..5b869044fc 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -1,45 +1,9 @@ use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; -/// Size of each arena block (1 MB — issue #179 tier 1 #1). -/// -/// Formerly 8 MB. The recent-5-blocks safety window (where LLVM caller- -/// saved registers might still hold uncaptured handles; see -/// `BLOCK_PERSIST_WINDOW` in gc.rs and `keep_low` in -/// `arena_reset_empty_blocks`) now reserves 5 × 1 MB = 5 MB of -/// non-reclaimable headroom instead of 5 × 8 MB = 40 MB. Combined with -/// the age-restricted block-persist from v0.5.193 this closes the -/// remaining `bench_json_roundtrip` RSS gap to within 5% of Node's -/// numbers without a speed regression. -/// -/// Measured on `bench_json_roundtrip` (best-of-5, macOS ARM64): -/// 8 MB blocks (v0.5.193): 384 ms / 213 MB -/// 2 MB blocks: 325 ms / 208 MB -/// 1 MB blocks: 320 ms / 199 MB -/// 512 KB blocks: 318 ms / 200 MB (diminishing returns) -/// -/// Picked 1 MB: RSS essentially tied with 512 KB, block-count overhead -/// 2× smaller, `bench_gc_pressure` / `object_create` unchanged. -/// -/// Trade-offs: -/// - More blocks in the arena for the same total bytes → walker loops -/// pay more per-block overhead. Measured: negligible — the walker is -/// O(objects), not O(blocks), once inside a block. -/// - More frequent "block full, advance to next" transitions in the -/// inline bump allocator's slow path. The slow path is a function -/// call; on `object_create` the cost is amortized across hundreds of -/// thousands of allocs per block before GC resets it. Measured: -/// `07_object_create` 0-1 ms unchanged. -/// - Large single allocations (Buffer.alloc(3 MB), big arena strings) -/// get a custom-sized block via `alloc_block(min_size)` that rounds -/// up to a BLOCK_SIZE multiple — unchanged mechanics, just rounds to -/// 1 MB granularity now. -/// - The GC's adaptive step (gc.rs `GC_THRESHOLD_INITIAL_BYTES = 128 -/// MB`) is unchanged; the workload still needs 128 MB of total arena -/// to trigger the first GC. With 1 MB blocks that's 128 blocks, and -/// `bench_json_roundtrip` hits that point at roughly the same -/// iteration as it did with 16 × 8 MB blocks — the adaptive step -/// shrinks appropriately on the first productive collection. +/// Keep main's usable block geometry for the allocation experiment. The OS +/// extent is 2 MiB-aligned; 2 MiB usable blocks require hwp's census-capacity +/// integration before they can run (see REGION-DESIGN). pub(crate) const BLOCK_SIZE: usize = 1024 * 1024; pub(crate) const FRESH_GENERAL_BLOCK_MIN_USED_BYTES: usize = 256 * 1024; @@ -124,13 +88,12 @@ impl Drop for BlockPool { if data.is_null() || size == 0 { continue; } - let layout = Layout::from_size_align(size, 16).unwrap(); unsafe { // #4665, mirroring `Arena::drop`: test builds keep freed blocks // mapped so unit tests holding raw GC pointers across a // collection read stale bytes instead of faulting. if !cfg!(test) { - std::alloc::dealloc(data, layout); + super::region::unmap(data, size); } } } @@ -299,13 +262,12 @@ pub(crate) fn release_arena_block(data: *mut u8, size: usize) -> ArenaBlockRelea return ArenaBlockRelease::Pooled; } if !data.is_null() && size != 0 { - let layout = Layout::from_size_align(size, 16).unwrap(); unsafe { // #4665: test builds retain otherwise-freed mappings so stale raw // GC pointers remain readable. The production disposition is still // Deallocated; focused pool tests observe the explicit drain census. if !cfg!(test) { - std::alloc::dealloc(data, layout); + super::region::unmap(data, size); } } } @@ -330,10 +292,9 @@ pub(crate) fn drain_block_pool() -> BlockPoolDrainStats { if data.is_null() || size == 0 { continue; } - let layout = Layout::from_size_align(size, 16).unwrap(); unsafe { if !cfg!(test) { - std::alloc::dealloc(data, layout); + super::region::unmap(data, size); } } } @@ -373,16 +334,21 @@ pub(crate) fn block_pool_explicit_drained_bytes_for_test() -> usize { BLOCK_POOL_EXPLICIT_DRAINED_BYTES.load(Ordering::Relaxed) } -fn try_alloc_block(min_size: usize, injectable: bool) -> Option { +fn try_alloc_block( + min_size: usize, + injectable: bool, + generation: HeapGeneration, +) -> Option { let size = block_size_for(min_size); - let layout = Layout::from_size_align(size, 16).unwrap(); #[cfg(test)] if injectable && FORCE_BLOCK_ALLOC_FAILURE.with(|f| f.replace(false)) { return None; } #[cfg(not(test))] let _ = injectable; + let kind = super::region::kind_for(generation, size); if let Some(data) = block_pool_take(size) { + unsafe { super::region::advise(data, size, kind) }; return Some(ArenaBlock { data, size, @@ -394,7 +360,7 @@ fn try_alloc_block(min_size: usize, injectable: bool) -> Option { idle_pages_discarded: false, }); } - let data = unsafe { alloc(layout) }; + let data = unsafe { super::region::map(kind, size) }; if data.is_null() { return None; } @@ -430,13 +396,18 @@ pub(crate) fn new_object_start_bitmap(size: usize) -> Box<[u64]> { /// violation [`arena_cell_alloc`] exists to avoid, on the out-of-memory path. /// `arena_cell_alloc` is the only caller; every `&mut self` path uses /// [`alloc_block_no_gc`]. +#[cfg(test)] pub(crate) fn reserve_arena_block(min_size: usize) -> ArenaBlock { - if let Some(block) = try_alloc_block(min_size, true) { + reserve_arena_block_for(min_size, HeapGeneration::Nursery) +} + +fn reserve_arena_block_for(min_size: usize, generation: HeapGeneration) -> ArenaBlock { + if let Some(block) = try_alloc_block(min_size, true, generation) { return block; } note_gc_trigger_arena_borrow_depth(); crate::gc::gc_try_emergency_reclaim(); - if let Some(block) = try_alloc_block(min_size, false) { + if let Some(block) = try_alloc_block(min_size, false, generation) { return block; } panic!( @@ -452,8 +423,8 @@ pub(crate) fn reserve_arena_block(min_size: usize) -> ArenaBlock { /// where starting another one is precisely what must not happen. The mutator /// allocation path — the one where heap exhaustion actually surfaces — keeps the /// reclaim via [`reserve_arena_block`]. -fn alloc_block_no_gc(min_size: usize) -> ArenaBlock { - try_alloc_block(min_size, false).unwrap_or_else(|| { +fn alloc_block_no_gc(min_size: usize, generation: HeapGeneration) -> ArenaBlock { + try_alloc_block(min_size, false, generation).unwrap_or_else(|| { panic!( "Failed to allocate arena block of {} bytes (heap exhausted)", block_size_for(min_size) @@ -501,8 +472,8 @@ pub(crate) struct ArenaBlock { impl ArenaBlock { /// The initial block of a thread's arena, built during `Arena::new` — i.e. /// while the arena does not exist yet, so nothing may collect here. - fn new() -> Self { - alloc_block_no_gc(BLOCK_SIZE) + fn new(generation: HeapGeneration) -> Self { + alloc_block_no_gc(BLOCK_SIZE, generation) } #[inline] @@ -644,13 +615,12 @@ impl Drop for Arena { block.data as usize, block.data as usize + block.size, ); - let layout = std::alloc::Layout::from_size_align(block.size, 16).unwrap(); unsafe { // #4665: in test builds keep freed blocks mapped (no munmap) so // unit tests holding raw GC pointers across a collection read stale // bytes instead of SIGSEGV-ing on an unmapped page. if !cfg!(test) { - std::alloc::dealloc(block.data, layout); + super::region::unmap(block.data, block.size); } } } @@ -669,7 +639,7 @@ impl Arena { } fn new(generation: HeapGeneration, space: HeapSpace) -> Self { - let initial = ArenaBlock::new(); + let initial = ArenaBlock::new(generation); register_block_space_with_object_starts( initial.data as usize, initial.size, @@ -804,7 +774,7 @@ impl Arena { /// Reserve **and** install a block. Never collects — see /// [`alloc_block_no_gc`] for why. pub(crate) fn install_fresh_block(&mut self, size: usize) { - self.install_reserved_block(alloc_block_no_gc(size)); + self.install_reserved_block(alloc_block_no_gc(size, self.generation)); } /// Install a block that was reserved with no arena borrow live (#7022). @@ -1050,7 +1020,7 @@ pub(crate) unsafe fn arena_cell_alloc(arena: *mut Arena, size: usize, align: usi // NO ARENA BORROW IS LIVE HERE either. (CodeRabbit caught this second path // on the first cut of #7022, where the reservation still happened inside // `alloc_fresh_block` under the borrow.) - let fresh = reserve_arena_block(size); + let fresh = reserve_arena_block_for(size, (*arena).generation); let _borrow = ArenaBorrowGuard::new(); (*arena).install_reserved_block(fresh); diff --git a/crates/perry-runtime/src/arena/block/reuse_window_tests.rs b/crates/perry-runtime/src/arena/block/reuse_window_tests.rs index 4417eca14c..68b477f074 100644 --- a/crates/perry-runtime/src/arena/block/reuse_window_tests.rs +++ b/crates/perry-runtime/src/arena/block/reuse_window_tests.rs @@ -4,7 +4,7 @@ use super::*; fn reused_blocks_restart_the_window_and_cold_blocks_are_advised_once() { crate::arena::tests::run_with_fresh_arenas(|| unsafe { let size = BLOCK_SIZE; - let raw = alloc(Layout::from_size_align(size, 16).unwrap()); + let raw = crate::arena::region::map(crate::arena::region::Kind::NurseryBlock, size); assert!(!raw.is_null()); assert!(block_pool_put(raw, size)); assert_eq!(advance_block_pool_reuse_window(), 0); @@ -23,7 +23,7 @@ fn reused_blocks_restart_the_window_and_cold_blocks_are_advised_once() { } unsafe fn dealloc_for_test(raw: *mut u8, size: usize) { - std::alloc::dealloc(raw, Layout::from_size_align(size, 16).unwrap()); + crate::arena::region::unmap(raw, size); } #[cfg(target_os = "linux")] @@ -48,7 +48,7 @@ fn real_collection_publication_keeps_warm_pages_and_releases_unused_pages() { // size: unrelated initial 1 MiB arenas cannot consume this 2 MiB entry. crate::gc::js_gc_collect(); let size = 2 * BLOCK_SIZE; - let raw = alloc(Layout::from_size_align(size, 16).unwrap()); + let raw = crate::arena::region::map(crate::arena::region::Kind::NurseryBlock, size); assert!(!raw.is_null()); std::ptr::write_bytes(raw, 0xa5, size); let resident = resident_pages(raw, size); diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index a94281c547..3e8f8449ad 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -4,7 +4,6 @@ //! Objects allocated here are not individually freed - the entire arena //! can be reset at once (e.g., at end of program or during GC). -pub(crate) use std::alloc::{alloc, Layout}; pub(crate) use std::cell::{Cell, RefCell, UnsafeCell}; pub(crate) use std::collections::hash_map::Entry; @@ -13,6 +12,7 @@ mod allocators; mod block; mod construction; mod from_space; +mod region; pub(crate) use construction::ConstructionBatch; mod inline; mod map_allocations; diff --git a/crates/perry-runtime/src/arena/region.rs b/crates/perry-runtime/src/arena/region.rs new file mode 100644 index 0000000000..203c2b9c30 --- /dev/null +++ b/crates/perry-runtime/src/arena/region.rs @@ -0,0 +1,185 @@ +//! Measurement prototype: OS backing beneath the existing per-agent block pool. +//! No pointer registry, cache, collector callback or object-layout change. +//! Production in-band descriptors and byte-store ownership are in REGION-DESIGN. + +use super::HeapGeneration; + +pub(super) const ALIGN: usize = 2 * 1024 * 1024; + +#[derive(Clone, Copy)] +pub(super) enum Kind { + NurseryBlock, + OldBlock, + LargeObject, +} + +pub(super) fn kind_for(generation: HeapGeneration, len: usize) -> Kind { + match generation { + HeapGeneration::Old | HeapGeneration::Longlived if len > ALIGN => Kind::LargeObject, + HeapGeneration::Old | HeapGeneration::Longlived => Kind::OldBlock, + _ => Kind::NurseryBlock, + } +} + +fn mapped_len(len: usize) -> Option { + if len == 0 || len % 4096 != 0 { + return None; + } + len.checked_add(ALIGN - 1).map(|n| n & !(ALIGN - 1)) +} + +/// Mature complete payload units are THP eligible. Nursery and sparse tail +/// units are not. Advice precedes writes; neither prefault nor collapse. +pub(super) unsafe fn advise(data: *mut u8, len: usize, kind: Kind) { + #[cfg(target_os = "linux")] + { + let mapped = mapped_len(len).expect("invalid region extent"); + assert_eq!(libc::madvise(data.cast(), mapped, libc::MADV_NOHUGEPAGE), 0); + if !matches!(kind, Kind::NurseryBlock) { + let dense = len / ALIGN * ALIGN; + if dense != 0 { + assert_eq!(libc::madvise(data.cast(), dense, libc::MADV_HUGEPAGE), 0); + } + } + } + #[cfg(not(target_os = "linux"))] + let _ = (data, len, kind); +} + +/// Fresh private anonymous backing, or null on refusal. This entry never +/// collects. The existing caller owns emergency reclaim outside arena borrows. +pub(super) unsafe fn map(kind: Kind, len: usize) -> *mut u8 { + #[cfg(target_os = "linux")] + { + let Some(mapped) = mapped_len(len) else { + return std::ptr::null_mut(); + }; + let Some(reserved) = mapped.checked_add(ALIGN) else { + return std::ptr::null_mut(); + }; + let raw = libc::mmap( + std::ptr::null_mut(), + reserved, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ); + if raw == libc::MAP_FAILED { + return std::ptr::null_mut(); + } + let address = raw as usize; + let aligned = (address + ALIGN - 1) & !(ALIGN - 1); + let prefix = aligned - address; + let suffix = reserved - prefix - mapped; + if prefix != 0 { + assert_eq!(libc::munmap(raw, prefix), 0); + } + if suffix != 0 { + assert_eq!(libc::munmap((aligned + mapped) as *mut _, suffix), 0); + } + let data = aligned as *mut u8; + advise(data, len, kind); + data + } + #[cfg(not(target_os = "linux"))] + { + let _ = kind; + std::alloc::alloc(std::alloc::Layout::from_size_align(len, 16).unwrap()) + } +} + +/// Physical destruction, used by pool overflow, pool drain and TLS teardown. +/// Callers remove page metadata and external owners before destruction. +pub(super) unsafe fn unmap(data: *mut u8, len: usize) { + #[cfg(target_os = "linux")] + assert_eq!( + libc::munmap(data.cast(), mapped_len(len).expect("invalid extent")), + 0, + "region unmap failed" + ); + #[cfg(not(target_os = "linux"))] + std::alloc::dealloc(data, std::alloc::Layout::from_size_align(len, 16).unwrap()); +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + fn flags_for(data: *mut u8) -> String { + let smaps = std::fs::read_to_string("/proc/self/smaps").unwrap(); + let mut found = false; + for line in smaps.lines() { + if let Some(range) = line.split_whitespace().next().filter(|v| v.contains('-')) { + let (start, end) = range.split_once('-').unwrap(); + if let (Ok(start), Ok(end)) = ( + usize::from_str_radix(start, 16), + usize::from_str_radix(end, 16), + ) { + found = (start..end).contains(&(data as usize)); + } + } + if found && line.starts_with("VmFlags:") { + return line.to_owned(); + } + } + panic!("region missing from smaps"); + } + + #[test] + fn aligned_os_backing_has_kind_specific_thp_and_releases_pages() { + unsafe { + for kind in [Kind::NurseryBlock, Kind::OldBlock, Kind::LargeObject] { + let data = map(kind, 2 * ALIGN); + assert!(!data.is_null()); + assert_eq!(data as usize & (ALIGN - 1), 0); + let flags = flags_for(data); + let expected = if matches!(kind, Kind::NurseryBlock) { + "nh" + } else { + "hg" + }; + assert!(flags.split_whitespace().any(|v| v == expected), "{flags}"); + std::ptr::write_bytes(data, 0xa5, 2 * ALIGN); + let mut resident = vec![0u8; 2 * ALIGN / 4096]; + assert_eq!( + libc::mincore(data.cast(), 2 * ALIGN, resident.as_mut_ptr()), + 0 + ); + assert!(resident.iter().all(|v| v & 1 == 1)); + assert_eq!( + libc::madvise(data.cast(), 2 * ALIGN, libc::MADV_DONTNEED), + 0 + ); + assert_eq!( + libc::mincore(data.cast(), 2 * ALIGN, resident.as_mut_ptr()), + 0 + ); + assert!(resident.iter().all(|v| v & 1 == 0)); + *data = 42; + assert_eq!(*data, 42); + unmap(data, 2 * ALIGN); + assert_eq!(libc::mincore(data.cast(), 4096, resident.as_mut_ptr()), -1); + } + } + } + + #[test] + fn reused_region_changes_advice_before_new_birth() { + unsafe { + let data = map(Kind::OldBlock, ALIGN); + advise(data, ALIGN, Kind::NurseryBlock); + assert!(flags_for(data).split_whitespace().any(|v| v == "nh")); + unmap(data, ALIGN); + } + } + + #[test] + fn invalid_extents_are_refused_without_collection() { + unsafe { + assert!(map(Kind::NurseryBlock, 0).is_null()); + assert!(map(Kind::NurseryBlock, ALIGN - 1).is_null()); + assert!(map(Kind::NurseryBlock, usize::MAX & !(ALIGN - 1)).is_null()); + } + } +} diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index e5e11a4847..8a58ebee2c 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -1308,8 +1308,8 @@ fn emergency_block_reclaim_runs_with_no_live_arena_borrow() { /// high-water instead of cumulative promotion volume. #[test] fn recycled_block_pool_reuses_released_blocks() { - let layout = std::alloc::Layout::from_size_align(BLOCK_SIZE, 16).unwrap(); - let raw = unsafe { std::alloc::alloc(layout) }; + let raw = + unsafe { crate::arena::region::map(crate::arena::region::Kind::NurseryBlock, BLOCK_SIZE) }; assert!(!raw.is_null()); let before = block_pool_bytes_for_test(); assert!( @@ -1329,7 +1329,7 @@ fn recycled_block_pool_reuses_released_blocks() { assert_eq!(block.offset, 0); assert_eq!(block_pool_bytes_for_test(), before); // Hand it back to the allocator so the test doesn't leak the mapping. - unsafe { std::alloc::dealloc(block.data, layout) }; + unsafe { crate::arena::region::unmap(block.data, block.size) }; } /// A thread exiting with a non-empty pool must run the pool's own `Drop` @@ -1350,8 +1350,9 @@ fn block_pool_is_per_thread_and_drops_with_its_thread() { let handle = std::thread::spawn(|| { // Fresh thread => fresh pool. assert_eq!(block_pool_bytes_for_test(), 0); - let layout = std::alloc::Layout::from_size_align(BLOCK_SIZE, 16).unwrap(); - let raw = unsafe { std::alloc::alloc(layout) }; + let raw = unsafe { + crate::arena::region::map(crate::arena::region::Kind::NurseryBlock, BLOCK_SIZE) + }; assert!(!raw.is_null()); assert!( block_pool_put(raw, BLOCK_SIZE), @@ -1420,8 +1421,8 @@ fn block_pool_cap_is_process_wide_across_live_threads() { #[test] fn allocation_failure_recovery_drains_mismatched_pooled_blocks() { - let layout = std::alloc::Layout::from_size_align(BLOCK_SIZE, 16).unwrap(); - let raw = unsafe { std::alloc::alloc(layout) }; + let raw = + unsafe { crate::arena::region::map(crate::arena::region::Kind::NurseryBlock, BLOCK_SIZE) }; assert!(!raw.is_null()); assert!(block_pool_put(raw, BLOCK_SIZE)); @@ -1433,8 +1434,7 @@ fn allocation_failure_recovery_drains_mismatched_pooled_blocks() { "emergency full collection must drain blocks unusable for the failed size" ); - let returned_layout = std::alloc::Layout::from_size_align(block.size, 16).unwrap(); - unsafe { std::alloc::dealloc(block.data, returned_layout) }; + unsafe { crate::arena::region::unmap(block.data, block.size) }; } // --------------------------------------------------------------------------- diff --git a/crates/perry-runtime/src/ffi/perry_memory_profile.c b/crates/perry-runtime/src/ffi/perry_memory_profile.c index 41c419a2b2..f93fedbe5a 100644 --- a/crates/perry-runtime/src/ffi/perry_memory_profile.c +++ b/crates/perry-runtime/src/ffi/perry_memory_profile.c @@ -1,17 +1,27 @@ /* Use mimalloc's actual header: libmimalloc-sys 0.1.49 does not expose the * allow_thp enum member in Rust, and its numeric id is not a stable API. */ #include -#include -#include +#include +#include +#ifndef PR_THP_DISABLE_EXCEPT_ADVISED +#define PR_THP_DISABLE_EXCEPT_ADVISED (1UL << 1) +#endif -/* Run before mimalloc's default-priority constructor and Rust startup. - * getenv/strcmp/mi_option_set_default do not allocate. The default setter - * preserves mimalloc's own environment parser and explicit operator settings. */ +/* GC backing belongs to Perry regions. Ordinary native allocations use + * mimalloc with process-wide THP disabled before its startup constructor. + * Explicit allocator options cannot re-enable sparse native huge pages. */ __attribute__((constructor(101))) static void perry_apply_small_process_default(void) { - const char *profile = getenv("PERRY_MEMORY_PROFILE"); - if (profile != NULL && strcmp(profile, "small") == 0) { - mi_option_set_default(mi_option_allow_thp, 0); + mi_option_set(mi_option_allow_thp, 0); + /* mimalloc normally sets the full process disable bit. Establish the + * advised-only state first: its OS initialization observes a nonzero + * PR_GET_THP_DISABLE and leaves it intact. No allocator patch or heap + * allocation here. Older kernels reject this flag and retain the safe + * all-base-pages fallback when mimalloc initializes. */ + /* Preserve an inherited strict operator disable (also the all-THP-off + * measurement control). Never relax a parent's explicit process policy. */ + if (prctl(PR_GET_THP_DISABLE, 0UL, 0UL, 0UL, 0UL) != 1) { + (void)prctl(PR_SET_THP_DISABLE, 1UL, PR_THP_DISABLE_EXCEPT_ADVISED, 0UL, 0UL); } } diff --git a/crates/perry-runtime/src/gc/tests/census_block_windows.rs b/crates/perry-runtime/src/gc/tests/census_block_windows.rs index 626689beac..ab5de7d1d5 100644 --- a/crates/perry-runtime/src/gc/tests/census_block_windows.rs +++ b/crates/perry-runtime/src/gc/tests/census_block_windows.rs @@ -5,7 +5,8 @@ //! is queried at every block edge, inside every block, between blocks and far //! outside the heap, and each answer is compared with the binary search it //! replaces. The sabotaged twin ignores the base inside a window and the -//! comparison notices. A fabricated set whose bases share a window keeps the +//! comparison notices on deliberately unaligned input. A fabricated set whose +//! bases share a window keeps the //! binary search. use super::super::*; @@ -99,8 +100,16 @@ fn the_block_window_index_answers_exactly_like_the_fence_search() { #[test] fn sabotaged_block_window_lookup_is_caught_by_the_comparison() { run_isolated(|| { - unsafe { plant() }; - let valid = ValidPointerSetBuilder::new().finish(); + // OS regions start on a window boundary, where ignoring the base + // offset is equivalent. Keep this adversarial input independent of + // allocator placement; the preceding test covers the real heap. + let mut valid = ValidPointerSet::new(); + let base = 0x7000_0000_1000usize; + valid.begin_arena_block(0, base, 64 * 1024); + valid.push_arena(base + GC_HEADER_SIZE); + valid.build_block_windows(); + assert!(!valid.block_windows.is_empty()); + assert_eq!(mismatches(&valid).1, 0, "unaltered lookup must agree"); let (_, wrong) = { let _sabotage = block_window_sabotage::Guard::arm(); mismatches(&valid) diff --git a/crates/perry-runtime/src/startup_memory_profile.rs b/crates/perry-runtime/src/startup_memory_profile.rs index 6ac0a635bd..e3cbecfb06 100644 --- a/crates/perry-runtime/src/startup_memory_profile.rs +++ b/crates/perry-runtime/src/startup_memory_profile.rs @@ -1,9 +1,6 @@ -//! Opt-in policy for small Linux processes. Configure the allocator before -//! Rust startup can allocate; setting this from `js_gc_init` is too late. -//! -//! `mi_option_set_default` leaves mimalloc's own environment parsing intact, -//! including its lowercase option spelling. No Rust allocation, environment -//! mutation, or lock belongs on this constructor path. +//! Native mimalloc THP is disabled before Rust startup. Linux advised-only +//! process mode permits Perry's region advice on supporting kernels; older +//! kernels safely retain full THP disable. The constructor never allocates. #[cfg(all( target_os = "linux", @@ -32,16 +29,7 @@ mod linux { // The runner launches this test in a fresh process for each // profile/override. Rust's test harness has already allocated; // js_gc_init is deliberately never called here. - let profile = std::env::var("PERRY_MEMORY_PROFILE").ok(); - let explicit = std::env::var("MIMALLOC_ALLOW_THP") - .or_else(|_| std::env::var("mimalloc_allow_thp")) - .ok(); - let expected = match explicit.as_deref() { - Some("0") => 0, - Some("1") => 1, - None => i64::from(profile.as_deref() != Some("small")), - value => panic!("probe expects an absent, 0, or 1 override: {value:?}"), - }; + let expected = 0; let actual = unsafe { super::perry_memory_profile_allow_thp() }; assert_eq!(actual as i64, expected); // On Linux this is the process-wide effect of allow_thp=0. The @@ -50,9 +38,9 @@ mod linux { let disabled = unsafe { libc::prctl(libc::PR_GET_THP_DISABLE, 0, 0, 0, 0) }; assert!(disabled >= 0, "PR_GET_THP_DISABLE must be available"); if expected == 0 { - assert_eq!( - disabled, 1, - "THP policy must already be applied before gc_init" + assert!( + disabled == 3 || disabled == 1, + "THP policy must be advised-only or the old-kernel disabled fallback: {disabled}" ); } } From 8c9419ee3628043d0007436aed87e96d1c465088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 02:09:59 +0000 Subject: [PATCH 2/6] gc: derive census bitmap capacity from block size --- crates/perry-runtime/src/gc/trace.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 7fad7ff4e0..a52a0c7291 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -119,16 +119,17 @@ crate::perry_thread_local! { /// its whole extent would be mostly zero words. const CENSUS_BITMAP_MAX_EXTENT: usize = crate::arena::BLOCK_SIZE; -/// Start bitmaps are allocated in chunks of this many words (8 KiB), one or two -/// per block. A bitmap kept in one contiguous vector grew past the allocator's +/// Start bitmaps are allocated in chunks of this many words (8 KiB). A bitmap +/// kept in one contiguous vector grew past the allocator's /// small and medium size classes and made it commit a fresh large page: /// `records_array_1m:sparse` read +4.8 MiB peak RSS for a ~100 KB index. The /// start runs this replaces were 8 KiB vectors too. const CENSUS_BITMAP_CHUNK_WORDS: usize = 1024; const CENSUS_BITMAP_CHUNK_WORD_SHIFT: u32 = 10; /// Chunks a bitmap block can need: `CENSUS_BITMAP_MAX_EXTENT` bytes at one bit -/// per 8 bytes is `2 * CENSUS_BITMAP_CHUNK_WORDS` words. -const CENSUS_BITMAP_MAX_CHUNKS: usize = 2; +/// per 8 bytes, with 64 bits per word and chunked word storage. +const CENSUS_BITMAP_MAX_CHUNKS: usize = + CENSUS_BITMAP_MAX_EXTENT.div_ceil(8 * 64 * CENSUS_BITMAP_CHUNK_WORDS); /// Arena object starts sit on 8-byte boundaries relative to their block's /// `data` pointer: `ArenaObjectCursor::next_budgeted` rounds every header From 464add9ceb506c60e147e483c39c9f34582ec38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 02:09:59 +0000 Subject: [PATCH 3/6] gc: a parse bump never lowers the armed whole-arena trigger The bump compared against the effective trigger, which is clamped to the nursery cap, so it could lower an armed whole-arena threshold and re-arm a minor on every small parse. Compare with the whole-arena threshold itself. --- crates/perry-runtime/src/gc/policy.rs | 9 ++-- crates/perry-runtime/src/gc/tests/triggers.rs | 48 +++++++++++++++---- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 3521bc4eff..b16a3e2b2d 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -2802,10 +2802,11 @@ pub(super) fn gc_bump_malloc_trigger_with_snapshot(current: usize, bytes_now: us // Only raise — never lower — so this can't accidentally trip a // pending collection that the existing trigger had already armed. GC_NEXT_TRIGGER_BYTES.with(|c| { - // Compare against the effective (budget-clamped) trigger, not the - // raw cell: on a small-budget device the cell's un-armed default - // (128 MB) would otherwise swallow every legitimate parse bump. - if bytes_trigger > effective_next_arena_trigger() { + // Compare whole-arena bytes with their own threshold. The effective + // trigger also clamps to the young-generation cap; comparing with + // that smaller value could LOWER an armed whole-arena threshold. + // Keep the device-budget clamp for an unarmed initializer. + if bytes_trigger > next_arena_trigger_base() { c.set(bytes_trigger); GC_TRIGGER_ARMED.with(|a| a.set(true)); if !is_tiny_parse { diff --git a/crates/perry-runtime/src/gc/tests/triggers.rs b/crates/perry-runtime/src/gc/tests/triggers.rs index 9ef3efee2f..4ffef5f9a3 100644 --- a/crates/perry-runtime/src/gc/tests/triggers.rs +++ b/crates/perry-runtime/src/gc/tests/triggers.rs @@ -157,15 +157,7 @@ fn test_gc_bump_medium_parse_allows_one_arena_bump_per_gc_cycle() { #[test] fn test_gc_bump_never_lowers_existing_arena_trigger() { - // The "never lower" invariant is asserted against the RAW trigger cell, whose - // relationship to the bump target only holds under legacy pacing: with moving - // mode on, `effective_next_arena_trigger` is clamped to the small nursery cap, - // so a bump target above that cap legitimately re-arms the cell (without ever - // lowering the EFFECTIVE, capped trigger). Pin legacy to keep asserting the - // raw-cell arithmetic this test was written for. (The new nursery-cap value - // itself is asserted under the default by - // `test_effective_arena_trigger_respects_armed_values`.) - let _legacy_pacing = crate::gc::policy::force_legacy_gc_pacing(); + let _moving_pacing = crate::gc::policy::force_moving_gc_pacing(); let existing_trigger = GC_TRIGGER_ABSOLUTE_CEILING + (32 * 1024 * 1024); let _guard = GcBumpTriggerTestGuard::new(existing_trigger, GC_THRESHOLD_INITIAL_BYTES); let bytes_now = GC_TRIGGER_ABSOLUTE_CEILING + (16 * 1024 * 1024); @@ -180,6 +172,44 @@ fn test_gc_bump_never_lowers_existing_arena_trigger() { assert!(!GcBumpTriggerTestGuard::trigger_bumped()); } +/// Fastify's tiny request parses keep several nearly full arenas alive. +/// Rounding each arena to a block moves the total across the 128 MiB ceiling +/// with 2 MiB blocks, without allocating another byte of live request data. +/// Parse completion must preserve the headroom armed by the previous minor. +#[test] +fn tiny_request_churn_preserves_minor_headroom_at_both_block_geometries() { + const MIB: usize = 1024 * 1024; + let _moving = crate::gc::policy::force_moving_gc_pacing(); + let armed_before = GC_TRIGGER_ARMED.with(|a| a.replace(true)); + let counts: Vec = [MIB, 2 * MIB] + .into_iter() + .map(|block| { + let _cells = GcBumpTriggerTestGuard::new(144 * MIB, 128 * MIB); + // Four independent arenas, each containing 30.5 MiB of data. + let rounded = (30 * MIB + MIB / 2).div_ceil(block) * block; + let total = 4 * rounded; + let mut minors = 0; + for _request in 0..1000 { + GcBumpTriggerTestGuard::set_pre_suppress(total); + assert!(gc_bump_malloc_trigger_with_snapshot(0, total)); + if total >= crate::gc::policy::next_arena_trigger_base() { + minors += 1; + GC_NEXT_TRIGGER_BYTES.with(|t| t.set(total + 16 * MIB)); + } + } + // A real 16 MiB of later allocation still crosses the boundary. + assert!(144 * MIB >= crate::gc::policy::next_arena_trigger_base()); + minors + }) + .collect(); + GC_TRIGGER_ARMED.with(|a| a.set(armed_before)); + assert_eq!( + counts, + [0, 0], + "mapped-capacity rounding must not rearm minors" + ); +} + /// #10928: the old-reclaim rule is PROPORTIONAL ONLY. /// /// **Replaces `test_old_reclaim_pressure_uses_threshold_and_growth`**, and From b388e94aa0b3c121234462bca528c82561a16e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 02:10:29 +0000 Subject: [PATCH 4/6] gc: two MiB GC blocks; huge pages chosen by complete units Linux GC blocks are 2 MiB in 2 MiB-aligned regions. Page size follows how much of a unit is used, not object lifetime: complete 2 MiB units get MADV_HUGEPAGE in every generation, sub-unit extents and rounded tails stay on base pages. The defrag selector decides from the allocation kind stored in the block, not a size threshold. --- changelog.d/gc-region-manager-2mib.md | 9 ++ crates/perry-runtime/src/arena/block.rs | 29 +++-- .../src/arena/block/reuse_window_tests.rs | 21 ++++ crates/perry-runtime/src/arena/mod.rs | 23 ++-- .../perry-runtime/src/arena/page_meta/mod.rs | 16 ++- crates/perry-runtime/src/arena/promote.rs | 13 +- crates/perry-runtime/src/arena/quarantine.rs | 3 + crates/perry-runtime/src/arena/region.rs | 117 +++++++++++++++--- .../src/ffi/perry_memory_profile.c | 4 +- .../src/gc/large_extent_selection_tests.rs | 92 ++++++++++++++ crates/perry-runtime/src/gc/oldgen_defrag.rs | 41 +++++- .../src/gc/tests/adopt_census.rs | 4 +- .../src/gc/tests/census_two_mib.rs | 37 ++++++ .../perry-runtime/src/gc/tests/debt_pacer.rs | 2 +- .../perry-runtime/src/gc/tests/evacuation.rs | 18 ++- .../src/gc/tests/incremental_sweep_reclaim.rs | 2 +- crates/perry-runtime/src/gc/tests/mod.rs | 2 + .../src/gc/tests/native_payload_streams.rs | 25 ++-- crates/perry-runtime/src/gc/tests/oldgen.rs | 16 ++- .../src/gc/tests/runtime_roots.rs | 2 +- .../src/gc/tests/sweep_window_births.rs | 2 +- .../src/startup_memory_profile.rs | 9 ++ .../_helpers/region_transfer_sender_exit.ts | 8 ++ .../test_gap_region_transfer_sender_exit.ts | 24 ++++ 24 files changed, 452 insertions(+), 67 deletions(-) create mode 100644 changelog.d/gc-region-manager-2mib.md create mode 100644 crates/perry-runtime/src/gc/large_extent_selection_tests.rs create mode 100644 crates/perry-runtime/src/gc/tests/census_two_mib.rs create mode 100644 test-files/_helpers/region_transfer_sender_exit.ts create mode 100644 test-files/test_gap_region_transfer_sender_exit.ts diff --git a/changelog.d/gc-region-manager-2mib.md b/changelog.d/gc-region-manager-2mib.md new file mode 100644 index 0000000000..1763ebc29f --- /dev/null +++ b/changelog.d/gc-region-manager-2mib.md @@ -0,0 +1,9 @@ +Use 2 MiB usable Linux GC blocks in 2 MiB-aligned OS regions. Derive census +bitmap capacity from block geometry and advise nursery backing as huge-page +eligible only after in-place promotion commits. Charge the existing block +pool cap for mapped extents, including rounded tails. + +This unifies usable block geometry with the OS huge-page unit without adding +a pointer registry or changing object layouts or marking. Preserve whole-arena +GC headroom after small parses by comparing with that arena threshold instead +of the nursery cap; block rounding no longer rearms an already satisfied minor. diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 5b869044fc..5d5a9e130d 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -1,9 +1,11 @@ use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; -/// Keep main's usable block geometry for the allocation experiment. The OS -/// extent is 2 MiB-aligned; 2 MiB usable blocks require hwp's census-capacity -/// integration before they can run (see REGION-DESIGN). +/// One usable block per aligned Linux region. Other backends keep their +/// existing geometry until their OS reservation implementation is validated. +#[cfg(target_os = "linux")] +pub(crate) const BLOCK_SIZE: usize = super::region::ALIGN; +#[cfg(not(target_os = "linux"))] pub(crate) const BLOCK_SIZE: usize = 1024 * 1024; pub(crate) const FRESH_GENERAL_BLOCK_MIN_USED_BYTES: usize = 256 * 1024; @@ -80,7 +82,7 @@ impl Drop for BlockPool { let bytes = self .blocks .iter() - .map(|block| block.size) + .map(|block| super::region::backing_len(block.size)) .fold(0usize, usize::saturating_add); block_pool_process_bytes_sub(bytes); for block in &self.blocks { @@ -201,8 +203,9 @@ pub(crate) fn block_pool_put(data: *mut u8, size: usize) -> bool { return false; } let cap = block_pool_cap_bytes(); - if BLOCK_POOL_BYTES.with(Cell::get).saturating_add(size) > cap - || !block_pool_process_try_reserve(size) + let charge = super::region::backing_len(size); + if BLOCK_POOL_BYTES.with(Cell::get).saturating_add(charge) > cap + || !block_pool_process_try_reserve(charge) { return false; } @@ -213,7 +216,7 @@ pub(crate) fn block_pool_put(data: *mut u8, size: usize) -> bool { reuse_window: 2, }); }); - BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_add(size))); + BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_add(charge))); true } @@ -223,8 +226,9 @@ fn block_pool_take(size: usize) -> Option<*mut u8> { let idx = pool.blocks.iter().rposition(|block| block.size == size)?; Some(pool.blocks.swap_remove(idx).data) })?; - BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_sub(size))); - block_pool_process_bytes_sub(size); + let charge = super::region::backing_len(size); + BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_sub(charge))); + block_pool_process_bytes_sub(charge); Some(taken) } @@ -281,7 +285,7 @@ pub(crate) fn drain_block_pool() -> BlockPoolDrainStats { let entries = BLOCK_POOL.with(|pool| std::mem::take(&mut pool.borrow_mut().blocks)); let bytes = entries .iter() - .map(|block| block.size) + .map(|block| super::region::backing_len(block.size)) .fold(0usize, usize::saturating_add); let tracked = BLOCK_POOL_BYTES.with(|cell| cell.replace(0)); debug_assert_eq!(tracked, bytes, "thread block-pool byte accounting drifted"); @@ -350,6 +354,7 @@ fn try_alloc_block( if let Some(data) = block_pool_take(size) { unsafe { super::region::advise(data, size, kind) }; return Some(ArenaBlock { + extent_kind: kind, data, size, offset: 0, @@ -365,6 +370,7 @@ fn try_alloc_block( return None; } Some(ArenaBlock { + extent_kind: kind, data, size, offset: 0, @@ -434,6 +440,8 @@ fn alloc_block_no_gc(min_size: usize, generation: HeapGeneration) -> ArenaBlock /// A single arena block pub(crate) struct ArenaBlock { + /// Allocation class of this backing, preserved when blocks change generation. + pub(crate) extent_kind: super::region::Kind, pub(crate) data: *mut u8, pub(crate) size: usize, pub(crate) offset: usize, @@ -692,6 +700,7 @@ impl Arena { fn new_lazy(generation: HeapGeneration, space: HeapSpace) -> Self { Arena { blocks: vec![ArenaBlock { + extent_kind: super::region::Kind::NurseryBlock, data: std::ptr::null_mut(), size: 0, offset: 0, diff --git a/crates/perry-runtime/src/arena/block/reuse_window_tests.rs b/crates/perry-runtime/src/arena/block/reuse_window_tests.rs index 68b477f074..37b743bb27 100644 --- a/crates/perry-runtime/src/arena/block/reuse_window_tests.rs +++ b/crates/perry-runtime/src/arena/block/reuse_window_tests.rs @@ -167,3 +167,24 @@ fn collection_entry_discards_only_previously_idle_eden_pages_and_keeps_reuse_saf fn idle_advice_state_fits_existing_arena_block_padding() { assert_eq!(std::mem::size_of::(), 48); } + +#[cfg(target_os = "linux")] +#[test] +fn the_pool_charges_rounded_mapping_tails_and_releases_the_same_charge() { + crate::arena::tests::run_with_fresh_arenas(|| unsafe { + // Deliberately use step-1 geometry to distinguish mapped from usable. + let size = super::super::region::ALIGN / 2; + let raw = super::super::region::map(super::super::region::Kind::NurseryBlock, size); + assert!(!raw.is_null()); + let before = block_pool_bytes_for_test(); + assert!(block_pool_put(raw, size)); + assert_eq!(block_pool_bytes_for_test() - before, 2 * size); + assert_eq!(block_pool_take(size), Some(raw)); + assert_eq!(block_pool_bytes_for_test(), before); + assert!(block_pool_put(raw, size)); + let drained = drain_block_pool(); + assert_eq!(drained.bytes, before + 2 * size); + assert_eq!(block_pool_bytes_for_test(), 0); + dealloc_for_test(raw, size); + }); +} diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 3e8f8449ad..9491216593 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -167,16 +167,16 @@ pub(crate) use stats::{old_gen_in_use_bytes_recomputed, old_gen_in_use_bytes_res pub(crate) use page_meta::{ arena_header_is_object_start, classify_heap_generation, classify_heap_space, classify_heap_space_in_range, generation_page_for_addr, - materialize_promoted_page_runs_for_object, old_arena_block_range_index, old_arena_block_ranges, - old_arena_page_index_remove_object, old_arena_source_blocks_for_pages, - old_arena_walk_objects_on_pages, old_object_page_overlaps, old_object_single_page, - old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object, - old_page_account_swept_object, old_page_account_swept_tally, old_page_clear_dirty, - old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, - old_pages_reset_sweep_accounting, record_arena_object_start, uniform_heap_generation, - unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace, - OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, - OldPageSweepTally, + materialize_promoted_page_runs_for_object, old_arena_block_range_index, + old_arena_movable_block_ranges, old_arena_page_index_remove_object, + old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps, + old_object_single_page, old_page_account_dirty_slot, old_page_account_dirty_slots, + old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally, + old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, + old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, record_arena_object_start, + uniform_heap_generation, unregister_old_object_pages, unregister_old_objects_batch, + HeapGeneration, HeapSpace, OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, + OldPageSummary, OldPageSweepTally, }; #[cfg(test)] @@ -187,3 +187,6 @@ pub(crate) use page_meta::{ register_block_space, register_promoted_page_run, reset_old_page_meta_snapshot_calls_for_tests, DEFERRED_OLD_PAGE_REGISTRATION_CAP, GENERATION_CLASS_SHIFT, GENERATION_PAGE_SIZE, }; + +#[cfg(test)] +pub(crate) use page_meta::old_arena_block_ranges; diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index ba39761480..24760e1ad8 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -1615,7 +1615,18 @@ pub(crate) fn reset_old_page_meta_snapshot_calls_for_tests() { /// page-granular and carries no block identity, which is why #9772's selection /// could predict 44 MB of "releasable block bytes" from page granules and /// release nothing. +#[cfg(test)] pub(crate) fn old_arena_block_ranges() -> Vec<(usize, usize, usize, usize)> { + old_arena_ranges(false) +} + +/// Movable backing only, using the allocation class stored by the arena. +/// Oversized object/byte extents stay excluded even if their geometry changes. +pub(crate) fn old_arena_movable_block_ranges() -> Vec<(usize, usize, usize, usize)> { + old_arena_ranges(true) +} + +fn old_arena_ranges(movable_only: bool) -> Vec<(usize, usize, usize, usize)> { let old_block_start = longlived_end(); OLD_ARENA.with(|arena| { let arena = unsafe { &*arena.get() }; @@ -1627,6 +1638,9 @@ pub(crate) fn old_arena_block_ranges() -> Vec<(usize, usize, usize, usize)> { if block.data.is_null() || block.size == 0 { return None; } + if movable_only && matches!(block.extent_kind, super::region::Kind::LargeObject) { + return None; + } let base = block.data as usize; Some((base, base + block.size, old_block_start + i, block.size)) }) @@ -1636,7 +1650,7 @@ pub(crate) fn old_arena_block_ranges() -> Vec<(usize, usize, usize, usize)> { }) } -/// Index into [`old_arena_block_ranges`] output for the block containing +/// Index into an old-arena range snapshot for the block containing /// `addr`, or `None` when the address is not in a live old-gen block. pub(crate) fn old_arena_block_range_index( ranges: &[(usize, usize, usize, usize)], diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs index 9fdf54c6d1..450160591e 100644 --- a/crates/perry-runtime/src/arena/promote.rs +++ b/crates/perry-runtime/src/arena/promote.rs @@ -343,12 +343,22 @@ pub(crate) fn finish_in_place_promotion( OLD_ARENA.with(|old| { let old = unsafe { &mut *old.get() }; for block in moved_blocks { - let base = block.data as usize; + let data = block.data; + let base = data as usize; let size = block.size; let offset = block.offset; super::block::old_gen_in_use_bytes_add(offset); install_block_into(old, block); retag_block_space(base, size, HeapGeneration::Old, HeapSpace::Old); + // Ownership/generation are committed; speculative retag and its + // rollback never change backing advice. + unsafe { + super::region::advise( + data, + size, + super::region::kind_for(HeapGeneration::Old, size), + ); + } } }); @@ -388,6 +398,7 @@ fn take_block(block: PromotedBlock) -> Option { Some(std::mem::replace( &mut arena.blocks[index], ArenaBlock { + extent_kind: super::region::Kind::NurseryBlock, data: std::ptr::null_mut(), size: 0, offset: 0, diff --git a/crates/perry-runtime/src/arena/quarantine.rs b/crates/perry-runtime/src/arena/quarantine.rs index 2873431d2a..67c2548e98 100644 --- a/crates/perry-runtime/src/arena/quarantine.rs +++ b/crates/perry-runtime/src/arena/quarantine.rs @@ -580,6 +580,7 @@ pub(crate) fn copying_quarantine_from_spaces_and_flip() -> ArenaResetStats { for block in recycled { reusable_bytes = reusable_bytes.saturating_add(block.used); arena.install_reserved_block(ArenaBlock { + extent_kind: super::region::Kind::NurseryBlock, data: block.data, size: block.size, offset: 0, @@ -1141,6 +1142,7 @@ mod tombstone_tests { fn tombstone() -> ArenaBlock { ArenaBlock { + extent_kind: super::region::Kind::NurseryBlock, data: std::ptr::null_mut(), size: 0, offset: 0, @@ -1179,6 +1181,7 @@ mod tombstone_tests { let mut arena = Arena { blocks: vec![ArenaBlock { + extent_kind: super::region::Kind::NurseryBlock, data: backing, size: SIZE, offset: 0, diff --git a/crates/perry-runtime/src/arena/region.rs b/crates/perry-runtime/src/arena/region.rs index 203c2b9c30..ace994c105 100644 --- a/crates/perry-runtime/src/arena/region.rs +++ b/crates/perry-runtime/src/arena/region.rs @@ -7,7 +7,7 @@ use super::HeapGeneration; pub(super) const ALIGN: usize = 2 * 1024 * 1024; #[derive(Clone, Copy)] -pub(super) enum Kind { +pub(crate) enum Kind { NurseryBlock, OldBlock, LargeObject, @@ -15,7 +15,9 @@ pub(super) enum Kind { pub(super) fn kind_for(generation: HeapGeneration, len: usize) -> Kind { match generation { - HeapGeneration::Old | HeapGeneration::Longlived if len > ALIGN => Kind::LargeObject, + HeapGeneration::Old | HeapGeneration::Longlived if len > super::BLOCK_SIZE => { + Kind::LargeObject + } HeapGeneration::Old | HeapGeneration::Longlived => Kind::OldBlock, _ => Kind::NurseryBlock, } @@ -28,18 +30,40 @@ fn mapped_len(len: usize) -> Option { len.checked_add(ALIGN - 1).map(|n| n & !(ALIGN - 1)) } -/// Mature complete payload units are THP eligible. Nursery and sparse tail -/// units are not. Advice precedes writes; neither prefault nor collapse. +/// Charge the pool for the owned OS extent, including rounded tails. Other +/// targets retain the allocator backend and its exact requested length. +pub(super) fn backing_len(len: usize) -> usize { + #[cfg(target_os = "linux")] + { + mapped_len(len).expect("invalid region extent") + } + #[cfg(not(target_os = "linux"))] + { + len + } +} + +/// Complete payload units are dense backing in every generation, including +/// refilled nursery blocks. Sub-unit extents and partial tails stay on base +/// pages. Advice precedes writes; neither prefault nor collapse. pub(super) unsafe fn advise(data: *mut u8, len: usize, kind: Kind) { #[cfg(target_os = "linux")] { let mapped = mapped_len(len).expect("invalid region extent"); - assert_eq!(libc::madvise(data.cast(), mapped, libc::MADV_NOHUGEPAGE), 0); - if !matches!(kind, Kind::NurseryBlock) { - let dense = len / ALIGN * ALIGN; - if dense != 0 { - assert_eq!(libc::madvise(data.cast(), dense, libc::MADV_HUGEPAGE), 0); - } + let _ = kind; + let dense = len / ALIGN * ALIGN; + if dense != 0 { + assert_eq!(libc::madvise(data.cast(), dense, libc::MADV_HUGEPAGE), 0); + } + if dense < mapped { + assert_eq!( + libc::madvise( + data.add(dense).cast(), + mapped - dense, + libc::MADV_NOHUGEPAGE + ), + 0 + ); } } #[cfg(not(target_os = "linux"))] @@ -127,18 +151,14 @@ mod tests { } #[test] - fn aligned_os_backing_has_kind_specific_thp_and_releases_pages() { + fn aligned_os_backing_has_whole_unit_thp_and_releases_pages() { unsafe { for kind in [Kind::NurseryBlock, Kind::OldBlock, Kind::LargeObject] { let data = map(kind, 2 * ALIGN); assert!(!data.is_null()); assert_eq!(data as usize & (ALIGN - 1), 0); let flags = flags_for(data); - let expected = if matches!(kind, Kind::NurseryBlock) { - "nh" - } else { - "hg" - }; + let expected = "hg"; assert!(flags.split_whitespace().any(|v| v == expected), "{flags}"); std::ptr::write_bytes(data, 0xa5, 2 * ALIGN); let mut resident = vec![0u8; 2 * ALIGN / 4096]; @@ -165,11 +185,11 @@ mod tests { } #[test] - fn reused_region_changes_advice_before_new_birth() { + fn reused_whole_region_keeps_dense_advice_in_nursery() { unsafe { let data = map(Kind::OldBlock, ALIGN); advise(data, ALIGN, Kind::NurseryBlock); - assert!(flags_for(data).split_whitespace().any(|v| v == "nh")); + assert!(flags_for(data).split_whitespace().any(|v| v == "hg")); unmap(data, ALIGN); } } @@ -182,4 +202,65 @@ mod tests { assert!(map(Kind::NurseryBlock, usize::MAX & !(ALIGN - 1)).is_null()); } } + + #[test] + fn dense_nursery_advice_survives_speculative_and_committed_promotion() { + crate::arena::tests::run_with_fresh_arenas(|| { + let user = crate::arena::arena_alloc_gc(24, 8, crate::gc::GC_TYPE_STRING); + let base = (user as usize & !(ALIGN - 1)) as *mut u8; + assert!(flags_for(base).split_whitespace().any(|v| v == "hg")); + let speculative = crate::arena::retag_young_for_in_place_promotion(true); + assert!( + !speculative.is_empty(), + "LIVE SUBJECT: retagged nursery block" + ); + assert!(flags_for(base).split_whitespace().any(|v| v == "hg")); + crate::arena::undo_in_place_promotion_retag(&speculative); + assert!(flags_for(base).split_whitespace().any(|v| v == "hg")); + let promotion = crate::arena::retag_young_for_in_place_promotion(false); + let stats = crate::arena::finish_in_place_promotion( + promotion, + crate::arena::PromotionLiveness::AssumeAllLive, + ); + assert!(stats.objects > 0); + assert!(crate::arena::pointer_in_old_gen(user as usize)); + assert!(flags_for(base).split_whitespace().any(|v| v == "hg")); + }); + } + + #[test] + fn mature_complete_units_keep_partial_tails_on_base_pages() { + unsafe { + let len = ALIGN + 4096; + let data = map(Kind::LargeObject, len); + assert!(!data.is_null()); + assert!(flags_for(data).split_whitespace().any(|v| v == "hg")); + assert!(flags_for(data.add(ALIGN)) + .split_whitespace() + .any(|v| v == "nh")); + unmap(data, len); + } + } + + #[test] + fn nursery_whole_units_are_huge_but_small_extents_and_tails_are_not() { + unsafe { + for len in [4096, ALIGN - 4096, ALIGN, ALIGN + 4096, 2 * ALIGN + 4096] { + let data = map(Kind::NurseryBlock, len); + assert!(!data.is_null(), "LIVE SUBJECT: mapped nursery backing"); + let dense = len / ALIGN * ALIGN; + for offset in (0..backing_len(len)).step_by(4096) { + // The kernel's VMA advice is the subject, independently of + // whether THP allocation succeeds on this machine. + let flags = flags_for(data.add(offset)); + let expected = if offset < dense { "hg" } else { "nh" }; + assert!( + flags.split_whitespace().any(|v| v == expected), + "len={len} offset={offset} expected={expected}: {flags}" + ); + } + unmap(data, len); + } + } + } } diff --git a/crates/perry-runtime/src/ffi/perry_memory_profile.c b/crates/perry-runtime/src/ffi/perry_memory_profile.c index f93fedbe5a..04a19d648c 100644 --- a/crates/perry-runtime/src/ffi/perry_memory_profile.c +++ b/crates/perry-runtime/src/ffi/perry_memory_profile.c @@ -21,7 +21,9 @@ static void perry_apply_small_process_default(void) { /* Preserve an inherited strict operator disable (also the all-THP-off * measurement control). Never relax a parent's explicit process policy. */ if (prctl(PR_GET_THP_DISABLE, 0UL, 0UL, 0UL, 0UL) != 1) { - (void)prctl(PR_SET_THP_DISABLE, 1UL, PR_THP_DISABLE_EXCEPT_ADVISED, 0UL, 0UL); + if (prctl(PR_SET_THP_DISABLE, 1UL, PR_THP_DISABLE_EXCEPT_ADVISED, 0UL, 0UL) != 0) { + (void)prctl(PR_SET_THP_DISABLE, 1UL, 0UL, 0UL, 0UL); + } } } diff --git a/crates/perry-runtime/src/gc/large_extent_selection_tests.rs b/crates/perry-runtime/src/gc/large_extent_selection_tests.rs new file mode 100644 index 0000000000..88e495dd4d --- /dev/null +++ b/crates/perry-runtime/src/gc/large_extent_selection_tests.rs @@ -0,0 +1,92 @@ +//! Oversized old object/byte backing must never be selected for relocation. +//! Real arena allocation supplies the ranges; explicit sweep accounting makes +//! each range fragmented without relying on collector scheduling. +use super::super::types::*; +use super::*; + +#[test] +fn old_defrag_and_idle_compaction_never_select_large_or_byte_extents() { + std::thread::spawn(|| unsafe { + let _triggers = + super::super::tests::support::GcTriggerThresholdTestGuard::suppress_automatic_triggers( + ); + let mut live = Vec::new(); + let mut dead = Vec::new(); + live.push(crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize); + dead.push(crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize); + let mut extents = Vec::new(); + for obj_type in [GC_TYPE_STRING, GC_TYPE_BUFFER_ARRAY_BUFFER] { + let big = crate::arena::arena_alloc_gc(3 * crate::arena::BLOCK_SIZE + 64, 8, obj_type) + as usize; + assert!(crate::arena::pointer_in_old_gen(big)); + extents.push(big); + live.push(big); + live.push(crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize); + dead.push(crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize); + } + crate::arena::old_pages_reset_sweep_accounting(); + for (objects, is_live) in [(&live, true), (&dead, false)] { + for &user in objects { + let header = user - GC_HEADER_SIZE; + let total = (*(header as *const GcHeader)).size as usize; + crate::arena::old_page_account_swept_object(header, total, is_live, false); + } + } + let snapshot = crate::arena::old_page_meta_snapshot(); + let ranges = crate::arena::old_arena_block_ranges(); + let large_ranges: Vec<_> = extents + .iter() + .map(|&user| { + let idx = crate::arena::old_arena_block_range_index(&ranges, user).unwrap(); + let range = ranges[idx]; + assert!(range.3 > crate::arena::BLOCK_SIZE); + assert!( + snapshot + .iter() + .any(|m| (range.0..range.1).contains(&m.page_base) + && m.live_bytes > 0 + && m.dead_bytes > 0 + && m.pinned_bytes == 0), + "LIVE SUBJECT: fragmented unpinned oversized extent" + ); + range + }) + .collect(); + for selection in [ + select_old_page_defrag_pages_from_snapshot(&snapshot, true), + select_whole_blocks(&snapshot, OldPageDefragSelection::default()), + ] { + assert!( + selection.selected_pages > 0, + "LIVE SUBJECT: ordinary old block selected" + ); + for range in &large_ranges { + assert!( + selection.pages.iter().all(|&page| { + !(range.0..range.1).contains(&crate::arena::generation_page_base(page)) + }), + "large/byte extent selected: {range:?}" + ); + } + } + // Restore the old selector's missing extent guard. Both selection + // paths must now select a forbidden extent, proving this witness has + // teeth independently of the relocation type check downstream. + let _sabotage = extent_selection_sabotage::Guard::arm(); + for selection in [ + select_old_page_defrag_pages_from_snapshot(&snapshot, true), + select_whole_blocks(&snapshot, OldPageDefragSelection::default()), + ] { + assert!( + large_ranges + .iter() + .any(|range| selection.pages.iter().any(|&page| { + (range.0..range.1).contains(&crate::arena::generation_page_base(page)) + })), + "SABOTAGE: missing extent guard must select large backing" + ); + } + }) + .join() + .expect("extent selection test panicked"); +} diff --git a/crates/perry-runtime/src/gc/oldgen_defrag.rs b/crates/perry-runtime/src/gc/oldgen_defrag.rs index f9d4e50b41..75d8a94988 100644 --- a/crates/perry-runtime/src/gc/oldgen_defrag.rs +++ b/crates/perry-runtime/src/gc/oldgen_defrag.rs @@ -49,6 +49,15 @@ pub(super) fn old_page_defrag_skipped_for_pin(meta: crate::arena::OldPageMeta) - /// what the next one takes. Lowering the fixed cost is separate work. pub(super) const IDLE_COMPACT_MOVE_BUDGET_BYTES: usize = 1024 * 1024; +/// Use the arena's allocation class, shared by both selection policies. +fn movable_old_ranges() -> Vec<(usize, usize, usize, usize)> { + #[cfg(test)] + if extent_selection_sabotage::enabled() { + return crate::arena::old_arena_block_ranges(); + } + crate::arena::old_arena_movable_block_ranges() +} + pub(super) fn select_old_page_defrag_pages_from_snapshot( snapshot: &[crate::arena::OldPageMeta], force: bool, @@ -66,8 +75,12 @@ pub(super) fn select_old_page_defrag_pages_from_snapshot( if idle_compact_armed() && idle_compact_block_selection_enabled() { return select_whole_blocks(snapshot, selection); } + let ranges = movable_old_ranges(); let mut candidates = Vec::new(); for &meta in snapshot { + if crate::arena::old_arena_block_range_index(&ranges, meta.page_base).is_none() { + continue; + } if old_page_defrag_skipped_for_pin(meta) { selection.skipped_pinned_pages = selection.skipped_pinned_pages.saturating_add(1); continue; @@ -388,7 +401,7 @@ fn select_whole_blocks( snapshot: &[crate::arena::OldPageMeta], mut selection: OldPageDefragSelection, ) -> OldPageDefragSelection { - let ranges = crate::arena::old_arena_block_ranges(); + let ranges = movable_old_ranges(); if ranges.is_empty() { return selection; } @@ -492,3 +505,29 @@ mod longlived_never_selected_tests { assert!(!selection.pages.contains(&page)); } } + +#[cfg(test)] +#[path = "large_extent_selection_tests.rs"] +mod large_extent_selection_tests; + +#[cfg(test)] +mod extent_selection_sabotage { + thread_local! { + static ENABLED: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + pub(super) fn enabled() -> bool { + ENABLED.with(std::cell::Cell::get) + } + pub(super) struct Guard; + impl Guard { + pub(super) fn arm() -> Self { + ENABLED.with(|v| v.set(true)); + Self + } + } + impl Drop for Guard { + fn drop(&mut self) { + ENABLED.with(|v| v.set(false)); + } + } +} diff --git a/crates/perry-runtime/src/gc/tests/adopt_census.rs b/crates/perry-runtime/src/gc/tests/adopt_census.rs index 46342938b0..409ff12b35 100644 --- a/crates/perry-runtime/src/gc/tests/adopt_census.rs +++ b/crates/perry-runtime/src/gc/tests/adopt_census.rs @@ -1,7 +1,7 @@ //! #10182: a promoted-cohort full adopts the census record the in-place //! promotion walk made of each block it promoted, instead of walking it again. //! -//! A young population of about two megabytes — a rooted young array of young +//! A young population spanning multiple blocks — a rooted young array of young //! arrays holding young strings, with dead strings between them — is promoted //! whole by an untraced minor while recording is armed. Half the held strings //! then die, and the cohort full runs. Every adopted block is re-walked by the @@ -31,7 +31,7 @@ fn promote_then_collect_cohort(sabotaged: bool) -> Outcome { cohort::seed_for_tests(0, 0, 0); const INNER: usize = 64; - const OUTER: usize = 64; + const OUTER: usize = crate::arena::BLOCK_SIZE / (16 * 1024); let mut outer = crate::array::js_array_alloc(OUTER as u32); let mut held: Vec<(usize, usize, usize)> = Vec::new(); for o in 0..OUTER { diff --git a/crates/perry-runtime/src/gc/tests/census_two_mib.rs b/crates/perry-runtime/src/gc/tests/census_two_mib.rs new file mode 100644 index 0000000000..cf5e50f13c --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/census_two_mib.rs @@ -0,0 +1,37 @@ +//! Regression for the step-1 two-pointer census overflow at chunk 2. +use super::super::*; +use super::support::*; + +#[test] +#[cfg(target_os = "linux")] +fn a_full_two_mib_block_is_censused_by_both_walks() { + std::thread::spawn(|| { + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _scan = ConservativeScanDisabledGuard::new(); + assert_eq!(crate::arena::BLOCK_SIZE, 2 * 1024 * 1024); + let mut users = Vec::new(); + for _ in 0..crate::arena::BLOCK_SIZE / 64 + 1 { + users.push(crate::arena::arena_alloc_gc_old(56, 8, GC_TYPE_STRING) as usize); + } + let whole = ValidPointerSetBuilder::new().finish(); + let mut builder = ValidPointerSetBuilder::new(); + while !builder.step(7) {} + let stepped = builder.finish(); + let full = whole + .arena_blocks + .iter() + .find(|b| b.extent == crate::arena::BLOCK_SIZE) + .expect("LIVE SUBJECT: a full regular 2 MiB block"); + assert!(!full.sorted); + assert_eq!(full.chunks.len(), 4); + assert!(full.chunks.iter().all(|p| !p.is_null())); + assert_eq!(whole.start_bitmap_chunks, stepped.start_bitmap_chunks); + for user in users { + assert!(whole.contains(&user), "whole census lost {user:#x}"); + assert!(stepped.contains(&user), "stepped census lost {user:#x}"); + assert!(!whole.contains(&(user + 1))); + } + }) + .join() + .unwrap(); +} diff --git a/crates/perry-runtime/src/gc/tests/debt_pacer.rs b/crates/perry-runtime/src/gc/tests/debt_pacer.rs index fbc1e03cb1..9f07c5b861 100644 --- a/crates/perry-runtime/src/gc/tests/debt_pacer.rs +++ b/crates/perry-runtime/src/gc/tests/debt_pacer.rs @@ -949,7 +949,7 @@ fn minor_sweep_retains_window_expired_growth_stub() { // Small allocations advance the GENERAL bump blocks (large ones are // routed to dedicated blocks and would leave `current` untouched). let general_before = crate::arena::general_block_count(); - for _ in 0..3_000 { + for _ in 0..8 * crate::arena::BLOCK_SIZE / (4 * 1024) { let filler = crate::arena::arena_alloc_gc(4 * 1024, 8, GC_TYPE_STRING); std::hint::black_box(filler); } diff --git a/crates/perry-runtime/src/gc/tests/evacuation.rs b/crates/perry-runtime/src/gc/tests/evacuation.rs index 8d1636a054..2d8b235d69 100644 --- a/crates/perry-runtime/src/gc/tests/evacuation.rs +++ b/crates/perry-runtime/src/gc/tests/evacuation.rs @@ -500,10 +500,16 @@ fn test_old_page_defrag_selection_reports_releasable_granule_bytes() { } } - let selected_a = meta(0x2000_0000, 100, 10, 90, 0); - let selected_b = meta(0x2000_1000, 100, 20, 80, 0); - let unselected_low_dead = meta(0x2000_2000, 100, 80, 20, 0); - let pinned = meta(0x2000_3000, 100, 10, 90, 8); + let _triggers = super::support::GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + // Selection now requires backing owned by the old arena. Keep the + // synthetic accounting, but put its page addresses inside a real block. + let user = crate::arena::arena_alloc_gc_old(6 * 4096, 8, GC_TYPE_STRING) as usize; + let base = (user + 4095) & !4095; + + let selected_a = meta(base, 100, 10, 90, 0); + let selected_b = meta(base + 1 * 4096, 100, 20, 80, 0); + let unselected_low_dead = meta(base + 2 * 4096, 100, 80, 20, 0); + let pinned = meta(base + 3 * 4096, 100, 10, 90, 8); let snapshot = [selected_a, selected_b, unselected_low_dead, pinned]; let selection = select_old_page_defrag_pages_from_snapshot(&snapshot, false); @@ -737,7 +743,7 @@ fn test_sweep_reports_and_retains_non_evacuation_forwarded_stub() { set_forwarding_address(stub_header, target); (*stub_header).gc_flags |= GC_FLAG_MARKED; } - for _ in 0..90_000 { + for _ in 0..7 * crate::arena::BLOCK_SIZE / (64 + GC_HEADER_SIZE) { let _ = crate::arena::arena_alloc_gc(64, 8, GC_TYPE_OBJECT); } @@ -770,7 +776,7 @@ fn test_sweep_reclaims_unreached_old_forwarded_stub() { unsafe { set_forwarding_address(stub_header, target); } - for _ in 0..90_000 { + for _ in 0..7 * crate::arena::BLOCK_SIZE / (64 + GC_HEADER_SIZE) { let _ = crate::arena::arena_alloc_gc(64, 8, GC_TYPE_OBJECT); } diff --git a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs index 53b31df041..280ad566f2 100644 --- a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs @@ -257,7 +257,7 @@ fn arena_sweep_pauses_before_block_cleanup_and_preserves_live_objects() { } js_shadow_slot_set(0, ptr_bits(live)); - for _ in 0..1200 { + for _ in 0..8 * crate::arena::BLOCK_SIZE / (8 * 1024) { let _ = crate::arena::arena_alloc_gc(8 * 1024, 8, GC_TYPE_STRING); } let in_use_after_alloc = crate::arena::arena_in_use_bytes(); diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 4097066ff5..c649bde5d3 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -133,3 +133,5 @@ mod eden_entry_residency; mod buffer_bytes; mod buffer_b4; + +mod census_two_mib; diff --git a/crates/perry-runtime/src/gc/tests/native_payload_streams.rs b/crates/perry-runtime/src/gc/tests/native_payload_streams.rs index e4c7fe0fdf..0a74833619 100644 --- a/crates/perry-runtime/src/gc/tests/native_payload_streams.rs +++ b/crates/perry-runtime/src/gc/tests/native_payload_streams.rs @@ -75,13 +75,22 @@ fn anon_rss_bytes() -> usize { /// finished stream alive (no static holds a step job). #[test] fn z8_churn_releases_every_codec_at_completion_and_drops_every_payload() { + // This churn can reach an automatic copying minor with either block size. + // Isolate scanner/cache roots just like the other copying-GC witnesses. + let _gc = super::support::CopyingNurseryTestGuard::new(0); let _reset = FamilyReset::new(); - const N: usize = 2000; + // Warm the larger backing before measuring six complete churn batches. + // Keep the RSS bound and all release/drop/finalization checks unchanged. + const BLOCK_SCALE: usize = crate::arena::BLOCK_SIZE / (1024 * 1024); + const WARMUP_BATCHES: usize = 2 * BLOCK_SCALE; + const BATCHES: usize = WARMUP_BATCHES + 6; + const PER_BATCH: usize = 250 * BLOCK_SCALE; + const N: usize = BATCHES * PER_BATCH; let input = vec![b'q'; 1024]; let before_cells = finalized(); let mut rss = Vec::new(); - for batch in 0..8 { - for _ in 0..N / 8 { + for batch in 0..BATCHES { + for _ in 0..PER_BATCH { let scope = crate::gc::RuntimeHandleScope::new(); let rot = scope.root_nanbox_f64(new_rot13(Rot13Opts::default())); on_data(rot.get_nanbox_f64(), crate::fn_info!(sink_data, 1)); @@ -129,17 +138,17 @@ fn z8_churn_releases_every_codec_at_completion_and_drops_every_payload() { CREATED.load(Ordering::SeqCst), RELEASED.load(Ordering::SeqCst), DROPPED.load(Ordering::SeqCst), - rss[2], - rss[2..].iter().max().unwrap(), - rss[2..].iter().max().unwrap() - rss[2..].iter().min().unwrap() + rss[WARMUP_BATCHES], + rss[WARMUP_BATCHES..].iter().max().unwrap(), + rss[WARMUP_BATCHES..].iter().max().unwrap() - rss[WARMUP_BATCHES..].iter().min().unwrap() ); assert!( cells >= 2 * N - 16, "the dead streams' cells were swept: {cells} of {}", 2 * N ); - let warm = *rss[2..].iter().min().unwrap(); - let peak = *rss[2..].iter().max().unwrap(); + let warm = *rss[WARMUP_BATCHES..].iter().min().unwrap(); + let peak = *rss[WARMUP_BATCHES..].iter().max().unwrap(); assert!( peak - warm < 4 << 20, "anonymous RSS flat after warmup: {}", diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index 3839a8f5b1..10868c5c8c 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -454,11 +454,17 @@ fn test_old_page_defrag_policy_selection_prefers_fragmented_unpinned_pages() { } } - let low_dead = meta(0x1000_0000, 100, 80, 20, 0); - let high_dead = meta(0x1000_1000, 100, 10, 90, 0); - let high_dead_more_live = meta(0x1000_2000, 100, 20, 80, 0); - let pinned = meta(0x1000_3000, 100, 10, 90, 8); - let empty = meta(0x1000_4000, 0, 0, 0, 0); + let _triggers = super::support::GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + // Selection now requires backing owned by the old arena. Keep the + // synthetic accounting, but put its page addresses inside a real block. + let user = crate::arena::arena_alloc_gc_old(6 * 4096, 8, GC_TYPE_STRING) as usize; + let base = (user + 4095) & !4095; + + let low_dead = meta(base, 100, 80, 20, 0); + let high_dead = meta(base + 1 * 4096, 100, 10, 90, 0); + let high_dead_more_live = meta(base + 2 * 4096, 100, 20, 80, 0); + let pinned = meta(base + 3 * 4096, 100, 10, 90, 8); + let empty = meta(base + 4 * 4096, 0, 0, 0, 0); let snapshot = [low_dead, high_dead_more_live, pinned, empty, high_dead]; let selection = select_old_page_defrag_pages_from_snapshot(&snapshot, false); diff --git a/crates/perry-runtime/src/gc/tests/runtime_roots.rs b/crates/perry-runtime/src/gc/tests/runtime_roots.rs index 25e101782d..28ee667b94 100644 --- a/crates/perry-runtime/src/gc/tests/runtime_roots.rs +++ b/crates/perry-runtime/src/gc/tests/runtime_roots.rs @@ -109,7 +109,7 @@ fn assert_panics_with(expected: &str, f: impl FnOnce()) { } pub(super) fn force_next_general_arena_alloc_slow() { - const TEST_BLOCK_SIZE: usize = 1024 * 1024; + const TEST_BLOCK_SIZE: usize = crate::arena::BLOCK_SIZE; let _ = crate::arena::arena_alloc(TEST_BLOCK_SIZE, 8); } diff --git a/crates/perry-runtime/src/gc/tests/sweep_window_births.rs b/crates/perry-runtime/src/gc/tests/sweep_window_births.rs index fc5424c0b5..80bdc6b4ba 100644 --- a/crates/perry-runtime/src/gc/tests/sweep_window_births.rs +++ b/crates/perry-runtime/src/gc/tests/sweep_window_births.rs @@ -147,7 +147,7 @@ fn inline_births_during_a_budgeted_block_cleanup_stay_inside_their_block() { // Several Eden blocks of garbage, so the cleanup resets blocks behind the // current one and then moves allocation back to the first of them. - for _ in 0..1200 { + for _ in 0..8 * crate::arena::BLOCK_SIZE / (8 * 1024) { let _ = crate::arena::arena_alloc_gc(8 * 1024, 8, GC_TYPE_STRING); } let inline_block_before = unsafe { (*crate::arena::js_inline_arena_state()).data } as usize; diff --git a/crates/perry-runtime/src/startup_memory_profile.rs b/crates/perry-runtime/src/startup_memory_profile.rs index e3cbecfb06..c7c4fdba7c 100644 --- a/crates/perry-runtime/src/startup_memory_profile.rs +++ b/crates/perry-runtime/src/startup_memory_profile.rs @@ -20,6 +20,15 @@ mod linux { // with multiple codegen units. This function does not apply a policy // late; its constructor must already have run before Rust startup. unsafe { perry_retain_memory_profile_init() }; + if crate::gc::gc_diag_enabled() { + let mode = unsafe { libc::prctl(libc::PR_GET_THP_DISABLE, 0, 0, 0, 0) }; + let name = match mode { + 3 => "except-advised", + 1 => "base-pages", + _ => "unexpected", + }; + eprintln!("[gc-region-thp] mode={name} prctl={mode}"); + } } #[cfg(test)] diff --git a/test-files/_helpers/region_transfer_sender_exit.ts b/test-files/_helpers/region_transfer_sender_exit.ts new file mode 100644 index 0000000000..0080996f6f --- /dev/null +++ b/test-files/_helpers/region_transfer_sender_exit.ts @@ -0,0 +1,8 @@ +import { parentPort } from "node:worker_threads"; +const store = new ArrayBuffer(32 * 1024 * 1024); +const bytes = new Uint8Array(store); +for (let i = 0; i < bytes.length; i += 4096) bytes[i] = (i / 4096) & 255; +bytes[bytes.length - 1] = 199; +parentPort!.postMessage(store, [store]); +if (store.byteLength !== 0) process.exit(7); +// With no listeners or pending work, reaching the end exits this worker. diff --git a/test-files/test_gap_region_transfer_sender_exit.ts b/test-files/test_gap_region_transfer_sender_exit.ts new file mode 100644 index 0000000000..26eb42e246 --- /dev/null +++ b/test-files/test_gap_region_transfer_sender_exit.ts @@ -0,0 +1,24 @@ +// parity-node-argv: --expose-gc +// Read transferred bytes only after the worker has exited and its arena died. +import { Worker } from "node:worker_threads"; +const guard = setTimeout(() => process.exit(2), 60000); +let received: any = null; +let messages = 0; +const worker = new Worker(new URL("./_helpers/region_transfer_sender_exit.ts", import.meta.url)); +worker.on("message", (value: any) => { received = value; messages++; }); +worker.on("exit", (code: number) => { + if (code !== 0 || messages !== 1 || !received) process.exit(3); + const bytes = new Uint8Array(received); + if (bytes.length !== 32 * 1024 * 1024) process.exit(4); + // Churn the receiver after sender teardown, before reading the payload. + for (let i = 0; i < 12; i++) { + const churn = Buffer.alloc(2 * 1024 * 1024, i); + if (churn[0] !== i) process.exit(5); + } + if (typeof globalThis.gc === "function") globalThis.gc(); + let sum = 0; + for (let i = 0; i < bytes.length; i += 4096) sum += bytes[i]; + console.log("after sender exit", bytes.length, sum, bytes[bytes.length - 1]); + if (sum !== 1044480 || bytes[bytes.length - 1] !== 199) process.exit(6); + clearTimeout(guard); +}); From 7961e3722f089ddac918a2582b643b79eefa4d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 02:51:39 +0000 Subject: [PATCH 5/6] Restore GC root scanners in the stream churn test --- changelog.d/gc-region-manager-2mib.md | 5 +++++ .../src/gc/tests/native_payload_streams.rs | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/changelog.d/gc-region-manager-2mib.md b/changelog.d/gc-region-manager-2mib.md index 1763ebc29f..e41adc207e 100644 --- a/changelog.d/gc-region-manager-2mib.md +++ b/changelog.d/gc-region-manager-2mib.md @@ -7,3 +7,8 @@ This unifies usable block geometry with the OS huge-page unit without adding a pointer registry or changing object layouts or marking. Preserve whole-arena GC headroom after small parses by comparing with that arena threshold instead of the nursery cap; block rounding no longer rearms an already satisfied minor. + +Restore the stream churn test's payload-prototype, runtime-handle, shape and +queue scanners inside its isolated copying-GC setup so batch collections +retain the roots the workload uses. Keep the release, drop, finalization and +anonymous-RSS assertions unchanged. diff --git a/crates/perry-runtime/src/gc/tests/native_payload_streams.rs b/crates/perry-runtime/src/gc/tests/native_payload_streams.rs index 0a74833619..a4f0b11959 100644 --- a/crates/perry-runtime/src/gc/tests/native_payload_streams.rs +++ b/crates/perry-runtime/src/gc/tests/native_payload_streams.rs @@ -79,6 +79,16 @@ fn z8_churn_releases_every_codec_at_completion_and_drops_every_payload() { // Isolate scanner/cache roots just like the other copying-GC witnesses. let _gc = super::support::CopyingNurseryTestGuard::new(0); let _reset = FamilyReset::new(); + // The isolation guard suppresses automatic scanner registration. Restore + // the roots used by this stream workload, including its cached prototype + // and handles, before a batch collection can sweep or move them. + super::support::register_runtime_handle_root_scanner_for_tests(); + crate::gc::gc_register_mutable_root_scanner( + crate::native_payload::scan_payload_prototype_roots_mut, + ); + crate::gc::gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::gc::gc_register_mutable_root_scanner(crate::promise::scan_promise_roots_mut); + crate::gc::gc_register_mutable_root_scanner(crate::timer::scan_timer_roots_mut); // Warm the larger backing before measuring six complete churn batches. // Keep the RSS bound and all release/drop/finalization checks unchanged. const BLOCK_SCALE: usize = crate::arena::BLOCK_SIZE / (1024 * 1024); From 0ec09df7e07eb0bfb5e927f3072e327bc14320be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 7 Oct 2026 05:04:10 +0200 Subject: [PATCH 6/6] changelog: name the fragment for #12132 --- changelog.d/12132-gc-region-manager.md | 1 + changelog.d/gc-region-manager-2mib.md | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) create mode 100644 changelog.d/12132-gc-region-manager.md delete mode 100644 changelog.d/gc-region-manager-2mib.md diff --git a/changelog.d/12132-gc-region-manager.md b/changelog.d/12132-gc-region-manager.md new file mode 100644 index 0000000000..f9f99f1586 --- /dev/null +++ b/changelog.d/12132-gc-region-manager.md @@ -0,0 +1 @@ +GC arena blocks on Linux now come straight from the OS in 2 MiB-aligned regions, pooled per agent, instead of from the Rust global allocator; mimalloc keeps ordinary native allocations. Linux GC blocks are 2 MiB, and page size follows how much of a unit is used: complete 2 MiB units get huge-page advice in every generation, while sub-unit extents and tails stay on base pages. RSS drops 15–45% across tsc, Zod, qs, commander, fastify, Effect, buffer- and worker-heavy programs and upm, with system CPU within noise of main. A parse bump no longer lowers an already armed whole-arena GC trigger, which removed a minor-collection storm with larger blocks. Census bitmap capacity follows the block size, and the defrag selector decides from the allocation kind stored in the block. diff --git a/changelog.d/gc-region-manager-2mib.md b/changelog.d/gc-region-manager-2mib.md deleted file mode 100644 index e41adc207e..0000000000 --- a/changelog.d/gc-region-manager-2mib.md +++ /dev/null @@ -1,14 +0,0 @@ -Use 2 MiB usable Linux GC blocks in 2 MiB-aligned OS regions. Derive census -bitmap capacity from block geometry and advise nursery backing as huge-page -eligible only after in-place promotion commits. Charge the existing block -pool cap for mapped extents, including rounded tails. - -This unifies usable block geometry with the OS huge-page unit without adding -a pointer registry or changing object layouts or marking. Preserve whole-arena -GC headroom after small parses by comparing with that arena threshold instead -of the nursery cap; block rounding no longer rearms an already satisfied minor. - -Restore the stream churn test's payload-prototype, runtime-handle, shape and -queue scanners inside its isolated copying-GC setup so batch collections -retain the roots the workload uses. Keep the release, drop, finalization and -anonymous-RSS assertions unchanged.