gc(macos): back Apple GC blocks with tagged 1 MiB regions

crates/perry-runtime/src/arena/block.rs

@@ -49,7 +49,21 @@ fn block_size_for(min_size: usize) -> usize {
 /// Only empty, arena-owned payload pages may be advised while mapped.
 pub(super) fn discard_empty_block_pages(block: &ArenaBlock) -> usize {
     debug_assert_eq!(block.offset, 0);
-    unsafe { decommit::release(block.data, block.size) }
+    unsafe { release_block_pages(block.data, block.size) }
+}
+
+/// Apple blocks are whole tagged regions released as reusable; they are
+/// reactivated in [`block_pool_take`] and [`Arena::set_current`]. Other
+/// targets advise the interior pages and need no reactivation.
+unsafe fn release_block_pages(data: *mut u8, size: usize) -> usize {
+    #[cfg(target_vendor = "apple")]
+    {
+        super::region::release(data, size)
+    }
+    #[cfg(not(target_vendor = "apple"))]
+    {
+        decommit::release(data, size)
+    }
 }
 
 struct PooledBlock {
@@ -193,6 +207,9 @@ fn block_pool_cap_bytes() -> usize {
     crate::gc::gc_block_pool_cap_bytes()
 }
 
+// Apple releases whole regions instead; the interior-page helper stays
+// compiled there for its unit tests.
+#[cfg_attr(target_vendor = "apple", allow(dead_code))]
 #[path = "block/decommit.rs"]
 mod decommit;
 
@@ -224,7 +241,11 @@ fn block_pool_take(size: usize) -> Option<*mut u8> {
     let taken = BLOCK_POOL.with(|p| {
         let mut pool = p.borrow_mut();
         let idx = pool.blocks.iter().rposition(|block| block.size == size)?;
-        Some(pool.blocks.swap_remove(idx).data)
+        let block = pool.blocks.swap_remove(idx);
+        // A pooled block may have been released by its reuse window or as an
+        // idle Eden block before it was pooled; reactivate on every handoff.
+        unsafe { super::region::reuse(block.data, block.size) };
+        Some(block.data)
     })?;
     let charge = super::region::backing_len(size);
     BLOCK_POOL_BYTES.with(|c| c.set(c.get().saturating_sub(charge)));
@@ -251,7 +272,7 @@ pub(crate) fn advance_block_pool_reuse_window() -> usize {
                 // Only empty owned bytes are eligible; allocator boundary
                 // pages remain intact even after the reuse window expires.
                 released =
-                    released.saturating_add(unsafe { decommit::release(block.data, block.size) });
+                    released.saturating_add(unsafe { release_block_pages(block.data, block.size) });
             }
         }
         released
@@ -474,6 +495,7 @@ pub(crate) struct ArenaBlock {
     /// Empty Eden payload pages were already discarded in this idle interval.
     /// Reuse is detected at reset, after synchronizing the bump pointer. This
     /// avoids an extra store in either inline or runtime allocation paths.
+    /// Apple reactivates the pages in `Arena::set_current`.
     pub(crate) idle_pages_discarded: bool,
 }
 
@@ -489,6 +511,18 @@ impl ArenaBlock {
         self.object_starts.as_ptr() as *mut u64
     }
 
+    /// Reactivate released pages before the block is handed out. Idle discard
+    /// never touches the current block, and every block becomes current
+    /// before any of its bytes are written, so this is off the bump path.
+    #[inline]
+    fn prepare_reuse(&mut self) {
+        #[cfg(target_vendor = "apple")]
+        if self.idle_pages_discarded {
+            unsafe { super::region::reuse(self.data, self.size) };
+            self.idle_pages_discarded = false;
+        }
+    }
+
     #[inline]
     pub(crate) fn clear_object_starts(&mut self) {
         // Reset callers clear object starts before zeroing a used offset.
@@ -693,6 +727,7 @@ impl Arena {
     #[inline]
     pub(crate) fn set_current(&mut self, idx: usize) {
         self.current = idx;
+        self.blocks[idx].prepare_reuse();
         super::from_space::invalidate_sealed_young_bytes();
         // #10698: the trigger watermark bounds the offset of the block that
         // WAS current against the bytes sealed in the others; neither holds

crates/perry-runtime/src/arena/block/reuse_window_tests.rs

@@ -188,3 +188,145 @@ fn the_pool_charges_rounded_mapping_tails_and_releases_the_same_charge() {
         dealloc_for_test(raw, size);
     });
 }
+
+#[cfg(target_vendor = "apple")]
+#[test]
+fn the_apple_pool_charges_whole_regions_and_releases_the_same_charge() {
+    crate::arena::tests::run_with_fresh_arenas(|| unsafe {
+        // A sub-region request still owns, and is charged, a whole region.
+        let size = BLOCK_SIZE / 2;
+        let raw = super::super::region::map(super::super::region::Kind::NurseryBlock, size);
+        assert!(!raw.is_null());
+        let before = block_pool_bytes_for_test();
+        assert!(block_pool_put(raw, size));
+        assert_eq!(block_pool_bytes_for_test() - before, BLOCK_SIZE);
+        assert_eq!(block_pool_take(size), Some(raw));
+        assert_eq!(block_pool_bytes_for_test(), before);
+        dealloc_for_test(raw, size);
+    });
+}
+
+#[cfg(target_vendor = "apple")]
+#[test]
+fn pooled_apple_reuse_reactivates_pages_before_handing_memory_out() {
+    crate::arena::tests::run_with_fresh_arenas(|| unsafe {
+        let size = 2 * BLOCK_SIZE;
+        let raw = super::super::region::map(super::super::region::Kind::NurseryBlock, size);
+        assert!(!raw.is_null());
+        std::ptr::write_bytes(raw, 0xa5, size);
+        assert!(block_pool_put(raw, size));
+        assert_eq!(advance_block_pool_reuse_window(), 0);
+        assert!(advance_block_pool_reuse_window() >= size);
+        let reusable = super::super::region::apple_tests::info_for(raw).pages_reusable;
+        assert!(reusable > 0, "live subject: cold pool pages");
+        assert_eq!(block_pool_take(size), Some(raw));
+        let reusable = super::super::region::apple_tests::info_for(raw).pages_reusable;
+        assert_eq!(
+            reusable, 0,
+            "pool must call REUSE before handing memory out"
+        );
+        std::ptr::write_bytes(raw, 0x5a, size);
+        assert_eq!(*raw.add(size - 1), 0x5a);
+        dealloc_for_test(raw, size);
+    });
+}
+
+#[cfg(target_vendor = "apple")]
+#[test]
+fn already_discarded_apple_block_reactivates_even_inside_a_new_pool_window() {
+    crate::arena::tests::run_with_fresh_arenas(|| unsafe {
+        let size = 2 * BLOCK_SIZE;
+        let raw = super::super::region::map(super::super::region::Kind::NurseryBlock, size);
+        std::ptr::write_bytes(raw, 0xa5, size);
+        assert_eq!(super::super::region::release(raw, size), size);
+        let reusable = super::super::region::apple_tests::info_for(raw).pages_reusable;
+        assert!(reusable > 0, "live subject: arena discarded before pooling");
+        assert!(block_pool_put(raw, size));
+        assert_eq!(
+            block_pool_take(size),
+            Some(raw),
+            "new window has not expired"
+        );
+        let reusable = super::super::region::apple_tests::info_for(raw).pages_reusable;
+        assert_eq!(reusable, 0, "new warm window must not suppress REUSE");
+        dealloc_for_test(raw, size);
+    });
+}
+
+#[cfg(target_vendor = "apple")]
+#[test]
+fn idle_apple_block_reactivates_when_it_becomes_current() {
+    crate::arena::tests::run_with_fresh_arenas(|| unsafe {
+        // A standalone arena over backing this test owns and unmaps, so no
+        // block retained in the TLS pool for stale-pointer witnesses is used.
+        let size = BLOCK_SIZE;
+        let kind = super::super::region::Kind::NurseryBlock;
+        let block = |offset: usize| ArenaBlock {
+            data: super::super::region::map(kind, size),
+            extent_kind: kind,
+            size,
+            offset,
+            object_starts: new_object_start_bitmap(size),
+            dead_cycles: 0,
+            old_free_holes: false,
+            pinned_summary: false,
+            idle_pages_discarded: false,
+        };
+        let full = block(size);
+        let mut idle = block(0);
+        std::ptr::write_bytes(idle.data, 0xa5, idle.size);
+        assert_eq!(discard_empty_block_pages(&idle), idle.size);
+        idle.idle_pages_discarded = true;
+        let idle_data = idle.data;
+        let mut arena = Arena {
+            blocks: vec![full, idle],
+            current: 0,
+            generation: HeapGeneration::Nursery,
+            space: HeapSpace::Survivor0,
+            allocated_bytes: 0,
+            large_allocated_bytes: 0,
+        };
+        let reusable = super::super::region::apple_tests::info_for(idle_data).pages_reusable;
+        assert!(reusable > 0, "live subject: idle arena pages");
+
+        // The bump path itself issues no syscall: a direct block bump leaves
+        // the pages reusable and the flag set.
+        assert_eq!(arena.blocks[1].alloc(16, 8), Some(idle_data));
+        assert!(arena.blocks[1].idle_pages_discarded);
+        assert!(super::super::region::apple_tests::info_for(idle_data).pages_reusable > 0);
+        arena.blocks[1].offset = 0;
+
+        // The arena hands the block out only by making it current, which
+        // reactivates it first.
+        assert_eq!(arena.try_alloc_after_gc(16, 8), Some(idle_data));
+        assert_eq!(arena.current, 1);
+        assert!(!arena.blocks[1].idle_pages_discarded);
+        let reusable = super::super::region::apple_tests::info_for(idle_data).pages_reusable;
+        assert_eq!(
+            reusable, 0,
+            "a block must be reactivated before it is handed out"
+        );
+        std::ptr::write_bytes(idle_data, 0x5a, 16);
+        assert_eq!(*idle_data, 0x5a);
+
+        // A reset that picks the discarded block goes through the same funnel.
+        arena.blocks[0].offset = 0;
+        std::ptr::write_bytes(arena.blocks[0].data, 0xa5, size);
+        assert_eq!(discard_empty_block_pages(&arena.blocks[0]), size);
+        arena.blocks[0].idle_pages_discarded = true;
+        arena.set_current(0);
+        assert!(!arena.blocks[0].idle_pages_discarded);
+        let reusable =
+            super::super::region::apple_tests::info_for(arena.blocks[0].data).pages_reusable;
+        assert_eq!(
+            reusable, 0,
+            "set_current must reactivate the new current block"
+        );
+
+        for block in &mut arena.blocks {
+            dealloc_for_test(block.data, block.size);
+            block.data = std::ptr::null_mut();
+            block.size = 0;
+        }
+    });
+}

crates/perry-runtime/src/arena/mod.rs

@@ -197,3 +197,8 @@ pub(crate) use page_meta::{
 
 #[cfg(test)]
 pub(crate) use page_meta::old_arena_block_ranges;
+
+#[cfg(test)]
+pub(crate) use region::new_test_block_backing;
+#[cfg(all(test, target_vendor = "apple"))]
+pub(crate) use region::release_test_block_backing;

crates/perry-runtime/src/arena/region.rs

@@ -1,6 +1,7 @@
 //! Measurement prototype: OS backing beneath the existing per-agent block pool.
 //! Linux uses one reserved range and persistent in-band descriptors.
-//! Other targets retain their existing backing and registry.
+//! Apple maps one tagged region per block; it and the other targets keep the
+//! process-wide registry. Other targets retain allocator backing.
 //! Production in-band descriptors and byte-store ownership are in REGION-DESIGN.
 
 use super::HeapGeneration;
@@ -11,6 +12,10 @@ use std::sync::RwLock;
 
 #[cfg(target_os = "linux")]
 pub(crate) mod reservation;
+
+#[cfg(target_vendor = "apple")]
+#[path = "region/apple.rs"]
+mod apple;
 #[cfg(target_os = "linux")]
 pub(crate) use reservation::contains;
 
@@ -27,7 +32,11 @@ fn extent_len(len: usize) -> usize {
     {
         mapped_len(len).expect("invalid region extent")
     }
-    #[cfg(not(target_os = "linux"))]
+    #[cfg(target_vendor = "apple")]
+    {
+        apple::extent(len).expect("invalid region extent")
+    }
+    #[cfg(not(any(target_os = "linux", target_vendor = "apple")))]
     {
         len
     }
@@ -87,7 +96,7 @@ pub(super) fn backing_len(len: usize) -> usize {
     }
     #[cfg(not(target_os = "linux"))]
     {
-        len
+        extent_len(len)
     }
 }
 
@@ -135,6 +144,15 @@ pub(super) unsafe fn map(kind: Kind, len: usize) -> *mut u8 {
     #[cfg(not(target_os = "linux"))]
     {
         let _ = kind;
+        #[cfg(target_vendor = "apple")]
+        let data = match apple::extent(len) {
+            Some(extent) => {
+                crate::gc::purge_allocator_on_region_growth();
+                apple::map(extent)
+            }
+            None => std::ptr::null_mut(),
+        };
+        #[cfg(not(target_vendor = "apple"))]
         let data = std::alloc::alloc(std::alloc::Layout::from_size_align(len, 16).unwrap());
         if !data.is_null() {
             register(data, len);
@@ -143,6 +161,26 @@ pub(super) unsafe fn map(kind: Kind, len: usize) -> *mut u8 {
     }
 }
 
+/// Release the pages of an empty owned region and return the bytes the
+/// kernel accepted. They must pass through [`reuse`] before the next write.
+#[cfg(target_vendor = "apple")]
+pub(super) unsafe fn release(data: *mut u8, len: usize) -> usize {
+    if apple::release(data, len) {
+        len
+    } else {
+        0
+    }
+}
+
+/// Reactivate released pages before they are handed out again.
+#[inline]
+pub(super) unsafe fn reuse(data: *mut u8, len: usize) {
+    #[cfg(target_vendor = "apple")]
+    apple::reuse(data, len);
+    #[cfg(not(target_vendor = "apple"))]
+    let _ = (data, len);
+}
+
 #[cfg(not(target_os = "linux"))]
 fn register(data: *mut u8, len: usize) {
     let start = data as usize;
@@ -162,10 +200,27 @@ pub(super) unsafe fn unmap(data: *mut u8, len: usize) {
         .remove(&(data as usize));
     #[cfg(target_os = "linux")]
     reservation::unmap(data, mapped_len(len).expect("invalid extent"));
-    #[cfg(not(target_os = "linux"))]
+    #[cfg(target_vendor = "apple")]
+    apple::unmap(data, extent_len(len));
+    #[cfg(not(any(target_os = "linux", target_vendor = "apple")))]
     std::alloc::dealloc(data, std::alloc::Layout::from_size_align(len, 16).unwrap());
 }
 
+/// Matching backing for fixtures that seed the real region pool.
+#[cfg(test)]
+pub(crate) unsafe fn new_test_block_backing(size: usize) -> *mut u8 {
+    map(Kind::NurseryBlock, size)
+}
+
+#[cfg(all(test, target_vendor = "apple"))]
+pub(crate) unsafe fn release_test_block_backing(data: *mut u8, size: usize) {
+    unmap(data, size)
+}
+
+#[cfg(all(test, target_vendor = "apple"))]
+#[path = "region/apple_tests.rs"]
+pub(super) mod apple_tests;
+
 #[cfg(all(test, target_os = "linux"))]
 mod tests {
     use super::*;