perf(codegen): inline births are black and seeded; omit redundant root-store shading (#11929)

changelog.d/12059-inline-births-root-remark.md

@@ -0,0 +1,18 @@
+Generated stores into compiler-managed GC roots no longer emit an
+incremental-mark shading gate and out-of-line call at every assignment.
+Budgeted collections already rescan every local and module-global root during
+their final remark, while synchronous collections expose no mutator window;
+heap stores, runtime-owned roots, weak reads and allocation coloring keep their
+existing barriers. This removes a major source of repeated cold code in large
+functions without changing relocations or GC root rewriting.
+
+All inline GC headers now incorporate the runtime's live per-thread birth
+flags in their initialization store. Non-leaf inline births use the same
+runtime seed protocol after every slot is valid and before publication.
+The existing per-thread seed queue is resolved before the allocation window,
+so the shared seed operation needs no TLS resolver or collecting call.
+This covers both small array literals and inline class allocation, including
+barrier-disabled build windows and post-remark mutator windows; birth flags
+remain off after the sweep snapshot. Whole-module IR checks, poisoned early
+seed controls, local-only array/class witnesses and 200 stable-root homes
+guard the protocol without per-site root-shading workarounds.

crates/perry-codegen/src/expr/array_literal.rs

@@ -235,6 +235,8 @@ pub(crate) fn emit_array_from_lowered_values(
             PTR,
             &[(&raw_fast, &fast_pred_label), (&raw_slow, &slow_pred_label)],
         );
+        let birth_flags = super::inline_birth::flags(ctx, &state_ptr);
+        let blk = ctx.block();
 
         // Packed GcHeader (bits 0..7 obj_type, 8..15 gc_flags, 16..31
         // _reserved, 32..63 size). PR #1146 packs the layout-tag in the
@@ -284,7 +286,9 @@ pub(crate) fn emit_array_from_lowered_values(
             }
             None => gc_packed.to_string(),
         };
-        // GC_STORE_AUDIT(INIT): freshly allocated array header starts pointer-free until slot notes below.
+        let header_word = super::inline_birth::header(ctx, &header_word, &birth_flags);
+        let blk = ctx.block();
+        // GC_STORE_AUDIT(INIT): live runtime birth flags are part of the fresh array header.
         blk.store(I64, &header_word, &raw);
 
         // Packed ArrayHeader at raw+8 (length low 32 / capacity high 32).
@@ -341,15 +345,44 @@ pub(crate) fn emit_array_from_lowered_values(
                         false,
                     );
                 }
-                blk.call(
-                    I32,
-                    "js_array_mark_numeric_f64_layout",
-                    &[(I64, &user_ptr_as_i64)],
-                );
                 blk.br(&done_label);
             }
             ctx.current_block = done_idx;
-            return Ok(user_ptr_as_i64);
+            super::inline_birth::finish(ctx, &raw, &birth_flags, &state_ptr);
+            if all_plain == "true" {
+                // Statically canonical doubles cannot reach normalization.
+                // Do not reserve a phantom temporary root in their function.
+                return Ok(user_ptr_as_i64);
+            }
+            // The generic normalizer's receiver resolution is Reenters.
+            // Keep it OUTSIDE the no-safepoint initialization window, and
+            // protect/re-read the initialized, seeded array across the call.
+            // Plain doubles retain their no-call/no-temporary-root hot path.
+            let normalize_idx = ctx.new_block("arrlit.normalize");
+            let return_idx = ctx.new_block("arrlit.return");
+            let normalize_label = ctx.block_label(normalize_idx);
+            let return_label = ctx.block_label(return_idx);
+            let plain_pred = ctx.block().label.clone();
+            ctx.block()
+                .cond_br(&all_plain, &return_label, &normalize_label);
+            ctx.current_block = normalize_idx;
+            let normalized = rooting::with_rooted_group(ctx, 1, |ctx, group| {
+                let root = group.adopt_emitted(ctx, rooting::Repr::Ptr, &user_ptr_as_i64, true);
+                let array = group.reread_emitted(ctx, root);
+                ctx.block()
+                    .call(I32, "js_array_mark_numeric_f64_layout", &[(I64, &array)]);
+                Ok(group.reread_emitted(ctx, root))
+            })?;
+            let normalized_pred = ctx.block().label.clone();
+            ctx.block().br(&return_label);
+            ctx.current_block = return_idx;
+            return Ok(ctx.block().phi(
+                I64,
+                &[
+                    (&user_ptr_as_i64, &plain_pred),
+                    (&normalized, &normalized_pred),
+                ],
+            ));
         }
 
         // Elements at raw+16 + i*8.
@@ -371,6 +404,7 @@ pub(crate) fn emit_array_from_lowered_values(
             );
         }
 
+        super::inline_birth::finish(ctx, &raw, &birth_flags, &state_ptr);
         return Ok(user_ptr_as_i64);
     }
 

crates/perry-codegen/src/expr/array_push_guard_tests.rs

@@ -754,8 +754,11 @@ fn class_field_push_checks_method_before_shared_argument_and_calls_builtin() {
         .find("call double @js_process_memory_usage(")
         .expect("allocating argument");
     assert!(
-        slow[..arg].contains("@js_object_get_field_ic"),
-        "method read must precede argument:\n{slow}"
+        slow.contains("@js_object_get_field_ic")
+            && slow[..arg].contains("phi double")
+            && slow[..arg].contains("pget.recv_merge"),
+        "the argument block must receive the method read's merged value before evaluating the \
+         argument:\n{slow}"
     );
 }
 

crates/perry-codegen/src/expr/generic_overhead_tests.rs

@@ -104,12 +104,16 @@ fn scalar_global_stores_keep_the_store_without_root_shading() {
 }
 
 #[test]
-fn unknown_and_declared_number_globals_keep_root_shading() {
+fn every_registered_global_store_omits_redundant_root_shading() {
     for ty in [Type::Any, Type::Number] {
         let ir = global_store_ir(Expr::LocalGet(1), ty);
         assert!(
-            ir.contains("call void @js_write_barrier_root_nanbox("),
-            "annotation must not suppress the barrier:\n{ir}"
+            ir.contains("store double") && ir.contains("@perry_global_cost_test__99"),
+            "registered-global store disappeared:\n{ir}"
+        );
+        assert!(
+            !ir.contains("call void @js_write_barrier_root_nanbox("),
+            "generated roots are fully rescanned at FinalRootRemark:\n{ir}"
         );
     }
     for value in [
@@ -119,8 +123,8 @@ fn unknown_and_declared_number_globals_keep_root_shading() {
     ] {
         let ir = global_store_ir(value, Type::Any);
         assert!(
-            ir.contains("call void @js_write_barrier_root_nanbox("),
-            "heap barrier missing:\n{ir}"
+            !ir.contains("call void @js_write_barrier_root_nanbox("),
+            "heap-valued registered roots must use the final remark, not per-store shading:\n{ir}"
         );
     }
 }

crates/perry-codegen/src/expr/hit_path_access_tests.rs

@@ -236,8 +236,17 @@ fn plain_double_array_literal_skips_notes_and_marking() {
     );
     let noted = block_body(&ir, "arrlit.noted").unwrap_or_else(|| panic!("no noted arm:\n{ir}"));
     assert!(
-        noted.contains("@js_array_mark_numeric_f64_layout("),
-        "a boxed element must keep the marking walk:\n{noted}"
+        noted.contains("@js_gc_note_slot_layout(")
+            && !noted.contains("@js_array_mark_numeric_f64_layout("),
+        "the noted arm initializes metadata without running a collecting normalizer:\n{noted}"
+    );
+    let normalize = block_body(&ir, "arrlit.normalize")
+        .unwrap_or_else(|| panic!("boxed elements must keep a normalizer arm:\n{ir}"));
+    assert!(normalize.contains("@js_array_mark_numeric_f64_layout("));
+    assert!(
+        ir.find("@js_gc_note_black_birth(").unwrap()
+            < ir.find("@js_array_mark_numeric_f64_layout(").unwrap(),
+        "normalization must follow the completed birth seed"
     );
 }
 

crates/perry-codegen/src/expr/hit_path_tests.rs

@@ -325,7 +325,7 @@ fn increment_has_a_call_free_double_arm() {
 }
 
 #[test]
-fn module_global_increment_gates_its_root_barrier() {
+fn module_global_increment_uses_final_remark_instead_of_root_shading() {
     let mut module = Module::new("global_update.ts");
     module.init.push(let_any(X, "g", Expr::Undefined));
     let bump = function(
@@ -359,9 +359,13 @@ fn module_global_increment_gates_its_root_barrier() {
         "the double arm must not call:\n{body}"
     );
     assert!(
-        body.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT")
-            && body.contains("@js_write_barrier_root_nanbox("),
-        "the coercing arm's root store keeps its gated barrier:\n{body}"
+        body.matches("store double").count() >= 2,
+        "both increment arms must update the registered global:\n{body}"
+    );
+    assert!(
+        !body.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT")
+            && !body.contains("@js_write_barrier_root_nanbox("),
+        "compiler-managed roots use FinalRootRemark, not per-store shading:\n{body}"
     );
 }
 

crates/perry-codegen/src/expr/inline_birth.rs

@@ -0,0 +1,60 @@
+//! Inline allocations use the runtime's live birth color and seed protocol.
+use super::FnCtx;
+use crate::types::{I64, I8, PTR};
+
+pub(super) const BIRTH_FLAGS_OFFSET: &str = "24";
+pub(super) const BIRTH_SEEDS_OFFSET: &str = "32";
+
+/// Read the cell VALUE at every allocation, after the potentially collecting
+/// slow arm. Only the thread-local cell's stable address is cached in state.
+pub(crate) fn flags(ctx: &mut FnCtx<'_>, state: &str) -> String {
+    let blk = ctx.block();
+    let address_field = blk.gep(I8, state, &[(I64, BIRTH_FLAGS_OFFSET)]);
+    let address = blk.load(PTR, &address_field);
+    let flags = blk.next_reg();
+    blk.emit_raw(format!("{flags} = load volatile i8, ptr {address}"));
+    blk.zext(I8, &flags, I64)
+}
+
+/// Include live flags in the SAME packed header store, not a later patch.
+pub(crate) fn header(ctx: &mut FnCtx<'_>, packed: &str, flags: &str) -> String {
+    let blk = ctx.block();
+    let shifted = blk.shl(I64, flags, "8");
+    blk.or(I64, packed, &shifted)
+}
+
+/// All slots must be initialized before this GC-leaf call and before any
+/// collecting call. The runtime's type metadata decides whether to seed.
+/// Idle/sweep births pay no call; unlike the insertion-barrier counter this
+/// gate also covers BuildValidPointerSet's barrier-disabled mutator windows.
+pub(crate) fn finish(ctx: &mut FnCtx<'_>, raw: &str, flags: &str, state: &str) {
+    let active = ctx.block().icmp_ne(I64, flags, "0");
+    let seed_idx = ctx.new_block("birth.seed");
+    let done_idx = ctx.new_block("birth.ready");
+    let seed_label = ctx.block_label(seed_idx);
+    let done_label = ctx.block_label(done_idx);
+    ctx.block().cond_br(&active, &seed_label, &done_label);
+    ctx.current_block = seed_idx;
+    let seed_field = ctx.block().gep(I8, state, &[(I64, BIRTH_SEEDS_OFFSET)]);
+    let seeds = ctx.block().load(PTR, &seed_field);
+    ctx.block()
+        .call_void("js_gc_note_black_birth", &[(PTR, raw), (PTR, &seeds)]);
+    ctx.block().br(&done_label);
+    ctx.current_block = done_idx;
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn inline_birth_cell_offset_matches_runtime() {
+        let runtime = include_str!("../../../perry-runtime/src/arena/inline.rs");
+        assert!(runtime.contains(&format!(
+            "pub const INLINE_BIRTH_FLAGS_OFFSET_LP64: usize = {};",
+            super::BIRTH_FLAGS_OFFSET
+        )));
+        assert!(runtime.contains(&format!(
+            "pub const INLINE_BIRTH_SEEDS_OFFSET_LP64: usize = {};",
+            super::BIRTH_SEEDS_OFFSET
+        )));
+    }
+}

crates/perry-codegen/src/expr/mod.rs

@@ -88,6 +88,7 @@ mod bitset_test;
 pub(crate) mod body_call;
 pub(crate) mod folded_builtin_override;
 pub(crate) mod hot_tls;
+pub(crate) mod inline_birth;
 mod literal_descriptor;
 #[cfg(test)]
 mod map_entry_at_tests;
@@ -300,10 +301,10 @@ pub(crate) use scalar_slot_root::{
     root_scalar_replaced_slot_unconditional,
 };
 pub(crate) use shadow_slot::{
-    current_closure_ptr_value, emit_persistent_shadow_root_barrier,
-    emit_shadow_slot_bind_for_local, emit_shadow_slot_clear, emit_shadow_slot_update_for_expr,
-    enable_persistent_shadow_slot_for_array_alias, expr_is_known_non_pointer_shadow_value,
-    root_inlined_ctor_pointer_locals, try_current_closure_ptr_value,
+    current_closure_ptr_value, emit_shadow_slot_bind_for_local, emit_shadow_slot_clear,
+    emit_shadow_slot_update_for_expr, enable_persistent_shadow_slot_for_array_alias,
+    expr_is_known_non_pointer_shadow_value, root_inlined_ctor_pointer_locals,
+    try_current_closure_ptr_value,
 };
 
 /// One in-flight inline-constructor return target. See
@@ -2646,7 +2647,11 @@ pub(crate) fn load_inline_arena_state(ctx: &mut FnCtx<'_>) -> String {
             let blk = ctx.block();
             let state = blk.load(PTR, &state_ptr);
             let data = blk.load(PTR, &state);
-            let initialised = blk.icmp_ne(PTR, &data, "null");
+            let data_initialised = blk.icmp_ne(PTR, &data, "null");
+            let birth_field = blk.gep(I8, &state, &[(I64, inline_birth::BIRTH_FLAGS_OFFSET)]);
+            let birth_address = blk.load(PTR, &birth_field);
+            let birth_initialised = blk.icmp_ne(PTR, &birth_address, "null");
+            let initialised = blk.and(crate::types::I1, &data_initialised, &birth_initialised);
             blk.cond_br(&initialised, &ready_label, &slow_label);
             state
         };
@@ -3145,8 +3150,6 @@ mod index_set_packed_loop;
 mod index_set_typed_array;
 mod instance_misc1;
 mod member_update;
-#[cfg(test)]
-mod packed_loop_shadow_barrier_tests;
 mod typed_array_rmw;
 mod typed_array_update;
 pub(crate) use instance_misc1::builtin_parent_reserved_class_id;