diff --git a/changelog.d/12059-inline-births-root-remark.md b/changelog.d/12059-inline-births-root-remark.md new file mode 100644 index 0000000000..ebca5e72e5 --- /dev/null +++ b/changelog.d/12059-inline-births-root-remark.md @@ -0,0 +1,18 @@ +Generated stores into compiler-managed GC roots no longer emit an +incremental-mark shading gate and out-of-line call at every assignment. +Budgeted collections already rescan every local and module-global root during +their final remark, while synchronous collections expose no mutator window; +heap stores, runtime-owned roots, weak reads and allocation coloring keep their +existing barriers. This removes a major source of repeated cold code in large +functions without changing relocations or GC root rewriting. + +All inline GC headers now incorporate the runtime's live per-thread birth +flags in their initialization store. Non-leaf inline births use the same +runtime seed protocol after every slot is valid and before publication. +The existing per-thread seed queue is resolved before the allocation window, +so the shared seed operation needs no TLS resolver or collecting call. +This covers both small array literals and inline class allocation, including +barrier-disabled build windows and post-remark mutator windows; birth flags +remain off after the sweep snapshot. Whole-module IR checks, poisoned early +seed controls, local-only array/class witnesses and 200 stable-root homes +guard the protocol without per-site root-shading workarounds. diff --git a/crates/perry-codegen/src/expr/array_literal.rs b/crates/perry-codegen/src/expr/array_literal.rs index 70223f7a6a..1e77816f06 100644 --- a/crates/perry-codegen/src/expr/array_literal.rs +++ b/crates/perry-codegen/src/expr/array_literal.rs @@ -235,6 +235,8 @@ pub(crate) fn emit_array_from_lowered_values( PTR, &[(&raw_fast, &fast_pred_label), (&raw_slow, &slow_pred_label)], ); + let birth_flags = super::inline_birth::flags(ctx, &state_ptr); + let blk = ctx.block(); // Packed GcHeader (bits 0..7 obj_type, 8..15 gc_flags, 16..31 // _reserved, 32..63 size). PR #1146 packs the layout-tag in the @@ -284,7 +286,9 @@ pub(crate) fn emit_array_from_lowered_values( } None => gc_packed.to_string(), }; - // GC_STORE_AUDIT(INIT): freshly allocated array header starts pointer-free until slot notes below. + let header_word = super::inline_birth::header(ctx, &header_word, &birth_flags); + let blk = ctx.block(); + // GC_STORE_AUDIT(INIT): live runtime birth flags are part of the fresh array header. blk.store(I64, &header_word, &raw); // Packed ArrayHeader at raw+8 (length low 32 / capacity high 32). @@ -341,15 +345,44 @@ pub(crate) fn emit_array_from_lowered_values( false, ); } - blk.call( - I32, - "js_array_mark_numeric_f64_layout", - &[(I64, &user_ptr_as_i64)], - ); blk.br(&done_label); } ctx.current_block = done_idx; - return Ok(user_ptr_as_i64); + super::inline_birth::finish(ctx, &raw, &birth_flags, &state_ptr); + if all_plain == "true" { + // Statically canonical doubles cannot reach normalization. + // Do not reserve a phantom temporary root in their function. + return Ok(user_ptr_as_i64); + } + // The generic normalizer's receiver resolution is Reenters. + // Keep it OUTSIDE the no-safepoint initialization window, and + // protect/re-read the initialized, seeded array across the call. + // Plain doubles retain their no-call/no-temporary-root hot path. + let normalize_idx = ctx.new_block("arrlit.normalize"); + let return_idx = ctx.new_block("arrlit.return"); + let normalize_label = ctx.block_label(normalize_idx); + let return_label = ctx.block_label(return_idx); + let plain_pred = ctx.block().label.clone(); + ctx.block() + .cond_br(&all_plain, &return_label, &normalize_label); + ctx.current_block = normalize_idx; + let normalized = rooting::with_rooted_group(ctx, 1, |ctx, group| { + let root = group.adopt_emitted(ctx, rooting::Repr::Ptr, &user_ptr_as_i64, true); + let array = group.reread_emitted(ctx, root); + ctx.block() + .call(I32, "js_array_mark_numeric_f64_layout", &[(I64, &array)]); + Ok(group.reread_emitted(ctx, root)) + })?; + let normalized_pred = ctx.block().label.clone(); + ctx.block().br(&return_label); + ctx.current_block = return_idx; + return Ok(ctx.block().phi( + I64, + &[ + (&user_ptr_as_i64, &plain_pred), + (&normalized, &normalized_pred), + ], + )); } // Elements at raw+16 + i*8. @@ -371,6 +404,7 @@ pub(crate) fn emit_array_from_lowered_values( ); } + super::inline_birth::finish(ctx, &raw, &birth_flags, &state_ptr); return Ok(user_ptr_as_i64); } diff --git a/crates/perry-codegen/src/expr/array_push_guard_tests.rs b/crates/perry-codegen/src/expr/array_push_guard_tests.rs index 1398d68321..514fff8da9 100644 --- a/crates/perry-codegen/src/expr/array_push_guard_tests.rs +++ b/crates/perry-codegen/src/expr/array_push_guard_tests.rs @@ -754,8 +754,11 @@ fn class_field_push_checks_method_before_shared_argument_and_calls_builtin() { .find("call double @js_process_memory_usage(") .expect("allocating argument"); assert!( - slow[..arg].contains("@js_object_get_field_ic"), - "method read must precede argument:\n{slow}" + slow.contains("@js_object_get_field_ic") + && slow[..arg].contains("phi double") + && slow[..arg].contains("pget.recv_merge"), + "the argument block must receive the method read's merged value before evaluating the \ + argument:\n{slow}" ); } diff --git a/crates/perry-codegen/src/expr/generic_overhead_tests.rs b/crates/perry-codegen/src/expr/generic_overhead_tests.rs index 32d779ab98..891930c681 100644 --- a/crates/perry-codegen/src/expr/generic_overhead_tests.rs +++ b/crates/perry-codegen/src/expr/generic_overhead_tests.rs @@ -104,12 +104,16 @@ fn scalar_global_stores_keep_the_store_without_root_shading() { } #[test] -fn unknown_and_declared_number_globals_keep_root_shading() { +fn every_registered_global_store_omits_redundant_root_shading() { for ty in [Type::Any, Type::Number] { let ir = global_store_ir(Expr::LocalGet(1), ty); assert!( - ir.contains("call void @js_write_barrier_root_nanbox("), - "annotation must not suppress the barrier:\n{ir}" + ir.contains("store double") && ir.contains("@perry_global_cost_test__99"), + "registered-global store disappeared:\n{ir}" + ); + assert!( + !ir.contains("call void @js_write_barrier_root_nanbox("), + "generated roots are fully rescanned at FinalRootRemark:\n{ir}" ); } for value in [ @@ -119,8 +123,8 @@ fn unknown_and_declared_number_globals_keep_root_shading() { ] { let ir = global_store_ir(value, Type::Any); assert!( - ir.contains("call void @js_write_barrier_root_nanbox("), - "heap barrier missing:\n{ir}" + !ir.contains("call void @js_write_barrier_root_nanbox("), + "heap-valued registered roots must use the final remark, not per-store shading:\n{ir}" ); } } diff --git a/crates/perry-codegen/src/expr/hit_path_access_tests.rs b/crates/perry-codegen/src/expr/hit_path_access_tests.rs index 61f5b5af98..a78d790d8d 100644 --- a/crates/perry-codegen/src/expr/hit_path_access_tests.rs +++ b/crates/perry-codegen/src/expr/hit_path_access_tests.rs @@ -236,8 +236,17 @@ fn plain_double_array_literal_skips_notes_and_marking() { ); let noted = block_body(&ir, "arrlit.noted").unwrap_or_else(|| panic!("no noted arm:\n{ir}")); assert!( - noted.contains("@js_array_mark_numeric_f64_layout("), - "a boxed element must keep the marking walk:\n{noted}" + noted.contains("@js_gc_note_slot_layout(") + && !noted.contains("@js_array_mark_numeric_f64_layout("), + "the noted arm initializes metadata without running a collecting normalizer:\n{noted}" + ); + let normalize = block_body(&ir, "arrlit.normalize") + .unwrap_or_else(|| panic!("boxed elements must keep a normalizer arm:\n{ir}")); + assert!(normalize.contains("@js_array_mark_numeric_f64_layout(")); + assert!( + ir.find("@js_gc_note_black_birth(").unwrap() + < ir.find("@js_array_mark_numeric_f64_layout(").unwrap(), + "normalization must follow the completed birth seed" ); } diff --git a/crates/perry-codegen/src/expr/hit_path_tests.rs b/crates/perry-codegen/src/expr/hit_path_tests.rs index 5138f775a8..76e2489aa9 100644 --- a/crates/perry-codegen/src/expr/hit_path_tests.rs +++ b/crates/perry-codegen/src/expr/hit_path_tests.rs @@ -325,7 +325,7 @@ fn increment_has_a_call_free_double_arm() { } #[test] -fn module_global_increment_gates_its_root_barrier() { +fn module_global_increment_uses_final_remark_instead_of_root_shading() { let mut module = Module::new("global_update.ts"); module.init.push(let_any(X, "g", Expr::Undefined)); let bump = function( @@ -359,9 +359,13 @@ fn module_global_increment_gates_its_root_barrier() { "the double arm must not call:\n{body}" ); assert!( - body.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT") - && body.contains("@js_write_barrier_root_nanbox("), - "the coercing arm's root store keeps its gated barrier:\n{body}" + body.matches("store double").count() >= 2, + "both increment arms must update the registered global:\n{body}" + ); + assert!( + !body.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT") + && !body.contains("@js_write_barrier_root_nanbox("), + "compiler-managed roots use FinalRootRemark, not per-store shading:\n{body}" ); } diff --git a/crates/perry-codegen/src/expr/inline_birth.rs b/crates/perry-codegen/src/expr/inline_birth.rs new file mode 100644 index 0000000000..16b0e9af7c --- /dev/null +++ b/crates/perry-codegen/src/expr/inline_birth.rs @@ -0,0 +1,60 @@ +//! Inline allocations use the runtime's live birth color and seed protocol. +use super::FnCtx; +use crate::types::{I64, I8, PTR}; + +pub(super) const BIRTH_FLAGS_OFFSET: &str = "24"; +pub(super) const BIRTH_SEEDS_OFFSET: &str = "32"; + +/// Read the cell VALUE at every allocation, after the potentially collecting +/// slow arm. Only the thread-local cell's stable address is cached in state. +pub(crate) fn flags(ctx: &mut FnCtx<'_>, state: &str) -> String { + let blk = ctx.block(); + let address_field = blk.gep(I8, state, &[(I64, BIRTH_FLAGS_OFFSET)]); + let address = blk.load(PTR, &address_field); + let flags = blk.next_reg(); + blk.emit_raw(format!("{flags} = load volatile i8, ptr {address}")); + blk.zext(I8, &flags, I64) +} + +/// Include live flags in the SAME packed header store, not a later patch. +pub(crate) fn header(ctx: &mut FnCtx<'_>, packed: &str, flags: &str) -> String { + let blk = ctx.block(); + let shifted = blk.shl(I64, flags, "8"); + blk.or(I64, packed, &shifted) +} + +/// All slots must be initialized before this GC-leaf call and before any +/// collecting call. The runtime's type metadata decides whether to seed. +/// Idle/sweep births pay no call; unlike the insertion-barrier counter this +/// gate also covers BuildValidPointerSet's barrier-disabled mutator windows. +pub(crate) fn finish(ctx: &mut FnCtx<'_>, raw: &str, flags: &str, state: &str) { + let active = ctx.block().icmp_ne(I64, flags, "0"); + let seed_idx = ctx.new_block("birth.seed"); + let done_idx = ctx.new_block("birth.ready"); + let seed_label = ctx.block_label(seed_idx); + let done_label = ctx.block_label(done_idx); + ctx.block().cond_br(&active, &seed_label, &done_label); + ctx.current_block = seed_idx; + let seed_field = ctx.block().gep(I8, state, &[(I64, BIRTH_SEEDS_OFFSET)]); + let seeds = ctx.block().load(PTR, &seed_field); + ctx.block() + .call_void("js_gc_note_black_birth", &[(PTR, raw), (PTR, &seeds)]); + ctx.block().br(&done_label); + ctx.current_block = done_idx; +} + +#[cfg(test)] +mod tests { + #[test] + fn inline_birth_cell_offset_matches_runtime() { + let runtime = include_str!("../../../perry-runtime/src/arena/inline.rs"); + assert!(runtime.contains(&format!( + "pub const INLINE_BIRTH_FLAGS_OFFSET_LP64: usize = {};", + super::BIRTH_FLAGS_OFFSET + ))); + assert!(runtime.contains(&format!( + "pub const INLINE_BIRTH_SEEDS_OFFSET_LP64: usize = {};", + super::BIRTH_SEEDS_OFFSET + ))); + } +} diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 9cd5c4695a..9a4fca60e7 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -88,6 +88,7 @@ mod bitset_test; pub(crate) mod body_call; pub(crate) mod folded_builtin_override; pub(crate) mod hot_tls; +pub(crate) mod inline_birth; mod literal_descriptor; #[cfg(test)] mod map_entry_at_tests; @@ -300,10 +301,10 @@ pub(crate) use scalar_slot_root::{ root_scalar_replaced_slot_unconditional, }; pub(crate) use shadow_slot::{ - current_closure_ptr_value, emit_persistent_shadow_root_barrier, - emit_shadow_slot_bind_for_local, emit_shadow_slot_clear, emit_shadow_slot_update_for_expr, - enable_persistent_shadow_slot_for_array_alias, expr_is_known_non_pointer_shadow_value, - root_inlined_ctor_pointer_locals, try_current_closure_ptr_value, + current_closure_ptr_value, emit_shadow_slot_bind_for_local, emit_shadow_slot_clear, + emit_shadow_slot_update_for_expr, enable_persistent_shadow_slot_for_array_alias, + expr_is_known_non_pointer_shadow_value, root_inlined_ctor_pointer_locals, + try_current_closure_ptr_value, }; /// One in-flight inline-constructor return target. See @@ -2646,7 +2647,11 @@ pub(crate) fn load_inline_arena_state(ctx: &mut FnCtx<'_>) -> String { let blk = ctx.block(); let state = blk.load(PTR, &state_ptr); let data = blk.load(PTR, &state); - let initialised = blk.icmp_ne(PTR, &data, "null"); + let data_initialised = blk.icmp_ne(PTR, &data, "null"); + let birth_field = blk.gep(I8, &state, &[(I64, inline_birth::BIRTH_FLAGS_OFFSET)]); + let birth_address = blk.load(PTR, &birth_field); + let birth_initialised = blk.icmp_ne(PTR, &birth_address, "null"); + let initialised = blk.and(crate::types::I1, &data_initialised, &birth_initialised); blk.cond_br(&initialised, &ready_label, &slow_label); state }; @@ -3145,8 +3150,6 @@ mod index_set_packed_loop; mod index_set_typed_array; mod instance_misc1; mod member_update; -#[cfg(test)] -mod packed_loop_shadow_barrier_tests; mod typed_array_rmw; mod typed_array_update; pub(crate) use instance_misc1::builtin_parent_reserved_class_id; diff --git a/crates/perry-codegen/src/expr/packed_loop_shadow_barrier_tests.rs b/crates/perry-codegen/src/expr/packed_loop_shadow_barrier_tests.rs deleted file mode 100644 index 007ce41a6a..0000000000 --- a/crates/perry-codegen/src/expr/packed_loop_shadow_barrier_tests.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! The packed-numeric clone's counter read is a proven Number, so binding it to -//! a `const` must not shade a GC root — and `arr[i ± c]` must still shade one. -//! -//! `expr_is_known_non_pointer_shadow_value` suppresses -//! `emit_persistent_shadow_root_barrier` for a value that cannot be a heap -//! reference. Inside an active packed-numeric loop fact the entry guard has -//! proved a dense raw-f64 plain Array, the clone has no safepoint and no growth -//! (#9379), and the fast condition bounds the counter by the length read at loop -//! entry — so `arr[i]` reads a raw numeric word. -//! -//! `arr[i + 1]` has none of that: the index can leave the array, and an -//! out-of-bounds element read consults the prototype chain, where -//! `Array.prototype[7] = {}` is a genuine heap reference that must stay rooted. -//! The suppression is therefore restricted to offset 0, and this file is the -//! assertion that the restriction is real. -//! -//! Both reads live in the SAME fast clone, so neither direction can pass -//! vacuously: if the offset read were wrongly admitted the barrier count in the -//! fast body would be 0, and if the counter read were wrongly refused it would -//! be 2. Every test also asserts the clone was entered at all — a barrier count -//! taken over blocks that were never emitted is CLAUDE.md hazard 4. - -use perry_hir::types::Type; -use perry_hir::{BinaryOp, CompareOp, Expr, Function, Module as HirModule, Param, Stmt, UpdateOp}; - -const ARR: u32 = 0; -const SUM: u32 = 1; -const IDX: u32 = 2; -const BOUND_V: u32 = 3; -const BOUND_W: u32 = 4; - -/// The root-shading barrier's inline arming test, emitted once per shaded store. -const SHADING_TEST: &str = "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT"; - -fn compile(name: &str, body: Vec) -> String { - let mut hir = HirModule::new(name); - hir.functions.push(Function { - id: 0, - name: "build".to_string(), - type_params: Vec::new(), - params: vec![Param { - id: ARR, - name: "a".to_string(), - ty: Type::Array(Box::new(Type::Number)), - default: None, - decorators: Vec::new(), - is_rest: false, - arguments_object: None, - }], - return_type: Type::Number, - body, - is_async: false, - is_generator: false, - is_strict: true, - is_exported: false, - captures: Vec::new(), - decorators: Vec::new(), - was_plain_async: false, - was_unrolled: false, - }); - let opts = crate::CompileOptions { - emit_ir_only: true, - ..Default::default() - }; - String::from_utf8(crate::compile_module(&hir, opts).expect("test module compiles")) - .expect("LLVM IR is UTF-8") -} - -/// The first emitted block whose label starts with `prefix`, up to the next -/// top-level label. Panics when the block is absent, so an assertion can never -/// be taken over a clone that was not emitted. -fn block(ir: &str, prefix: &str) -> String { - let start = ir - .find(&format!("\n{prefix}")) - .unwrap_or_else(|| panic!("no block labelled {prefix}* was emitted:\n{ir}")); - let rest = &ir[start + 1..]; - let body_start = rest.find(":\n").expect("a block label ends in a colon") + 2; - let mut end = rest.len(); - let mut at = body_start; - for line in rest[body_start..].split_inclusive('\n') { - let trimmed = line.trim_end(); - if trimmed.ends_with(':') && !trimmed.starts_with(' ') && !trimmed.is_empty() { - end = at; - break; - } - at += line.len(); - } - rest[..end].to_string() -} - -/// The clone's fast body — where a counter read's binding store lands. -fn fast_body(ir: &str) -> String { - block(ir, "for.packed_f64_fast.body") -} - -/// Where an `arr[i ± c]` binding store lands instead: the offset read carries an -/// inline bounds check that side-exits to the slow preheader, and its store sits -/// past that check rather than in the body block. -fn offset_read_block(ir: &str) -> String { - block(ir, "packed_f64_loop.foreign.inbounds") -} - -/// `let s = 0; for (let i = 0; i < a.length; i++) { } return s;` -fn packed_loop_with(bindings: Vec) -> Vec { - vec![ - Stmt::Let { - id: SUM, - name: "s".into(), - ty: Type::Number, - init: Some(Expr::Number(0.0)), - mutable: true, - }, - Stmt::For { - init: Some(Box::new(Stmt::Let { - id: IDX, - name: "i".into(), - ty: Type::Number, - init: Some(Expr::Integer(0)), - mutable: true, - })), - condition: Some(Expr::Compare { - op: CompareOp::Lt, - left: Box::new(Expr::LocalGet(IDX)), - right: Box::new(Expr::PropertyGet { - object: Box::new(Expr::LocalGet(ARR)), - property: "length".to_string(), - byte_offset: 0, - }), - }), - update: Some(Expr::Update { - id: IDX, - op: UpdateOp::Increment, - prefix: false, - }), - body: bindings, - }, - Stmt::Return(Some(Expr::LocalGet(SUM))), - ] -} - -/// `Type::Any`, not `Type::Number`: a number-annotated local is never given a -/// shadow slot, so a barrier could not be emitted for it under any predicate -/// and both arms of the comparison would read 0. The `for…of` desugaring this -/// models erases the element type, which is what earns the slot in the first -/// place — and what makes the suppression worth anything. -fn bind(id: u32, name: &str, index: Expr) -> Stmt { - Stmt::Let { - id, - name: name.into(), - ty: Type::Any, - init: Some(Expr::IndexGet { - object: Box::new(Expr::LocalGet(ARR)), - index: Box::new(index), - }), - mutable: false, - } -} - -fn accumulate(id: u32) -> Stmt { - Stmt::Expr(Expr::LocalSet( - SUM, - Box::new(Expr::Binary { - op: BinaryOp::Add, - left: Box::new(Expr::LocalGet(SUM)), - right: Box::new(Expr::LocalGet(id)), - }), - )) -} - -fn counter_read() -> Expr { - Expr::LocalGet(IDX) -} - -fn offset_read() -> Expr { - Expr::Binary { - op: BinaryOp::Add, - left: Box::new(Expr::LocalGet(IDX)), - right: Box::new(Expr::Integer(1)), - } -} - -/// `const v = a[i]` alone: the clone's fast body shades nothing. -#[test] -fn a_packed_loop_counter_read_binding_shades_no_root() { - let ir = compile( - "packed_counter_read", - packed_loop_with(vec![ - bind(BOUND_V, "v", counter_read()), - accumulate(BOUND_V), - ]), - ); - let body = fast_body(&ir); - assert_eq!( - body.matches(SHADING_TEST).count(), - 0, - "a[i] under a packed-numeric fact is a proven Number; binding it must shade no root:\n{body}" - ); -} - -/// `const w = a[i + 1]` alone: still shaded, because the read can leave the -/// array and reach a prototype index holding a heap reference. -#[test] -fn a_packed_loop_offset_read_binding_still_shades_its_root() { - let ir = compile( - "packed_offset_read", - packed_loop_with(vec![bind(BOUND_W, "w", offset_read()), accumulate(BOUND_W)]), - ); - let guarded = offset_read_block(&ir); - assert_eq!( - guarded.matches(SHADING_TEST).count(), - 1, - "a[i + 1] can read past the array into the prototype chain; its binding must stay \ - shaded:\n{guarded}" - ); -} - -/// Both in one clone, which is what makes neither direction vacuous: the -/// counter read must contribute nothing to the fast body and the offset read -/// must still contribute exactly one to its own guarded block. A non-zero body -/// count would mean the counter read stopped being recognised and the -/// optimisation is dead; a zero guarded count would mean the offset read was -/// wrongly admitted and a prototype-held object could go unshaded. -#[test] -fn only_the_offset_read_shades_when_both_live_in_one_clone() { - let ir = compile( - "packed_counter_and_offset", - packed_loop_with(vec![ - bind(BOUND_V, "v", counter_read()), - accumulate(BOUND_V), - bind(BOUND_W, "w", offset_read()), - accumulate(BOUND_W), - ]), - ); - assert_eq!( - fast_body(&ir).matches(SHADING_TEST).count(), - 0, - "the counter read's binding must shade nothing:\n{}", - fast_body(&ir) - ); - assert_eq!( - offset_read_block(&ir).matches(SHADING_TEST).count(), - 1, - "the offset read's binding must still be shaded:\n{}", - offset_read_block(&ir) - ); -} diff --git a/crates/perry-codegen/src/expr/scalar_slot_root.rs b/crates/perry-codegen/src/expr/scalar_slot_root.rs index 16de41008e..8ffbbd35e0 100644 --- a/crates/perry-codegen/src/expr/scalar_slot_root.rs +++ b/crates/perry-codegen/src/expr/scalar_slot_root.rs @@ -33,13 +33,13 @@ //! rewrite pass reach the real alloca rather than a stale mirror. //! //! The bind runs **once, in the function-entry setup** — not at each store. -//! What a bind does is `slot_ptrs[idx] = alloca; stack[idx] = *alloca; -//! active[idx] = true; root_barrier(*alloca)`. For an entry-hoisted alloca the -//! first three are loop-invariant: the address never changes, and every reader +//! It records `slot_ptrs[idx] = alloca`, making the alloca an active mutable +//! root. For an entry-hoisted alloca this is loop-invariant: the address never +//! changes, and every reader //! of a bound slot (`visit_shadow_stack_root_slots`, `js_shadow_slot_get`) //! dereferences `slot_ptrs[idx]` in preference to the `stack[idx]` mirror, so -//! the mirror is dead storage. Only the root barrier is per-store work, and it -//! is emitted inline and guarded (`emit_persistent_shadow_root_barrier`). +//! the mirror is dead storage. FinalRootRemark rescans the alloca, so there is +//! no per-store root barrier work. //! //! This is the same treatment `enable_persistent_shadow_slot_for_array_alias` //! already gives a `const item = arr[i]` alias, for the same reason. @@ -71,17 +71,14 @@ use super::*; use perry_hir::Expr; -use crate::types::{I32, I64, PTR}; +use crate::types::{I32, PTR}; /// Root the scalar-replacement alloca `slot` against the value expression /// that was just stored into it. /// -/// Call *after* the `store` — the emitted root barrier reads the alloca back, -/// so the new value has to be in place. Callers that store a canonicalized raw -/// `f64` (the `numeric_store` arm of `expr::property_set`) must not call this -/// at all: those bits are a plain double by construction, and the shared -/// root-word decoder rejects them, but reserving a slot for them would be pure -/// waste. +/// Callers that store a canonicalized raw `f64` (the `numeric_store` arm of +/// `expr::property_set`) must not call this at all: those bits are a plain +/// double by construction, and reserving a slot for them would be pure waste. pub(crate) fn root_scalar_replaced_slot(ctx: &mut FnCtx<'_>, slot: &str, value: &Expr) { if expr_is_known_non_pointer_shadow_value(ctx, value) { return; @@ -143,8 +140,8 @@ pub(crate) fn entry_init_load_rooted_global( /// hoisted to entry setup, which makes the slot `active` from function entry /// — the collector dereferences it before any store reaches it, and /// uninitialized stack garbage can pass `is_plausible_heap_addr`. -/// 2. **Call this *after* the store**, not before: the emitted root barrier -/// reads the alloca back. +/// 2. Call this on every path that can first make the alloca pointer-capable, +/// so the entry bind is emitted independent of control-flow order. /// /// Binding (rather than temp-rooting) is what makes this a one-line fix at /// ~30 read sites: every reader already does `load DOUBLE, ptr `, and @@ -168,19 +165,4 @@ pub(crate) fn root_entry_alloca(ctx: &mut FnCtx<'_>, slot: &str) { &[(I32, &idx.to_string()), (PTR, slot)], ); } - emit_scalar_slot_store_barrier(ctx, slot); -} - -/// The per-store remainder of a bind: shade the newly stored value so an -/// in-flight incremental mark cannot miss it. -/// -/// The operand is read back from the alloca rather than threaded down from the -/// caller's value register **because that is precisely what the bind it -/// replaces did** (`js_shadow_slot_bind` dereferences `value_slot`). The load -/// sits in the same block, immediately after the store that produced the value, -/// with nothing in between — it cannot observe a later write, and LLVM forwards -/// it to the stored register. -fn emit_scalar_slot_store_barrier(ctx: &mut FnCtx<'_>, slot: &str) { - let value_bits = ctx.block().load(I64, slot); - crate::expr::emit_persistent_shadow_root_barrier(ctx, &value_bits); } diff --git a/crates/perry-codegen/src/expr/shadow_inline.rs b/crates/perry-codegen/src/expr/shadow_inline.rs index 2a03e34333..25debde766 100644 --- a/crates/perry-codegen/src/expr/shadow_inline.rs +++ b/crates/perry-codegen/src/expr/shadow_inline.rs @@ -52,9 +52,10 @@ //! at the store site, in the same position the call occupied, and written //! immediately. Nothing re-reads the slot at a later safepoint. //! -//! The incremental-mark root shading barrier is emitted inline too, behind the -//! same `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` gate the runtime and -//! `emit_persistent_shadow_root_barrier` already use. +//! Generated roots need no incremental-mark insertion barrier: every budgeted +//! cycle rescans them in `FinalRootRemark`, and synchronous cycles have no +//! mutator window between root scan and sweep. Runtime callers of +//! `js_shadow_slot_bind` retain that API's conservative barrier. use super::*; @@ -237,7 +238,7 @@ fn emit_inline_slot_write(ctx: &mut FnCtx<'_>, slot_idx: u32, what: InlineSlotWr .or(I64, &bound, &SHADOW_SLOT_ACTIVE_BIT.to_string()); ctx.block().store(I64, &value, &entry); ctx.block().store(I64, &meta, &meta_ptr); - emit_inline_root_shading_barrier(ctx, &value, &done_label); + ctx.block().br(&done_label); } InlineSlotWrite::Clear => { // Codegen's "dead from here" clear: drop the liveness bit but keep @@ -260,34 +261,6 @@ fn emit_inline_slot_write(ctx: &mut FnCtx<'_>, slot_idx: u32, what: InlineSlotWr true } -/// The incremental-mark root shading barrier, gated inline on -/// `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT`. -/// -/// Identical in kind to `emit_persistent_shadow_root_barrier` and to the -/// runtime's own `root_shading_barrier`: a zero count *proves* this thread's -/// `INCREMENTAL_MARK_BARRIER_VALID_PTRS` is null, because -/// `incremental_mark_barrier_enable` increments the count before installing -/// the thread-local and disable clears the thread-local before decrementing -/// the count. Skipping the call on a zero count is therefore observationally -/// identical, not a weaker barrier. The LLVM `monotonic` load matches the -/// runtime's Rust `Relaxed` readers; this gate does not publish other memory. -/// -/// Terminates the current block with a branch to `done_label`. -fn emit_inline_root_shading_barrier(ctx: &mut FnCtx<'_>, value_bits: &str, done_label: &str) { - let active = - ctx.block() - .load_atomic_monotonic(I32, "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT", 4); - let needed = ctx.block().icmp_ne(I32, &active, "0"); - let barrier_idx = ctx.new_block("ss.barrier"); - let barrier_label = ctx.block_label(barrier_idx); - ctx.block().cond_br(&needed, &barrier_label, done_label); - - ctx.current_block = barrier_idx; - ctx.block() - .call_void("js_write_barrier_root_nanbox", &[(I64, value_bits)]); - ctx.block().br(done_label); -} - #[cfg(test)] mod tests { use super::*; @@ -440,21 +413,6 @@ mod tests { blocks } - /// The register mirroring the value into the entry: the first "value - /// word" store in an inline store block (always emitted before the meta - /// word — see `pointer_store_roots_inline_with_the_runtime_entry_layout`'s - /// "value first" comment). - fn stored_value_reg(blk: &str) -> String { - blk.lines() - .find_map(|l| { - l.trim() - .strip_prefix("store i64 %r") - .and_then(|rest| rest.split(',').next()) - .map(|s| s.to_string()) - }) - .unwrap_or_else(|| panic!("no value-word store in block:\n{blk}")) - } - /// The frame push must go through `js_shadow_frame_enter` and derive the /// pop handle from `frame_top`, because the state pointer — not the handle /// — is what the inline stores need. @@ -585,38 +543,29 @@ mod tests { ); } - /// The incremental-mark root shading barrier must survive inlining, gated - /// on the counter the runtime uses. - /// - /// Sabotage check: drop the `emit_inline_root_shading_barrier` call — a - /// pointer written into a root after the collector scanned roots is then - /// never shaded, and an in-flight incremental cycle frees a live object. + /// A generated root bind keeps the root metadata but emits no root-shading + /// gate or call. FinalRootRemark is the collection-side insertion barrier. /// - /// The barrier's argument register is derived from the bind block's own - /// value-word store (`stored_value_reg`) rather than pinned to a literal - /// number: #10812's entry-level stack-guard check adds registers ahead of - /// the function body, so the exact number shifts, but the barrier must - /// still shade the *same* register the bind just stored. + /// Sabotage check: restoring either the active-count load or the call makes + /// this witness fail on the exact code-size regression from #11929. #[test] - fn inline_bind_keeps_the_gated_root_shading_barrier() { + fn inline_bind_uses_final_remark_instead_of_per_store_shading() { // This test asserts on the SHADOW-STACK lowering. Native roots are the // default now, so it has to say which lowering it is testing. let _shadow = crate::codegen::helpers::NativeRootsPin::shadow(); let body = roots_body(&rooted_local_ir()); + let bind = bind_block(&body); assert!( - body.contains( - "load atomic i32, ptr @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT monotonic" - ), - "inline bind must use the runtime's relaxed ordering for the incremental-mark gate; \ - body:\n{body}" + bind.contains("store i64 %") && bind.contains("ptrtoint ptr %"), + "the value mirror and bound-address metadata must remain:\n{bind}" ); - let value_reg = stored_value_reg(&bind_block(&body)); assert!( - body.contains(&format!( - "call void @js_write_barrier_root_nanbox(i64 %r{value_reg})" - )), - "inline bind must shade the value it just stored (%r{value_reg}) when a \ - cycle is in flight; body:\n{body}" + !body.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT"), + "generated root bind retained an active-cycle gate:\n{body}" + ); + assert!( + !body.contains("call void @js_write_barrier_root_nanbox("), + "generated root bind retained per-store shading:\n{body}" ); } diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 5a8bdf35d6..87bc201549 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -261,9 +261,9 @@ pub(crate) fn emit_shadow_slot_clear(ctx: &mut FnCtx<'_>, slot_idx: u32) { /// The alloca is entry-hoisted and initialized to `undefined`, so the early /// bind is valid even when the declaration itself sits in a loop or branch. /// Every later iteration writes the same alloca, which the GC scanner follows -/// through `slot_ptrs`. Pointer-capable updates still emit the root shading -/// barrier required when an incremental collection has already scanned roots; -/// only the repeated TLS slot rebinding and lexical-death clear are removed. +/// through `slot_ptrs`. FinalRootRemark rescans that alloca, so pointer-capable +/// updates need neither repeated TLS rebinding nor per-store root shading; the +/// lexical-death clear is removed as well. pub(crate) fn enable_persistent_shadow_slot_for_array_alias( ctx: &mut FnCtx<'_>, local_id: u32, @@ -321,23 +321,22 @@ pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32 /// /// The caller owns the pairing of `slot_idx` and `slot_ptr`; everything else /// — the stack-map textual marker, the #7088 inline frame write, the FFI -/// fallback, and the incremental root-shading barrier — is identical to a -/// named local's bind, which is the point: a temp rooted through here is -/// indistinguishable to the collector and to the RS4GC/stack-map lowering -/// from a local. +/// fallback, and the FinalRootRemark contract — is identical to a named +/// local's bind, which is the point: a temp rooted through here is +/// indistinguishable to the collector and to the RS4GC/stack-map lowering from +/// a local. pub(crate) fn emit_shadow_slot_bind_ptr(ctx: &mut FnCtx<'_>, slot_idx: u32, slot_ptr: &str) { ctx.shadow_slots_bound.insert(slot_idx); if crate::codegen::helpers::native_stack_roots_enabled() { // Kept temporarily as a textual marker: LlFunction's final stack-map // lowering records `slot_idx -> slot_ptr` and removes this call. - // The incremental root barrier remains real because the native slot - // can be updated after an in-flight cycle scanned this frame. + // The slot needs no insertion barrier: budgeted collections rescan it + // during FinalRootRemark, while synchronous collections expose no + // intervening mutator window. See `emit_gated_root_nanbox_store`. ctx.block().call_void( "js_shadow_slot_bind", &[(I32, &slot_idx.to_string()), (PTR, slot_ptr)], ); - let value_bits = ctx.block().load(I64, slot_ptr); - emit_persistent_shadow_root_barrier(ctx, &value_bits); return; } // #7088: the hot per-store root write. Emitted inline against this @@ -352,47 +351,6 @@ pub(crate) fn emit_shadow_slot_bind_ptr(ctx: &mut FnCtx<'_>, slot_idx: u32, slot ); } -/// Emit the incremental-mark root shading barrier for a value that has just -/// been written into an already-bound (persistent) root slot. -/// -/// This is the only part of `js_shadow_slot_bind` that is genuinely per-store: -/// re-recording `slot_ptrs[idx]` and re-mirroring the value are loop-invariant -/// for an entry-hoisted alloca, but a pointer stored into a root *after* the -/// collector scanned roots still has to be shaded. Guarding on -/// `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` inline keeps the common -/// (no incremental cycle in flight) path down to a load, a compare, and a -/// not-taken branch instead of a TLS-touching call. The load is LLVM -/// `monotonic`, matching the runtime's Rust `Relaxed` readers: the counter is -/// only a gate and does not publish accompanying memory. -pub(crate) fn emit_persistent_shadow_root_barrier(ctx: &mut FnCtx<'_>, value_bits: &str) { - // #8583-followup: if computing the value diverged (a throwing sub-expression - // — e.g. a TDZ access on a captured `let` — emitted `unreachable`), the - // current block is terminated. `LlBlock` drops instructions emitted after a - // terminator, so `value_bits`' defining instruction was silently discarded; - // the barrier block created below would then reference an undefined register - // ("register %rN used but never defined"). The root store is unreachable on - // this path, so emit no barrier. - if ctx.block().is_terminated() { - return; - } - let active = - ctx.block() - .load_atomic_monotonic(I32, "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT", 4); - let barrier_needed = ctx.block().icmp_ne(I32, &active, "0"); - let barrier_idx = ctx.new_block("shadow.root.barrier"); - let done_idx = ctx.new_block("shadow.root.barrier.done"); - let barrier_label = ctx.block_label(barrier_idx); - let done_label = ctx.block_label(done_idx); - ctx.block() - .cond_br(&barrier_needed, &barrier_label, &done_label); - - ctx.current_block = barrier_idx; - ctx.block() - .call_void("js_write_barrier_root_nanbox", &[(I64, value_bits)]); - ctx.block().br(&done_label); - ctx.current_block = done_idx; -} - /// #9081: root the pointer locals of a constructor body spliced inline into /// the CURRENT function — the `super(...)` parent-body inline, the `new`-site /// own/inherited-ctor inline, and `let_stmt`'s scalar-ctor variants. @@ -448,7 +406,7 @@ pub(crate) fn root_inlined_ctor_pointer_locals( pub(crate) fn emit_shadow_slot_update_for_expr( ctx: &mut FnCtx<'_>, local_id: u32, - value_reg: &str, + _value_reg: &str, rhs: &Expr, ) { // A clone-scoped Number local (5L): the clone's entry test checked its @@ -468,10 +426,8 @@ pub(crate) fn emit_shadow_slot_update_for_expr( return; }; if ctx.persistent_shadow_slots.contains(&slot_idx) { - if !expr_is_known_non_pointer_shadow_value(ctx, rhs) { - let value_bits = ctx.block().bitcast_double_to_i64(value_reg); - emit_persistent_shadow_root_barrier(ctx, &value_bits); - } + // The collector reads the bound alloca directly. FinalRootRemark is + // the insertion barrier for generated roots; no per-store code. return; } if expr_is_known_non_pointer_shadow_value(ctx, rhs) { diff --git a/crates/perry-codegen/src/expr/write_barrier.rs b/crates/perry-codegen/src/expr/write_barrier.rs index bc0b1b0be1..906d38da0c 100644 --- a/crates/perry-codegen/src/expr/write_barrier.rs +++ b/crates/perry-codegen/src/expr/write_barrier.rs @@ -151,9 +151,10 @@ const GC_FLAG_TENURED_I8: &str = "32"; // 0x20 /// `INCREMENTAL_MARK_BARRIER_VALID_PTRS` is null, because /// `incremental_mark_barrier_enable` increments the count BEFORE installing /// the thread-local, while disable clears the thread-local BEFORE -/// decrementing the count. This is the same gate, on the same global, that -/// `expr/shadow_inline.rs` and `expr/shadow_slot.rs` already emit for the -/// root shading barrier. It is an LLVM `monotonic` load (Rust `Relaxed`): +/// decrementing the count. Birth color instead reads the per-thread live +/// birth flags (including barrier-disabled build windows). Generated root +/// stores themselves need no shade (#11929). This is an LLVM `monotonic` +/// load (Rust `Relaxed`): /// the counter is authoritative state, not a publication fence for other /// memory. /// @@ -326,42 +327,37 @@ pub(crate) fn emit_write_barrier_slot_value_and_generation_tested( ctx.current_block = done_idx; } -/// Use the same construction proof as precise local roots. A scalar cannot -/// introduce a new heap edge during incremental marking; the registered root -/// slot still receives the store, including overwrites of old heap values. -/// TypeScript annotations alone do not satisfy this proof. +/// Store an expression result into a compiler-managed registered root. +/// FinalRootRemark rescans the slot regardless of the expression's type or +/// representation; see `emit_gated_root_nanbox_store` for that proof. pub(crate) fn emit_root_nanbox_store_for_expr( ctx: &mut FnCtx<'_>, value: &str, root_slot: &str, - expr: &Expr, + _expr: &Expr, ) { - if super::expr_is_known_non_pointer_shadow_value(ctx, expr) { - // GC_STORE_AUDIT(ROOT): proven scalar in a registered mutable root. - ctx.block().store(DOUBLE, value, root_slot); - } else { - emit_gated_root_nanbox_store(ctx, value, root_slot); - } + emit_gated_root_nanbox_store(ctx, value, root_slot); } -/// [`emit_root_nanbox_store_on_block`] with the runtime's own idle test -/// inlined: `js_write_barrier_root_nanbox` returns immediately while -/// `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` is zero, so the call is taken -/// only while some thread is incrementally marking. +/// Store into a compiler-managed registered root. +/// +/// No insertion barrier is needed here. Budgeted collections rescan every +/// mutable root in `FinalRootRemark`; synchronous collections have no mutator +/// window between root scan and sweep. After that remark, a value generated +/// code can hold is already marked, was born black, came through a barriered +/// heap store, or came through the weak-read barrier. Root dominance forbids +/// an unrooted register from being the value's only home across a collection +/// point. Runtime-owned caches do not share that proof and keep their runtime +/// root barriers. pub(crate) fn emit_gated_root_nanbox_store(ctx: &mut FnCtx<'_>, value: &str, root_slot: &str) { - // GC_STORE_AUDIT(ROOT): module-global slot registered as a mutable GC - // root; the gated root barrier below covers incremental marking. + // GC_STORE_AUDIT(ROOT): module-global slot registered as a mutable GC root. ctx.block().store(DOUBLE, value, root_slot); - let value_bits = ctx.block().bitcast_double_to_i64(value); - super::emit_persistent_shadow_root_barrier(ctx, &value_bits); } pub(crate) fn emit_root_nanbox_store_on_block(blk: &mut LlBlock, value: &str, root_slot: &str) { - // GC_STORE_AUDIT(ROOT): module-global slot registered as a mutable GC - // root; the root-barrier call below covers incremental marking. + // GC_STORE_AUDIT(ROOT): module-global slot registered as a mutable GC root. + // See `emit_gated_root_nanbox_store` for the final-remark proof. blk.store(DOUBLE, value, root_slot); - let value_bits = blk.bitcast_double_to_i64(value); - blk.call_void("js_write_barrier_root_nanbox", &[(I64, &value_bits)]); } pub(crate) fn emit_root_heap_word_store_on_block( @@ -369,9 +365,9 @@ pub(crate) fn emit_root_heap_word_store_on_block( value_bits: &str, root_slot: &str, ) { - // GC_STORE_AUDIT(ROOT): registered mutable GC root slot; root barrier below. + // GC_STORE_AUDIT(ROOT): registered mutable GC root slot. See + // `emit_gated_root_nanbox_store` for the final-remark proof. blk.store(I64, value_bits, root_slot); - blk.call_void("js_write_barrier_root_heap_word", &[(I64, value_bits)]); } /// GC layout-note emission (refs #1090) — at heap-slot stores whose diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 5f5273767e..26694bc091 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -1320,6 +1320,7 @@ js_gc_memory_pressure Reenters js_gc_module_collect Reenters js_gc_module_idle_hint AllocOnly js_gc_module_minor AllocOnly +js_gc_note_black_birth Leaf js_gc_note_slot_layout Leaf js_gc_note_slot_layout_aware Leaf js_gc_pause_stats Leaf diff --git a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv index 2c57db24fa..0e8935399c 100644 --- a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv +++ b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv @@ -1320,6 +1320,7 @@ js_gc_memory_pressure Reenters js_gc_module_collect Reenters js_gc_module_idle_hint AllocOnly js_gc_module_minor AllocOnly +js_gc_note_black_birth Leaf js_gc_note_slot_layout Leaf js_gc_note_slot_layout_aware Leaf js_gc_pause_stats Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index 660556e866..879541b4ce 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -1320,6 +1320,7 @@ js_gc_memory_pressure Reenters js_gc_module_collect Reenters js_gc_module_idle_hint AllocOnly js_gc_module_minor AllocOnly +js_gc_note_black_birth Leaf js_gc_note_slot_layout Leaf js_gc_note_slot_layout_aware Leaf js_gc_pause_stats Leaf diff --git a/crates/perry-codegen/src/lower_call/alloc_hot_tests.rs b/crates/perry-codegen/src/lower_call/alloc_hot_tests.rs index f3d6496d21..30f9cfb716 100644 --- a/crates/perry-codegen/src/lower_call/alloc_hot_tests.rs +++ b/crates/perry-codegen/src/lower_call/alloc_hot_tests.rs @@ -483,6 +483,35 @@ fn a_self_recursive_function_inlines_its_bump_allocator() { ); } +#[test] +fn every_inline_birth_uses_live_flags_and_seeds_initialized_slots() { + assert_inline_new_not_forced(); + let mut module = walk_module(true); + // Whole module: both numeric branches, noted pointer elements, different + // sizes, and specialized clones. No subject-function filtering is allowed. + module.functions[0].body.insert( + 0, + Stmt::Expr(Expr::Array(vec![Expr::LocalGet(N_ID), Expr::Number(3.0)])), + ); + module + .init + .push(Stmt::Expr(Expr::Array(vec![Expr::Number(1.0)]))); + module + .init + .push(Stmt::Expr(Expr::Array(vec![Expr::Bool(true); 16]))); + module + .init + .push(Stmt::Expr(Expr::Array(vec![Expr::Array(vec![ + Expr::Number(2.0), + ])]))); + let ir = ir_for(module); + inline_birth_invariant::check(&ir); + inline_birth_invariant::sabotage_controls(&ir); +} + +#[path = "inline_birth_invariant.rs"] +mod inline_birth_invariant; + /// #8591: the public entry resolves the thread's stable arena state once, and /// the internal recursive body forwards it through every self call. #[test] @@ -535,10 +564,9 @@ fn the_inline_allocator_stores_its_header_prefix_as_one_vector_image() { "the inline allocation site must store the `<2 x i64>` header image:\n{ir}" ); let merge_at = ir.find("\nalloc.merge").unwrap(); - let merge_end = ir[merge_at + 1..] - .find("\nshadow.root.barrier") - .map_or(ir.len(), |at| merge_at + 1 + at); - let allocation_merge = &ir[merge_at..merge_end]; + let merge_tail = &ir[merge_at..]; + let merge_end = merge_tail.find("\n\n").unwrap_or(merge_tail.len()); + let allocation_merge = &merge_tail[..merge_end]; assert!( !allocation_merge.contains("shl i64 1,") && !allocation_merge.contains("lshr i64"), "ordinary inline objects must not pay to update the Map-only object-start bitmap:\n{allocation_merge}" diff --git a/crates/perry-codegen/src/lower_call/ctor_prologue_stores.rs b/crates/perry-codegen/src/lower_call/ctor_prologue_stores.rs index 563f10494a..8041ce8ec1 100644 --- a/crates/perry-codegen/src/lower_call/ctor_prologue_stores.rs +++ b/crates/perry-codegen/src/lower_call/ctor_prologue_stores.rs @@ -37,7 +37,7 @@ //! | precheck condition | why it holds | //! |---|---| //! | `GcHeader.obj_type == GC_TYPE_OBJECT` | low byte of the packed `gc_packed` constant | -//! | not forwarded | `gc_flags` is exactly `GC_FLAG_ARENA` | +//! | not forwarded | fresh nursery birth: `GC_FLAG_ARENA` plus live runtime birth flags; never forwarded | //! | receiver is an ordinary object | the emitted precheck reads `class_id` @0 and the ShapeId @4; #8113 deleted the `object_type` word this row used to name | //! | `class_id == ` | same word, `cid` is this site's class | //! | live-slot bound > slot | the bound is the class's own field count (the ShapeId descriptor's `live_inline_slot_count` since #8113), and every slot in the plan indexes a declared field | diff --git a/crates/perry-codegen/src/lower_call/inline_birth_invariant.rs b/crates/perry-codegen/src/lower_call/inline_birth_invariant.rs new file mode 100644 index 0000000000..b7aacc82bc --- /dev/null +++ b/crates/perry-codegen/src/lower_call/inline_birth_invariant.rs @@ -0,0 +1,348 @@ +//! Whole-module inline-birth invariant: follow actual SSA header operands and +//! every CFG path back from each seed, rather than counting names in one function. +use std::collections::{BTreeMap, BTreeSet}; + +fn store_target(line: &str) -> Option<&str> { + line.strip_prefix("store ")? + .split_once(", ptr ") + .map(|(_, target)| target.split(',').next().unwrap().trim()) +} + +/// Sabotage the actual emitted header's SSA dependency, separately for each +/// allocator. Keep the packed type/size word and the seed unchanged: only the +/// live color is dropped. Also move a real seed immediately after its header. +pub(super) fn sabotage_controls(ir: &str) { + for vector in [false, true] { + let mut changed = false; + let mut pieces = ir + .split("\ndefine ") + .map(str::to_string) + .collect::>(); + for function in pieces.iter_mut().skip(1) { + let lines = function.lines().map(str::trim).collect::>(); + let defs: BTreeMap<_, _> = lines + .iter() + .filter_map(|line| line.split_once(" = ")) + .map(|(lhs, rhs)| (lhs.to_string(), rhs.to_string())) + .collect(); + let header = lines.iter().find(|line| { + line.starts_with(if vector { + "store <2 x i64> %" + } else { + "store i64 %" + }) && registers(line).first().is_some_and(|value| { + let mut deps = BTreeSet::new(); + dependencies(value, &defs, &mut deps); + deps.iter() + .filter_map(|r| defs.get(r)) + .any(|rhs| rhs.starts_with("load volatile i8")) + }) + }); + let Some(header) = header else { continue }; + let mut deps = BTreeSet::new(); + dependencies(®isters(header)[0], &defs, &mut deps); + let (name, rhs) = defs + .iter() + .find(|(name, rhs)| { + deps.contains(*name) && rhs.starts_with("shl i64 ") && rhs.ends_with(", 8") + }) + .unwrap(); + *function = function.replacen( + &format!("{name} = {rhs}"), + &format!("{name} = shl i64 0, 8"), + 1, + ); + changed = true; + break; + } + assert!(changed, "both allocator sabotage subjects must be live"); + assert!( + std::panic::catch_unwind(|| check(&pieces.join("\ndefine "))).is_err(), + "missing header birth flags must fail for vector={vector}" + ); + } + let mut pieces = ir + .split("\ndefine ") + .map(str::to_string) + .collect::>(); + let mut changed = false; + for function in pieces.iter_mut().skip(1) { + let mut lines = function.lines().map(str::to_string).collect::>(); + let Some(seed) = lines + .iter() + .position(|l| l.contains("call ") && l.contains("@js_gc_note_black_birth(")) + else { + continue; + }; + let call = lines.remove(seed); + let raw = call + .split("@js_gc_note_black_birth(ptr ") + .nth(1) + .unwrap() + .split(',') + .next() + .unwrap(); + let header = lines + .iter() + .position(|l| store_target(l.trim()) == Some(raw)) + .unwrap(); + lines.insert(header + 1, call); + *function = lines.join("\n"); + changed = true; + break; + } + assert!(changed); + assert!( + std::panic::catch_unwind(|| check(&pieces.join("\ndefine "))).is_err(), + "seed before initialization must fail" + ); +} + +fn registers(text: &str) -> Vec { + text.split('%') + .skip(1) + .map(|s| { + format!( + "%{}", + s.chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '.') + .collect::() + ) + }) + .collect() +} + +fn dependencies(value: &str, defs: &BTreeMap, seen: &mut BTreeSet) { + if !seen.insert(value.to_string()) { + return; + } + if let Some(rhs) = defs.get(value) { + for child in registers(rhs) { + dependencies(&child, defs, seen); + } + } +} + +pub(super) fn check(ir: &str) { + let mut sites = 0; + let mut arrays = 0; + let mut objects = 0; + for function in ir.split("\ndefine ").skip(1) { + let lines: Vec<_> = function + .split_once("\n}") + .unwrap() + .0 + .lines() + .map(str::trim) + .collect(); + let defs: BTreeMap<_, _> = lines + .iter() + .filter_map(|line| line.split_once(" = ")) + .map(|(lhs, rhs)| (lhs.to_string(), rhs.to_string())) + .collect(); + let mut blocks = BTreeMap::::new(); + let mut current = "entry".to_string(); + let mut begin = 0; + for (i, line) in lines.iter().enumerate() { + if let Some(label) = line.strip_suffix(':') { + blocks.insert(current, (begin, i)); + current = label.to_string(); + begin = i + 1; + } + } + blocks.insert(current, (begin, lines.len())); + let mut predecessors = BTreeMap::>::new(); + for (label, &(start, end)) in &blocks { + for line in &lines[start..end] { + if line.starts_with("br ") { + for target in line.split("label %").skip(1) { + let name = target.split([',', ' ']).next().unwrap().to_string(); + predecessors.entry(name).or_default().push(label.clone()); + } + } + } + } + for slow in lines.iter().filter(|l| l.contains(super::INLINE_SLOW_CALL)) { + sites += 1; + let slow_reg = slow.split_once(" = ").unwrap().0; + let (raw, _) = defs + .iter() + .find(|(_, rhs)| { + rhs.starts_with("phi ptr ") && registers(rhs).iter().any(|r| r == slow_reg) + }) + .expect("raw fast/slow merge"); + let (header_at, header) = lines + .iter() + .enumerate() + .find(|(_, l)| store_target(l) == Some(raw.as_str())) + .expect("each raw allocation must write a header"); + let operand = registers(header) + .into_iter() + .next() + .expect("header must carry live flags, not a constant"); + let mut deps = BTreeSet::new(); + dependencies(&operand, &defs, &mut deps); + let (flags_reg, _) = defs + .iter() + .find(|(name, rhs)| { + deps.contains(*name) && rhs.starts_with("load volatile i8, ptr ") + }) + .expect("header must depend on a LIVE birth flag load"); + let address_reg = registers(&defs[flags_reg])[0].clone(); + let field_reg = registers(&defs[&address_reg])[0].clone(); + assert!( + defs[&field_reg].starts_with("getelementptr i8, ptr ") + && defs[&field_reg].ends_with(", i64 24"), + "must read InlineArenaState's runtime birth cell address" + ); + let packed = deps + .iter() + .filter_map(|r| defs.get(r)) + .find(|rhs| rhs.starts_with("or i64 ")) + .expect("birth flags must be ORed in the header store"); + assert!( + deps.iter() + .filter_map(|r| defs.get(r)) + .any(|rhs| rhs.starts_with("shl i64 ") && rhs.ends_with(", 8")), + "birth flags occupy GcHeader byte 1" + ); + let constant: u64 = std::iter::once(packed) + .chain(deps.iter().filter_map(|r| defs.get(r))) + .flat_map(|rhs| rhs.split_whitespace()) + .filter_map(|s| s.trim_matches([',', '>']).parse::().ok()) + .find(|word| (16..=4096).contains(&(word >> 32))) + .expect("packed header size"); + let base = if header.starts_with("store <2 x i64>") { + objects += 1; + 24 + } else { + arrays += 1; + 16 + }; + let total = (constant >> 32) as usize; + let slots = (total - base) / 8; + let seed_needle = format!("@js_gc_note_black_birth(ptr {raw}, ptr "); + let seed_at = lines + .iter() + .position(|l| l.contains(&seed_needle)) + .unwrap_or_else(|| { + panic!( + "every non-leaf inline birth must seed {raw}; calls: {:?}", + lines + .iter() + .filter(|l| l.contains("js_gc_note_black_birth")) + .collect::>() + ) + }); + assert!(seed_at > header_at, "seed cannot precede header"); + let seed_args = registers(lines[seed_at]); + assert_eq!(seed_args.len(), 2, "seed must receive its resolved queue"); + assert_eq!(&seed_args[0], raw); + let queue_load = &defs[&seed_args[1]]; + assert!(queue_load.starts_with("load ptr, ptr ")); + let queue_field = &defs[®isters(queue_load)[0]]; + assert!(queue_field.starts_with("getelementptr i8, ptr ")); + assert!(queue_field.ends_with(", i64 32")); + let queue_state = registers(queue_field)[0].clone(); + assert!( + deps.iter().any(|dep| defs.get(dep).is_some_and(|rhs| { + rhs.starts_with("getelementptr i8, ptr ") + && rhs.ends_with(", i64 24") + && registers(rhs).first() == Some(&queue_state) + })), + "flags and seed queue must come from the same thread's inline state" + ); + let (seed_block, _) = blocks + .iter() + .find(|(_, (start, end))| *start <= seed_at && seed_at < *end) + .unwrap(); + let mut pending = vec![( + seed_block.clone(), + seed_at, + BTreeSet::new(), + BTreeSet::new(), + )]; + while let Some((block, end, mut initialized, mut visited)) = pending.pop() { + assert!( + visited.insert(block.clone()), + "unexpected cycle before birth publication" + ); + let start = blocks[&block].0; + for &line in lines[start..end].iter().rev() { + if line.contains("call ") && line.contains(" asm ") { + assert!( + line.contains("asm \"\", \"=r,0\"") + && line.contains("\"gc-leaf-function\""), + "only the non-collecting RS4GC root-reload launder is allowed: {line}" + ); + } else if line.contains("call ") { + let callee = line.split('@').nth(1).unwrap().split('(').next().unwrap(); + let effects = include_str!("../gc_effects/linux-x86_64.tsv"); + assert!( + effects + .lines() + .any(|effect| effect == format!("{callee}\tLeaf")), + "collecting call before completed birth: {line}" + ); + } + if line.starts_with("store ") { + if let Some(value) = registers(line.split(", ptr ").next().unwrap()).first() + { + let mut stored_deps = BTreeSet::new(); + dependencies(value, &defs, &mut stored_deps); + assert!( + !stored_deps.contains(raw), + "birth published before seed: {line}" + ); + } + if let Some(ptr) = line.split(", ptr ").nth(1) { + if let Some(gep) = defs.get(ptr.split(',').next().unwrap()) { + if gep.contains(&format!(", ptr {raw}, i64 ")) { + let offset: usize = + gep.rsplit("i64 ").next().unwrap().parse().unwrap(); + if offset >= 8 { + initialized.insert(offset); + } + } + } + } + } + if line == *header { + let prefix_slot = if base == 24 { 16 } else { 8 }; + assert!(initialized.contains(&prefix_slot) && (0..slots).all(|slot| initialized.contains(&(base + slot * 8))), "seed before ALL slots/header metadata initialized: {raw}, slots={slots}, initialized={initialized:?}"); + break; + } + } + if start <= header_at && header_at < end { + continue; + } + let preds = predecessors + .get(&block) + .expect("seed must follow allocation initialization"); + for pred in preds { + pending.push(( + pred.clone(), + blocks[pred].1, + initialized.clone(), + visited.clone(), + )); + } + } + } + } + assert!( + arrays >= 3 && objects > 0, + "both independent inline allocators must be live" + ); + assert_eq!( + sites, + ir.lines() + .filter(|l| l.contains("call ") && l.contains("@js_gc_note_black_birth(")) + .count(), + "count EVERY inline slow site in the WHOLE module" + ); + assert!( + !ir.contains("call void @js_write_barrier_root_heap_word("), + "no per-site root-shading workaround" + ); +} diff --git a/crates/perry-codegen/src/lower_call/new_alloc.rs b/crates/perry-codegen/src/lower_call/new_alloc.rs index 9c446ef244..3ff076c882 100644 --- a/crates/perry-codegen/src/lower_call/new_alloc.rs +++ b/crates/perry-codegen/src/lower_call/new_alloc.rs @@ -797,11 +797,18 @@ fn emit_instance_alloc_inner( } crate::expr::HeaderImageSource::EntryValue(value) => value, }; + let birth_flags = crate::expr::inline_birth::flags(ctx, &state_ptr); + let born_packed = + crate::expr::inline_birth::header(ctx, &gc_packed.to_string(), &birth_flags); let blk = ctx.block(); + let born_image = blk.next_reg(); + blk.emit_raw(format!( + "{born_image} = insertelement <2 x i64> {header_image}, i64 {born_packed}, i32 0" + )); // GC_STORE_AUDIT(INIT): inline headers initialize freshly allocated unpublished object storage. blk.emit_raw(format!( "store <2 x i64> {}, ptr {}, align 8", - header_image, raw + born_image, raw )); // #6759 Phase B: null the `meta` record pointer — the LAST header @@ -848,7 +855,12 @@ fn emit_instance_alloc_inner( // function-call path returned). Convert to i64 to match what // the existing nanbox_pointer_inline expects. let user_ptr = blk.gep(I8, &raw, &[(I64, "8")]); - blk.ptrtoint(&user_ptr, I64) + let handle = blk.ptrtoint(&user_ptr, I64); + + // Seed only after EVERY field has a valid default, before the + // constructor can allocate/poll. Same protocol as runtime births. + crate::expr::inline_birth::finish(ctx, &raw, &birth_flags, &state_ptr); + handle } } else { // Fallback: build the packed-keys string at this site and diff --git a/crates/perry-codegen/src/rooting/temp_root.rs b/crates/perry-codegen/src/rooting/temp_root.rs index 0edebb591b..17aeb76cd3 100644 --- a/crates/perry-codegen/src/rooting/temp_root.rs +++ b/crates/perry-codegen/src/rooting/temp_root.rs @@ -149,7 +149,8 @@ fn temp_pool_acquire(ctx: &mut FnCtx<'_>) -> Option { } /// Root-store for an alloca-mode handle: plain store, then the same -/// bind + root-shading emission every named-local store uses. The bind must +/// root binding every named-local store uses. FinalRootRemark rescans this +/// generated root, so its bind needs no per-store shading. The bind must /// be emitted here — after the store, before whatever collects — so the /// rooted location dominates the collection point (#7192's invariant). fn temp_slot_store(ctx: &mut FnCtx<'_>, handle: &str, value_i64: &str) { diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index 38897c66f6..8d982e794e 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -1201,6 +1201,7 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // in sync with the GEPs we emit in `lower_call::compile_new`. module.declare_function("js_inline_arena_state", PTR, &[]); module.declare_function("js_inline_arena_slow_alloc", PTR, &[PTR, I64, I64]); + module.declare_function("js_gc_note_black_birth", VOID, &[PTR, PTR]); module.declare_function("js_object_delete_field", I32, &[I64, I64]); // Primitive-safe `delete` wrappers: take the RAW NaN-boxed receiver (DOUBLE) // so `delete (number).x` / `delete (number)[k]` no-op to `true` instead of diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index d4b4b551d8..2ea3fde720 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -1456,6 +1456,7 @@ js_gc_memory_pressure i32u i32u js_gc_module_collect f64 js_gc_module_idle_hint f64 js_gc_module_minor f64 +js_gc_note_black_birth void ptr,ptr js_gc_note_slot_layout void i64,i32u,i64 js_gc_note_slot_layout_aware void i64,i32u,i64,i64 js_gc_pause_stats void ptr,ptr,ptr,ptr diff --git a/crates/perry-codegen/tests/scalar_replaced_slot_roots.rs b/crates/perry-codegen/tests/scalar_replaced_slot_roots.rs index 9e51f30d21..9e2703c9b4 100644 --- a/crates/perry-codegen/tests/scalar_replaced_slot_roots.rs +++ b/crates/perry-codegen/tests/scalar_replaced_slot_roots.rs @@ -477,10 +477,10 @@ fn later_store_into_a_scalar_replaced_field_is_bound() { // The bind is per SLOT, not per STORE (#7013). // // #7007 emitted `js_shadow_slot_bind` at every store into a scalar-replacement -// alloca. That call is loop-invariant apart from its root barrier: the alloca -// is entry-hoisted, so `slot_ptrs[idx]` never changes, and every reader of a -// bound slot dereferences `slot_ptrs[idx]` rather than the `stack[idx]` mirror -// the bind refreshes. In a loop it cost ~4 ns per iteration for nothing. +// alloca. The alloca is entry-hoisted, so `slot_ptrs[idx]` never changes, and +// every reader of a bound slot dereferences `slot_ptrs[idx]` rather than the +// `stack[idx]` mirror the bind refreshes. FinalRootRemark rescans the alloca; +// in a loop, rebinding it cost ~4 ns per iteration for nothing. // --------------------------------------------------------------------------- /// Two heap stores into the SAME scalar-replaced field must emit exactly one @@ -529,17 +529,13 @@ fn repeated_stores_into_one_scalar_slot_bind_once() { ); } -/// Every store still shades its value, so an in-flight incremental mark cannot -/// miss a pointer written into an already-scanned root. +/// Repeated stores into a bound scalar-replacement alloca need no per-store +/// shading. Budgeted collections rescan the alloca at FinalRootRemark, and a +/// synchronous collection has no mutator window between root scan and sweep. /// -/// This is the part of the bind that is genuinely per-store, and dropping it -/// while hoisting the rest would be a silent incremental-GC miscompile. -/// -/// Teeth: pre-hoist the scalar-slot path emitted no -/// `js_write_barrier_root_nanbox` at all (the shading happened inside -/// `js_shadow_slot_bind`), so the old compiler produces 0 and fails. +/// Teeth: restoring either generated root barrier makes the count nonzero. #[test] -fn every_store_into_a_hoisted_scalar_slot_shades_its_value() { +fn every_store_into_a_hoisted_scalar_slot_uses_final_remark() { let _pin = NativeRootsPin::shadow(); let ir = ir_for( "scalar_field_two_stores_barrier.ts", @@ -563,24 +559,19 @@ fn every_store_into_a_hoisted_scalar_slot_shades_its_value() { ], ); + let main = main_ir(&ir); assert_eq!( - value_slot_barriers(main_ir(&ir)), - 2, - "each of the two heap stores must shade the value it wrote; the \ - hoisted bind only shades what the alloca held at function entry \ - (#7013). Barriers by slot: {:?}\n{}", - perry_codegen::testing::root_slots::barriers_by_slot(main_ir(&ir)), - main_ir(&ir) + value_slot_barriers(main), + 0, + "the two heap stores must rely on the collector's final root rescan, \ + not generated per-store shading. Barriers by slot: {:?}\n{}", + perry_codegen::testing::root_slots::barriers_by_slot(main), + main ); - - // The barrier must be the guarded form, not an unconditional call: the - // whole point of hoisting is that the common path (no incremental cycle in - // flight) stays a load + compare + not-taken branch. assert!( - ir.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT"), - "the per-store shading barrier must be guarded on the incremental-mark \ - active count, otherwise the hoist just trades one unconditional call \ - for another (#7013):\n{ir}" + !main.contains("@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT") + && !main.contains("call void @js_write_barrier_root_nanbox("), + "generated scalar roots must emit neither an active-cycle gate nor a root barrier:\n{main}" ); } diff --git a/crates/perry-codegen/tests/shadow_slot_hygiene.rs b/crates/perry-codegen/tests/shadow_slot_hygiene.rs index c84eb69d3c..53f076bee6 100644 --- a/crates/perry-codegen/tests/shadow_slot_hygiene.rs +++ b/crates/perry-codegen/tests/shadow_slot_hygiene.rs @@ -892,7 +892,7 @@ fn top_level_loop_body_shadow_slots_clear_each_iteration() { } #[test] -fn immutable_index_alias_binds_once_but_keeps_incremental_root_barrier() { +fn immutable_index_alias_binds_once_and_uses_final_remark() { let _pin = NativeRootsPin::shadow(); let ir = String::from_utf8( compile_module(&persistent_index_alias_shadow_module(), entry_opts()).unwrap(), @@ -912,15 +912,12 @@ fn immutable_index_alias_binds_once_but_keeps_incremental_root_barrier() { "persistent index alias must not be cleared on each backedge" ); assert!( - main_ir.contains("call void @js_write_barrier_root_nanbox(i64 %"), - "pointer-capable alias updates must still shade a newly installed root" + !main_ir.contains("call void @js_write_barrier_root_nanbox(i64 %"), + "generated alias updates must rely on the final root rescan" ); assert!( - main_ir.contains( - "load atomic i32, ptr @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT monotonic, align 4" - ) && main_ir.contains("shadow.root.barrier"), - "the relaxed global gate should let an inactive incremental collector skip the \ - TLS-backed root barrier call" + !main_ir.contains("shadow.root.barrier"), + "persistent generated aliases must not retain per-store root-barrier blocks" ); } diff --git a/crates/perry-codegen/tests/typed_feedback.rs b/crates/perry-codegen/tests/typed_feedback.rs index 7c46979529..0add8c098e 100644 --- a/crates/perry-codegen/tests/typed_feedback.rs +++ b/crates/perry-codegen/tests/typed_feedback.rs @@ -1286,8 +1286,9 @@ fn typed_feedback_guards_numeric_array_push_specialization() { #[test] fn typed_feedback_marks_numeric_array_literals() { // Serialize against the array-literal full-outline test and pin it off, so - // this test always observes the inline numeric-array construction - // (js_array_mark_numeric_f64_layout) rather than the outlined builder. + // this test always observes the inline numeric-array header rather than + // the outlined builder. Canonical doubles set their layout at birth: + // normalization would be unreachable and must not reserve a root slot. let _lock = env_lock(); let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("0")); let numeric_ir = ir_for(module( @@ -1305,7 +1306,11 @@ fn typed_feedback_marks_numeric_array_literals() { ])))], )); - assert!(numeric_ir.contains("call i32 @js_array_mark_numeric_f64_layout")); + // GcHeader: size=40, pointer-free/raw-f64 reserved bits=0x4080, + // flags=ARENA, type=ARRAY. Live birth flags are ORed into this word. + let numeric_header = (40u64 << 32) | (0x4080u64 << 16) | 0x0201; + assert!(numeric_ir.contains(&numeric_header.to_string())); + assert!(!numeric_ir.contains("call i32 @js_array_mark_numeric_f64_layout")); let mixed_ir = ir_for(module( "typed_feedback_mixed_array_literal.ts", @@ -1318,6 +1323,8 @@ fn typed_feedback_marks_numeric_array_literals() { )); assert!(!mixed_ir.contains("call i32 @js_array_mark_numeric_f64_layout")); + let mixed_raw_f64_header = (32u64 << 32) | (0x4080u64 << 16) | 0x0201; + assert!(!mixed_ir.contains(&mixed_raw_f64_header.to_string())); } #[test] diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 9d5b1d579b..00f51668f3 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -1257,6 +1257,8 @@ thread_local! { data: std::ptr::null_mut(), offset: 0, size: 0, + birth_flags: std::ptr::null(), + birth_seeds: std::ptr::null_mut(), }) }; } diff --git a/crates/perry-runtime/src/arena/inline.rs b/crates/perry-runtime/src/arena/inline.rs index 518a8fe76c..76f8dd8e40 100644 --- a/crates/perry-runtime/src/arena/inline.rs +++ b/crates/perry-runtime/src/arena/inline.rs @@ -13,8 +13,23 @@ pub struct InlineArenaState { pub data: *mut u8, // offset 0 — current block's data pointer pub offset: usize, // offset 8 — bump pointer (mutated inline) pub size: usize, // offset 16 — current block's size + /// Stable address of THIS thread's live birth flags, never a cached value. + /// Generated code reads the byte again after every raw allocation merge. + pub birth_flags: *const u8, // offset 24 on the 64-bit inline-allocation ABI + /// Existing per-thread mark-seed queue, not its reallocating backing buffer. + /// Resolving TLS here keeps the unpublished birth's seeding call GC-leaf. + pub birth_seeds: *mut std::ffi::c_void, // offset 32 on LP64 } +pub const INLINE_BIRTH_FLAGS_OFFSET_LP64: usize = 24; +pub const INLINE_BIRTH_SEEDS_OFFSET_LP64: usize = 32; +#[cfg(target_pointer_width = "64")] +const _: () = + assert!(std::mem::offset_of!(InlineArenaState, birth_flags) == INLINE_BIRTH_FLAGS_OFFSET_LP64); +#[cfg(target_pointer_width = "64")] +const _: () = + assert!(std::mem::offset_of!(InlineArenaState, birth_seeds) == INLINE_BIRTH_SEEDS_OFFSET_LP64); + /// Get the per-thread inline arena state pointer. Called once per JS /// function entry; the codegen caches the result in a stack slot and /// reuses it for every `new ClassName()` in that function. The address @@ -37,6 +52,10 @@ pub extern "C" fn js_inline_arena_state() -> *mut InlineArenaState { // out, resolved once per thread instead of once per call. unsafe { let state = &mut *super::block::hot_inline_state(); + if state.birth_flags.is_null() { + state.birth_seeds = crate::gc::mark_seed_queue_address(); + state.birth_flags = crate::gc::gc_birth_flags_address(); + } if state.data.is_null() { // Lazy init: copy from underlying ARENA's current block. let arena = &*super::block::hot_arena(); diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index ecb17384cd..ebea9d47eb 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -319,9 +319,9 @@ thread_local! { pub(super) static INCREMENTAL_MARK_BARRIER_VALID_PTRS: Cell<*const ValidPointerSet> = const { Cell::new(std::ptr::null()) }; - /// Extra GcHeader flags stamped on RUNTIME-path allocations at birth: - /// `GC_FLAG_MARKED` while an incremental mark barrier is active, 0 - /// otherwise (allocate-black). A budgeted cycle's sweep may only collect + /// Extra GcHeader flags stamped by ALL allocators at birth: + /// `GC_FLAG_MARKED` throughout cycle marking (including barrier-disabled + /// build windows), 0 after the sweep snapshot. A budgeted sweep only collects /// what its own trace could have seen; an object born mid-cycle and /// installed via a runtime-internal RAW store (a grown array's elements /// buffer, a map entry node, a string builder's data — none of which pass @@ -332,9 +332,10 @@ thread_local! { /// `gc()` mixed in. Born-marked objects survive to the NEXT cycle — /// bounded floating garbage, already priced by the debt pacer. /// - /// Codegen's inline bump allocator (lower_call.rs IR) does NOT read this - /// flag; codegen-born objects are ordinary JS values whose installs all - /// go through codegen store barriers → `incremental_mark_barrier_value`. + /// Runtime and generated inline allocations read this SAME live cell at + /// each birth. Generated non-leaf births seed after their slots are valid; + /// no collector step may see an incompletely initialized runtime birth + /// either. Root stores need no shade, including after FinalRootRemark. /// The runtime choke points below cover every raw-install allocation. pub(crate) static GC_BIRTH_EXTRA_FLAGS: Cell = const { Cell::new(0) }; @@ -493,13 +494,20 @@ pub(crate) fn incremental_mark_barrier_globally_idle() -> bool { PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::Relaxed) == 0 } -/// Allocate-black birth flags for runtime-path allocations — see +/// Allocate-black birth flags for all allocations — see /// `GC_BIRTH_EXTRA_FLAGS`. #[inline(always)] pub fn gc_birth_extra_flags() -> u8 { hot_birth_extra_flags().get() } +/// The thread-local cell's address is stable; its VALUE changes at GC steps. +/// InlineArenaState caches only this address, never the flags across a poll. +#[inline(always)] +pub(crate) fn gc_birth_flags_address() -> *const u8 { + hot_birth_extra_flags().as_ptr() +} + /// A born-black object must also be TRACED: marking treats MARKED as /// "already visited", so without a seed the object's children are reachable /// through it only via the store-time shade — and the insertion barrier is @@ -511,6 +519,8 @@ pub fn gc_birth_extra_flags() -> u8 { /// old fiber tree). Seeding every black birth closes this for all phases; /// the trace drains absorb seeds continuously, so the cost is one worklist /// visit per mid-cycle runtime allocation. +/// GC leaf: queues work, never runs a collector or user code. Generated +/// callers initialize all slots before this call and before publication. #[inline] pub(crate) fn gc_note_black_birth(header: *mut GcHeader) { if hot_birth_extra_flags().get() & GC_FLAG_MARKED == 0 { @@ -523,7 +533,33 @@ pub(crate) fn gc_note_black_birth(header: *mut GcHeader) { if unsafe { gc_type_is_pointer_free((*header).obj_type) } { return; } - push_mark_seed(header); + note_black_birth_to_queue(header, mark_seed_queue_address()); +} + +/// Shared runtime/generated seed operation. The caller resolves this thread's +/// queue BEFORE entering an unpublished birth's no-safepoint window. No TLS +/// resolver, collector, or user callback is reachable from this leaf operation. +/// Runtime callers retain their live-flags gate above; inline callers read the +/// same live cell for the header and gate. The header is the birth's own color. +#[no_mangle] +pub extern "C" fn js_gc_note_black_birth(header: *mut GcHeader, seeds: *mut std::ffi::c_void) { + unsafe { + if (*header).gc_flags & GC_FLAG_MARKED != 0 && !gc_type_is_pointer_free((*header).obj_type) + { + note_black_birth_to_queue(header, seeds); + } + } +} + +/// The identical queue operation for runtime and generated births. Each caller +/// has already checked its live birth color and pointer-free type. Keeping +/// those gates outside this kernel avoids repeating them in runtime allocators; +/// inlining the existing push also avoids an extra exported-call relocation. +#[inline(always)] +fn note_black_birth_to_queue(header: *mut GcHeader, seeds: *mut std::ffi::c_void) { + unsafe { + (*seeds.cast::>()).push(header); + } } /// Is an incremental mark cycle in progress **on this thread**, or is this diff --git a/crates/perry-runtime/src/gc/tests/cycle_state.rs b/crates/perry-runtime/src/gc/tests/cycle_state.rs index 3bce61f3ca..720371e1c4 100644 --- a/crates/perry-runtime/src/gc/tests/cycle_state.rs +++ b/crates/perry-runtime/src/gc/tests/cycle_state.rs @@ -1434,6 +1434,47 @@ fn full_cycle_global_root_store_after_root_scan_preserves_new_value() { ); } +#[test] +fn final_remark_preserves_unshaded_generated_global_root() { + let _guard = CopyingNurseryTestGuard::new(0); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + + let mut root_slot = 0_u64; + js_gc_register_global_root(&mut root_slot as *mut u64 as i64); + let child = gc_malloc( + std::mem::size_of::(), + GC_TYPE_CLOSURE, + ); + unsafe { + init_test_closure(child); + } + + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::ArenaBytes)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + run_cycle_until_phase(&mut state, GcCyclePhase::BlockPersistence); + + // Match generated module-global code after #11929: publish the value into + // its registered root with no per-store shading call. This malloc object + // cannot be retained by arena block persistence, so FinalRootRemark is the + // only operation that can discover it now. + root_slot = ptr_bits(child as usize); + run_cycle_in_single_unit_steps(&mut state); + std::hint::black_box(root_slot); + + assert!( + malloc_user_ptr_tracked(child), + "FinalRootRemark must retain an unshaded compiler-managed global root" + ); +} + +#[path = "cycle_state/inline_birth.rs"] +mod inline_birth; + +#[path = "cycle_state/birth_color_controls.rs"] +mod birth_color_controls; +#[path = "cycle_state/root_remark.rs"] +mod root_remark; + #[test] fn full_cycle_path_module_root_store_after_root_scan_preserves_new_value() { let _guard = CopyingNurseryTestGuard::new(0); diff --git a/crates/perry-runtime/src/gc/tests/cycle_state/birth_color_controls.rs b/crates/perry-runtime/src/gc/tests/cycle_state/birth_color_controls.rs new file mode 100644 index 0000000000..a5102a3b28 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/cycle_state/birth_color_controls.rs @@ -0,0 +1,99 @@ +//! Post-remark header-color sabotage with tracked malloc storage, so loss is +//! observed without dereferencing a swept arena address. Payload/header writes +//! model the two emitted inline births; a seed alone does NOT mark its owner. +use super::*; + +fn post_remark_birth_survives(kind: u8, include_live_flags: bool) -> std::thread::Result { + let _guard = CopyingNurseryTestGuard::new(9); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + crate::weakref::test_support::clear_weak_holders(); + // Full-mark drains now accumulate remembered entries directly; the former + // sliced rebuild no longer opens a mutator window. Weak holders ensure the + // remaining post-remark weak-processing window actually returns to us. + for slot in 0..8 { + let target = crate::object::js_object_alloc(0, 0); + let holder = crate::weakref::js_weakref_new(f64::from_bits(ptr_bits(target as usize))); + js_shadow_slot_set(slot, ptr_bits(holder as usize)); + } + let mut local = 0_u64; + js_shadow_slot_bind(8, &mut local); + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::Manual)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + for step in 0..100_000 { + if state.atomic_finalize_subphase_for_tests() == Some("weak_processing") { + break; + } + state.step(GcWorkBudget::bounded(1)); + assert!(step < 99_999, "post-remark window must be exercised"); + } + assert_eq!(state.phase(), GcCyclePhase::AtomicFinalize); + assert!(incremental_mark_barrier_active()); + let (prefix, slots) = if kind == GC_TYPE_ARRAY { + (8, 2) + } else { + (std::mem::size_of::(), 8) + }; + let user = gc_malloc(prefix + slots * 8, kind); + unsafe { + let live = *(*crate::arena::js_inline_arena_state()).birth_flags; + assert_eq!(live, GC_FLAG_MARKED); + // This is the sabotage: omit the live flags from the newborn header. + // Malloc storage has no ARENA bit; otherwise the color/seed protocol is + // identical to the inline header store. gc_malloc's queued seed remains + // in BOTH arms, proving that a seed cannot repair a white owner. + (*header_from_user_ptr(user)).gc_flags = if include_live_flags { live } else { 0 }; + if kind == GC_TYPE_ARRAY { + (user as *mut crate::array::ArrayHeader).write(crate::array::ArrayHeader { + length: slots as u32, + capacity: slots as u32, + }); + } else { + (user as *mut crate::object::ObjectHeader).write(crate::object::ObjectHeader { + class_id: 0, + parent_class_id: 0, + meta: std::ptr::null_mut(), + }); + } + for slot in 0..slots { + (user.add(prefix) as *mut u64) + .add(slot) + .write(crate::value::TAG_UNDEFINED); + } + js_gc_note_black_birth( + header_from_user_ptr(user), + (*crate::arena::js_inline_arena_state()).birth_seeds, + ); + } + local = ptr_bits(user as usize); + // Setup/color assertions above deliberately remain OUTSIDE the catch: a + // missing window must fail the sabotage rather than count as lost-object + // detection. Only the verifier/collection of the deliberately white owner + // may panic in the negative arm. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_cycle_in_single_unit_steps(&mut state); + std::hint::black_box(local); + malloc_user_ptr_tracked(user) + })) +} + +#[test] +fn post_remark_array_birth_color_preserves_local_only_owner() { + assert!(post_remark_birth_survives(GC_TYPE_ARRAY, true).expect("valid array birth")); +} + +#[test] +fn post_remark_object_birth_color_preserves_local_only_owner() { + assert!(post_remark_birth_survives(GC_TYPE_OBJECT, true).expect("valid object birth")); +} + +#[test] +fn missing_array_birth_flags_sabotage_detects_swept_local_only_owner() { + let result = post_remark_birth_survives(GC_TYPE_ARRAY, false); + assert!(result.is_err() || !result.unwrap()); +} + +#[test] +fn missing_object_birth_flags_sabotage_detects_swept_local_only_owner() { + let result = post_remark_birth_survives(GC_TYPE_OBJECT, false); + assert!(result.is_err() || !result.unwrap()); +} diff --git a/crates/perry-runtime/src/gc/tests/cycle_state/inline_birth.rs b/crates/perry-runtime/src/gc/tests/cycle_state/inline_birth.rs new file mode 100644 index 0000000000..144e2e42b8 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/cycle_state/inline_birth.rs @@ -0,0 +1,164 @@ +//! The same allocation sequence as codegen, with an adversarial early-drain +//! control: a seed consumed before payload initialization loses the real child. +use super::*; + +unsafe fn raw_birth(kind: u8, shape: u32, child: *mut u8, premature_drain: bool) -> *mut u8 { + let slots = if kind == GC_TYPE_ARRAY { 2 } else { 8 }; + let prefix = if kind == GC_TYPE_ARRAY { + 8 + } else { + std::mem::size_of::() + }; + let total = GC_HEADER_SIZE + prefix + slots * 8; + let state = crate::arena::js_inline_arena_state(); + let raw = crate::arena::js_inline_arena_slow_alloc(state, total, 8); + (raw as *mut GcHeader).write(GcHeader { + obj_type: kind, + gc_flags: GC_FLAG_ARENA | *(*state).birth_flags, + _reserved: 0, + size: total as u32, + }); + let user = raw.add(GC_HEADER_SIZE); + if kind == GC_TYPE_ARRAY { + (user as *mut crate::array::ArrayHeader).write(crate::array::ArrayHeader { + length: slots as u32, + capacity: slots as u32, + }); + } else { + let object = user as *mut crate::object::ObjectHeader; + object.write(crate::object::ObjectHeader { + class_id: 0, + parent_class_id: 0, + meta: std::ptr::null_mut(), + }); + crate::object::shapes::store_kind::premark_plain_ordinary(object); + (*object).parent_class_id = shape; + } + let fields = user.add(prefix) as *mut u64; + for i in 0..slots { + fields.add(i).write(if premature_drain { + 0xDEADBEEFBAADF0DE + } else { + crate::value::TAG_UNDEFINED + }); + } + if premature_drain { + // Previously recycled bytes look like valid old slot values. Force a + // collector drain IMMEDIATELY after this deliberately premature seed, + // before the real child stores. MARKED then prevents another visit. + js_gc_note_black_birth(raw as *mut GcHeader, (*state).birth_seeds); + let valid = build_valid_pointer_set(); + drain_incremental_mark_barrier_seeds(&valid); + } + fields.write(ptr_bits(child as usize)); + if !premature_drain { + js_gc_note_black_birth(raw as *mut GcHeader, (*state).birth_seeds); + } + user +} + +fn child_survives(kind: u8, premature_drain: bool) -> bool { + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let shape = if kind == GC_TYPE_OBJECT { + crate::object::shapes::shape_descriptor_ensure(std::ptr::null(), 0, 8).unwrap() + } else { + 0 + }; + let child = alloc_tracked_test_closure(); + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::Manual)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + assert_eq!(state.phase(), GcCyclePhase::BuildValidPointerSet); + assert!( + !incremental_mark_barrier_active(), + "must exercise barrier-disabled births" + ); + let parent = unsafe { raw_birth(kind, shape, child, premature_drain) }; + assert_marked_user_ptr(parent as usize, "inline live-cell birth"); + // Parent has no root or insertion barrier. Birth color retains the parent, + // and ONLY its post-initialization seed can discover the white child. + run_cycle_in_single_unit_steps(&mut state); + std::hint::black_box(parent); + malloc_user_ptr_tracked(child) +} + +#[test] +fn inline_array_black_birth_traces_child_in_barrier_disabled_build_window() { + assert!( + child_survives(GC_TYPE_ARRAY, false), + "initialized array seed lost its child" + ); +} + +#[test] +fn inline_object_black_birth_traces_child_in_barrier_disabled_build_window() { + assert!( + child_survives(GC_TYPE_OBJECT, false), + "initialized object seed lost its child" + ); +} + +#[test] +fn premature_inline_birth_seed_drain_detects_lost_array_child() { + let result = std::panic::catch_unwind(|| child_survives(GC_TYPE_ARRAY, true)); + assert!( + result.is_err() || !result.unwrap(), + "early-drain sabotage must lose the child or fail mark verification" + ); +} + +#[test] +fn premature_inline_birth_seed_drain_detects_lost_object_child() { + let result = std::panic::catch_unwind(|| child_survives(GC_TYPE_OBJECT, true)); + assert!( + result.is_err() || !result.unwrap(), + "early-drain sabotage must lose the child or fail mark verification" + ); +} + +#[test] +fn inline_birth_flags_address_tracks_cycle_and_sweep_snapshot() { + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let address = unsafe { (*crate::arena::js_inline_arena_state()).birth_flags }; + let queue = unsafe { (*crate::arena::js_inline_arena_state()).birth_seeds }; + assert_eq!(queue, mark_seed_queue_address()); + assert_eq!(address, gc_birth_flags_address()); + assert_eq!(unsafe { *address }, 0); + let old = unsafe { alloc_old_test_object(0) }.0; + js_shadow_slot_set(0, ptr_bits(old as usize)); + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::Manual)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + assert_eq!(unsafe { *address }, GC_FLAG_MARKED); + run_cycle_until_phase(&mut state, GcCyclePhase::Sweep); + assert_eq!( + unsafe { *address }, + GC_FLAG_MARKED, + "finalize->snapshot gap remains black" + ); + for _ in 0..100_000 { + if unsafe { *address } == 0 { + break; + } + state.step(GcWorkBudget::bounded(1)); + } + assert_eq!( + unsafe { *address }, + 0, + "budgeted sweep births must stay white after snapshot" + ); + assert_eq!( + unsafe { (*crate::arena::js_inline_arena_state()).birth_flags }, + address + ); + run_cycle_in_single_unit_steps(&mut state); + assert_eq!( + queue, + mark_seed_queue_address(), + "queue address survives drains" + ); + assert_eq!( + unsafe { (*crate::arena::js_inline_arena_state()).birth_seeds }, + queue + ); +} diff --git a/crates/perry-runtime/src/gc/tests/cycle_state/root_remark.rs b/crates/perry-runtime/src/gc/tests/cycle_state/root_remark.rs new file mode 100644 index 0000000000..db08aac73d --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/cycle_state/root_remark.rs @@ -0,0 +1,176 @@ +//! #11929: stable bound homes and post-final-remark mutator windows. +use super::*; + +#[test] +fn final_remark_visits_two_hundred_bound_shadow_frame_homes() { + const HOMES: usize = 200; + let _guard = CopyingNurseryTestGuard::new(HOMES as u32); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + // Same active bound-address entries as #11960's stable-home lowering. + // Bind while idle; later writes deliberately bypass the runtime bind API's + // conservative shading, so only the collector's final scan can retain them. + let mut homes = vec![0_u64; HOMES]; + for (index, home) in homes.iter_mut().enumerate() { + js_shadow_slot_bind(index as u32, home as *mut u64); + } + let children: Vec<_> = (0..HOMES).map(|_| alloc_tracked_test_closure()).collect(); + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::ArenaBytes)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + run_cycle_until_phase(&mut state, GcCyclePhase::BlockPersistence); + assert!(incremental_mark_barrier_active()); + for (home, child) in homes.iter_mut().zip(&children) { + assert_eq!( + unsafe { (*header_from_user_ptr(*child)).gc_flags & GC_FLAG_MARKED }, + 0, + "the initial root scan must not have discovered this child" + ); + *home = ptr_bits(*child as usize); + } + // Fresh runtime births during marking also enter the same bound homes. + // Keep the other half white until the remark so this cannot pass merely + // because allocate-black retained every object independently of scanning. + let births: Vec<_> = (0..HOMES / 2) + .map(|_| alloc_tracked_test_closure()) + .collect(); + for (home, child) in homes.iter_mut().zip(&births) { + assert_marked_user_ptr(*child as usize, "fresh birth during marking"); + *home = ptr_bits(*child as usize); + } + run_cycle_in_single_unit_steps(&mut state); + std::hint::black_box(&homes); + for child in children.into_iter().skip(HOMES / 2).chain(births) { + assert!( + malloc_user_ptr_tracked(child), + "FinalRootRemark missed a bound stable shadow-frame home" + ); + } +} + +fn post_remark_unshaded_root_stores_survive(window: &str) { + const HOLDERS: u32 = 8; + let _guard = CopyingNurseryTestGuard::new(HOLDERS + 2); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + crate::weakref::test_support::clear_weak_holders(); + // Keep an old graph in the mark set and enough weak holders to exercise a + // genuinely sliced post-remark weak-processing window. Full-mark drains + // now remember old-to-young entries directly; no rebuild window remains. + let mut old_roots: Vec = (0..8) + .map(|_| ptr_bits(unsafe { alloc_old_test_object(0) }.0 as usize)) + .collect(); + for root in &mut old_roots { + js_gc_register_global_root(root as *mut u64 as i64); + } + for slot in 0..HOLDERS { + let target = crate::object::js_object_alloc(0, 0); + let holder = crate::weakref::js_weakref_new(f64::from_bits(ptr_bits(target as usize))); + js_shadow_slot_set(slot, ptr_bits(holder as usize)); + } + let marked = alloc_tracked_test_closure(); + let mut source_home = ptr_bits(marked as usize); + let mut destination_home = 0_u64; + let mut global_root = 0_u64; + let mut inline_root = 0_u64; + js_shadow_slot_bind(HOLDERS, &mut source_home as *mut u64); + js_shadow_slot_bind(HOLDERS + 1, &mut destination_home as *mut u64); + js_gc_register_global_root(&mut global_root as *mut u64 as i64); + js_gc_register_global_root(&mut inline_root as *mut u64 as i64); + + let mut state = GcCycleState::new_full(trace_snapshot(GcTriggerKind::ArenaBytes)); + state.set_progress_kind(GcProgressKind::NormalIncremental); + for step in 0..100_000 { + if state.atomic_finalize_subphase_for_tests() == Some(window) { + break; + } + state.step(GcWorkBudget::bounded(1)); + assert!(step < 99_999, "post-remark window {window} never opened"); + } + assert_eq!(state.phase(), GcCyclePhase::AtomicFinalize); + assert_eq!(state.atomic_finalize_subphase_for_tests(), Some(window)); + assert!(incremental_mark_barrier_active()); + assert_marked_user_ptr(marked as usize, "pre-existing reachable source"); + + // Root-to-root transfer cannot create a white object after the complete + // remark/drain. A new malloc-path object is black at birth and seeded. + destination_home = source_home; + source_home = 0; + let birth = alloc_tracked_test_closure(); + let born_marked = unsafe { (*header_from_user_ptr(birth)).gc_flags & GC_FLAG_MARKED != 0 }; + global_root = ptr_bits(birth as usize); + // Model both inline arms after their merge using the SAME live cell and + // post-initialization seed protocol as the emitted allocation sequence. + let inline_birth = unsafe { + let total = GC_HEADER_SIZE + std::mem::size_of::() + 16; + let raw = crate::arena::js_inline_arena_slow_alloc( + crate::arena::js_inline_arena_state(), + total, + 8, + ); + (raw as *mut GcHeader).write(GcHeader { + obj_type: GC_TYPE_OBJECT, + gc_flags: GC_FLAG_ARENA | *(*crate::arena::js_inline_arena_state()).birth_flags, + _reserved: 0, + size: total as u32, + }); + let object = raw.add(GC_HEADER_SIZE) as *mut crate::object::ObjectHeader; + object.write(crate::object::ObjectHeader { + class_id: 0, + parent_class_id: 0, + meta: std::ptr::null_mut(), + }); + let fields = + (object as *mut u8).add(std::mem::size_of::()) as *mut u64; + fields.write(crate::value::TAG_UNDEFINED); + fields.add(1).write(crate::value::TAG_UNDEFINED); + object + }; + assert_eq!( + unsafe { (*header_from_user_ptr(inline_birth as *mut u8)).gc_flags & GC_FLAG_MARKED }, + GC_FLAG_MARKED + ); + js_gc_note_black_birth( + unsafe { header_from_user_ptr(inline_birth as *mut u8) }, + unsafe { (*crate::arena::js_inline_arena_state()).birth_seeds }, + ); + let inline_born_marked = + unsafe { (*header_from_user_ptr(inline_birth as *mut u8)).gc_flags & GC_FLAG_MARKED != 0 }; + inline_root = ptr_bits(inline_birth as usize); + // An owner's overflow record is swept if that owner is lost; the key is + // deliberately NOT a root. This observes actual loss, not just mark bits. + crate::object::test_seed_overflow_fields_root(inline_birth as usize, 42_f64.to_bits()); + // No bind call or root-shading call is allowed at either publication. + run_cycle_in_single_unit_steps(&mut state); + std::hint::black_box(( + source_home, + destination_home, + global_root, + inline_root, + old_roots, + )); + let inline_survived = crate::object::debug_overflow_entry_len(inline_birth as usize).is_some(); + crate::object::test_clear_overflow_fields_root(); + assert!( + inline_survived, + "post-remark inline birth was lost in {window}" + ); + assert!( + inline_born_marked, + "inline birth operation must mark before publication" + ); + assert!( + malloc_user_ptr_tracked(marked), + "marked root transfer was lost in {window}" + ); + assert!( + malloc_user_ptr_tracked(birth), + "post-remark birth was lost in {window}" + ); + assert!( + born_marked, + "post-remark runtime birth must be black in {window}" + ); +} + +#[test] +fn post_remark_root_stores_during_weak_processing_survive() { + post_remark_unshaded_root_stores_survive("weak_processing"); +} diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 90194b0054..7fad7ff4e0 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1217,6 +1217,11 @@ impl ValidPointerSetBuilder { } } +/// Stable address of the existing thread-local queue, including across `take`. +pub(crate) fn mark_seed_queue_address() -> *mut std::ffi::c_void { + MARK_SEEDS.with(|cell| cell.get().cast()) +} + pub(super) fn push_mark_seed(header: *mut GcHeader) { MARK_SEEDS.with(|cell| unsafe { (*cell.get()).push(header); diff --git a/test-files/test_gap_gc_11929_inline_births.ts b/test-files/test_gap_gc_11929_inline_births.ts new file mode 100644 index 0000000000..ae8923b698 --- /dev/null +++ b/test-files/test_gap_gc_11929_inline_births.ts @@ -0,0 +1,36 @@ +// parity-env: PERRY_GC_BUDGETED_OLD_RECLAIM=1 PERRY_GC_MAJOR_PACING_FLOOR_MB=1 PERRY_GC_MAJOR_PACING_GROWTH=1 +// Latest small array/class birth held ONLY in a local across sliced cycles. +class Birth { + value: number; + payload: any; + constructor(value: number) { + this.value = value; + this.payload = [value, value + 1]; + for (let i = 0; i < 3; i++) { + const scratch = new Uint8Array(8192); + scratch[0] = i; + } + } +} +function run(): void { + let latestArray: any = [0, "start"]; + let latestObject: any = new Birth(0); + let pressure: any = new Uint8Array(1); + let sum = 0; + for (let i = 1; i <= 4096; i++) { + latestArray = [i, { value: i + 1 }]; + latestObject = new Birth(i); + for (let j = 0; j < 12; j++) { + pressure = new Uint8Array(32768); + pressure[0] = j; + } + sum += latestArray[0] + latestArray[1].value + latestObject.value + latestObject.payload[1]; + } + // Finish pending work without copying either subject into another root. + for (let i = 0; i < 4096; i++) { + pressure = new Uint8Array(32768); + pressure[0] = i & 255; + } + console.log("inline", sum, latestArray[0], latestArray[1].value, latestObject.value, latestObject.payload[1], pressure[0]); +} +run(); diff --git a/test-files/test_gap_gc_11929_stable_home_marking.ts b/test-files/test_gap_gc_11929_stable_home_marking.ts new file mode 100644 index 0000000000..31d7ed383c --- /dev/null +++ b/test-files/test_gap_gc_11929_stable_home_marking.ts @@ -0,0 +1,626 @@ +// parity-env: PERRY_GC_BUDGETED_OLD_RECLAIM=1 PERRY_GC_MAJOR_PACING_FLOOR_MB=1 PERRY_GC_MAJOR_PACING_GROWTH=1 +// #11929/#11960: the default crossover must select stable shadow homes. +// Microtask turns advance budgeted full cycles; big() fills 200 homes with +// fresh objects during intervening mutator windows. Its final reads and gc() +// also exercise evacuation/rewrite. The runtime proof pins the marking phase. +declare function gc(): void; +function sink(x: any): any { return x; } +const pressure: Uint8Array[] = []; +export function big(seed: any): number { + let v0: any = sink({ k: seed, i: 0 }); + let v1: any = sink({ k: seed, i: 1 }); + let v2: any = sink({ k: seed, i: 2 }); + let v3: any = sink({ k: seed, i: 3 }); + let v4: any = sink({ k: seed, i: 4 }); + let v5: any = sink({ k: seed, i: 5 }); + let v6: any = sink({ k: seed, i: 6 }); + let v7: any = sink({ k: seed, i: 7 }); + let v8: any = sink({ k: seed, i: 8 }); + let v9: any = sink({ k: seed, i: 9 }); + let v10: any = sink({ k: seed, i: 10 }); + let v11: any = sink({ k: seed, i: 11 }); + let v12: any = sink({ k: seed, i: 12 }); + let v13: any = sink({ k: seed, i: 13 }); + let v14: any = sink({ k: seed, i: 14 }); + let v15: any = sink({ k: seed, i: 15 }); + let v16: any = sink({ k: seed, i: 16 }); + let v17: any = sink({ k: seed, i: 17 }); + let v18: any = sink({ k: seed, i: 18 }); + let v19: any = sink({ k: seed, i: 19 }); + let v20: any = sink({ k: seed, i: 20 }); + let v21: any = sink({ k: seed, i: 21 }); + let v22: any = sink({ k: seed, i: 22 }); + let v23: any = sink({ k: seed, i: 23 }); + let v24: any = sink({ k: seed, i: 24 }); + let v25: any = sink({ k: seed, i: 25 }); + let v26: any = sink({ k: seed, i: 26 }); + let v27: any = sink({ k: seed, i: 27 }); + let v28: any = sink({ k: seed, i: 28 }); + let v29: any = sink({ k: seed, i: 29 }); + let v30: any = sink({ k: seed, i: 30 }); + let v31: any = sink({ k: seed, i: 31 }); + let v32: any = sink({ k: seed, i: 32 }); + let v33: any = sink({ k: seed, i: 33 }); + let v34: any = sink({ k: seed, i: 34 }); + let v35: any = sink({ k: seed, i: 35 }); + let v36: any = sink({ k: seed, i: 36 }); + let v37: any = sink({ k: seed, i: 37 }); + let v38: any = sink({ k: seed, i: 38 }); + let v39: any = sink({ k: seed, i: 39 }); + let v40: any = sink({ k: seed, i: 40 }); + let v41: any = sink({ k: seed, i: 41 }); + let v42: any = sink({ k: seed, i: 42 }); + let v43: any = sink({ k: seed, i: 43 }); + let v44: any = sink({ k: seed, i: 44 }); + let v45: any = sink({ k: seed, i: 45 }); + let v46: any = sink({ k: seed, i: 46 }); + let v47: any = sink({ k: seed, i: 47 }); + let v48: any = sink({ k: seed, i: 48 }); + let v49: any = sink({ k: seed, i: 49 }); + let v50: any = sink({ k: seed, i: 50 }); + let v51: any = sink({ k: seed, i: 51 }); + let v52: any = sink({ k: seed, i: 52 }); + let v53: any = sink({ k: seed, i: 53 }); + let v54: any = sink({ k: seed, i: 54 }); + let v55: any = sink({ k: seed, i: 55 }); + let v56: any = sink({ k: seed, i: 56 }); + let v57: any = sink({ k: seed, i: 57 }); + let v58: any = sink({ k: seed, i: 58 }); + let v59: any = sink({ k: seed, i: 59 }); + let v60: any = sink({ k: seed, i: 60 }); + let v61: any = sink({ k: seed, i: 61 }); + let v62: any = sink({ k: seed, i: 62 }); + let v63: any = sink({ k: seed, i: 63 }); + let v64: any = sink({ k: seed, i: 64 }); + let v65: any = sink({ k: seed, i: 65 }); + let v66: any = sink({ k: seed, i: 66 }); + let v67: any = sink({ k: seed, i: 67 }); + let v68: any = sink({ k: seed, i: 68 }); + let v69: any = sink({ k: seed, i: 69 }); + let v70: any = sink({ k: seed, i: 70 }); + let v71: any = sink({ k: seed, i: 71 }); + let v72: any = sink({ k: seed, i: 72 }); + let v73: any = sink({ k: seed, i: 73 }); + let v74: any = sink({ k: seed, i: 74 }); + let v75: any = sink({ k: seed, i: 75 }); + let v76: any = sink({ k: seed, i: 76 }); + let v77: any = sink({ k: seed, i: 77 }); + let v78: any = sink({ k: seed, i: 78 }); + let v79: any = sink({ k: seed, i: 79 }); + let v80: any = sink({ k: seed, i: 80 }); + let v81: any = sink({ k: seed, i: 81 }); + let v82: any = sink({ k: seed, i: 82 }); + let v83: any = sink({ k: seed, i: 83 }); + let v84: any = sink({ k: seed, i: 84 }); + let v85: any = sink({ k: seed, i: 85 }); + let v86: any = sink({ k: seed, i: 86 }); + let v87: any = sink({ k: seed, i: 87 }); + let v88: any = sink({ k: seed, i: 88 }); + let v89: any = sink({ k: seed, i: 89 }); + let v90: any = sink({ k: seed, i: 90 }); + let v91: any = sink({ k: seed, i: 91 }); + let v92: any = sink({ k: seed, i: 92 }); + let v93: any = sink({ k: seed, i: 93 }); + let v94: any = sink({ k: seed, i: 94 }); + let v95: any = sink({ k: seed, i: 95 }); + let v96: any = sink({ k: seed, i: 96 }); + let v97: any = sink({ k: seed, i: 97 }); + let v98: any = sink({ k: seed, i: 98 }); + let v99: any = sink({ k: seed, i: 99 }); + let v100: any = sink({ k: seed, i: 100 }); + let v101: any = sink({ k: seed, i: 101 }); + let v102: any = sink({ k: seed, i: 102 }); + let v103: any = sink({ k: seed, i: 103 }); + let v104: any = sink({ k: seed, i: 104 }); + let v105: any = sink({ k: seed, i: 105 }); + let v106: any = sink({ k: seed, i: 106 }); + let v107: any = sink({ k: seed, i: 107 }); + let v108: any = sink({ k: seed, i: 108 }); + let v109: any = sink({ k: seed, i: 109 }); + let v110: any = sink({ k: seed, i: 110 }); + let v111: any = sink({ k: seed, i: 111 }); + let v112: any = sink({ k: seed, i: 112 }); + let v113: any = sink({ k: seed, i: 113 }); + let v114: any = sink({ k: seed, i: 114 }); + let v115: any = sink({ k: seed, i: 115 }); + let v116: any = sink({ k: seed, i: 116 }); + let v117: any = sink({ k: seed, i: 117 }); + let v118: any = sink({ k: seed, i: 118 }); + let v119: any = sink({ k: seed, i: 119 }); + let v120: any = sink({ k: seed, i: 120 }); + let v121: any = sink({ k: seed, i: 121 }); + let v122: any = sink({ k: seed, i: 122 }); + let v123: any = sink({ k: seed, i: 123 }); + let v124: any = sink({ k: seed, i: 124 }); + let v125: any = sink({ k: seed, i: 125 }); + let v126: any = sink({ k: seed, i: 126 }); + let v127: any = sink({ k: seed, i: 127 }); + let v128: any = sink({ k: seed, i: 128 }); + let v129: any = sink({ k: seed, i: 129 }); + let v130: any = sink({ k: seed, i: 130 }); + let v131: any = sink({ k: seed, i: 131 }); + let v132: any = sink({ k: seed, i: 132 }); + let v133: any = sink({ k: seed, i: 133 }); + let v134: any = sink({ k: seed, i: 134 }); + let v135: any = sink({ k: seed, i: 135 }); + let v136: any = sink({ k: seed, i: 136 }); + let v137: any = sink({ k: seed, i: 137 }); + let v138: any = sink({ k: seed, i: 138 }); + let v139: any = sink({ k: seed, i: 139 }); + let v140: any = sink({ k: seed, i: 140 }); + let v141: any = sink({ k: seed, i: 141 }); + let v142: any = sink({ k: seed, i: 142 }); + let v143: any = sink({ k: seed, i: 143 }); + let v144: any = sink({ k: seed, i: 144 }); + let v145: any = sink({ k: seed, i: 145 }); + let v146: any = sink({ k: seed, i: 146 }); + let v147: any = sink({ k: seed, i: 147 }); + let v148: any = sink({ k: seed, i: 148 }); + let v149: any = sink({ k: seed, i: 149 }); + let v150: any = sink({ k: seed, i: 150 }); + let v151: any = sink({ k: seed, i: 151 }); + let v152: any = sink({ k: seed, i: 152 }); + let v153: any = sink({ k: seed, i: 153 }); + let v154: any = sink({ k: seed, i: 154 }); + let v155: any = sink({ k: seed, i: 155 }); + let v156: any = sink({ k: seed, i: 156 }); + let v157: any = sink({ k: seed, i: 157 }); + let v158: any = sink({ k: seed, i: 158 }); + let v159: any = sink({ k: seed, i: 159 }); + let v160: any = sink({ k: seed, i: 160 }); + let v161: any = sink({ k: seed, i: 161 }); + let v162: any = sink({ k: seed, i: 162 }); + let v163: any = sink({ k: seed, i: 163 }); + let v164: any = sink({ k: seed, i: 164 }); + let v165: any = sink({ k: seed, i: 165 }); + let v166: any = sink({ k: seed, i: 166 }); + let v167: any = sink({ k: seed, i: 167 }); + let v168: any = sink({ k: seed, i: 168 }); + let v169: any = sink({ k: seed, i: 169 }); + let v170: any = sink({ k: seed, i: 170 }); + let v171: any = sink({ k: seed, i: 171 }); + let v172: any = sink({ k: seed, i: 172 }); + let v173: any = sink({ k: seed, i: 173 }); + let v174: any = sink({ k: seed, i: 174 }); + let v175: any = sink({ k: seed, i: 175 }); + let v176: any = sink({ k: seed, i: 176 }); + let v177: any = sink({ k: seed, i: 177 }); + let v178: any = sink({ k: seed, i: 178 }); + let v179: any = sink({ k: seed, i: 179 }); + let v180: any = sink({ k: seed, i: 180 }); + let v181: any = sink({ k: seed, i: 181 }); + let v182: any = sink({ k: seed, i: 182 }); + let v183: any = sink({ k: seed, i: 183 }); + let v184: any = sink({ k: seed, i: 184 }); + let v185: any = sink({ k: seed, i: 185 }); + let v186: any = sink({ k: seed, i: 186 }); + let v187: any = sink({ k: seed, i: 187 }); + let v188: any = sink({ k: seed, i: 188 }); + let v189: any = sink({ k: seed, i: 189 }); + let v190: any = sink({ k: seed, i: 190 }); + let v191: any = sink({ k: seed, i: 191 }); + let v192: any = sink({ k: seed, i: 192 }); + let v193: any = sink({ k: seed, i: 193 }); + let v194: any = sink({ k: seed, i: 194 }); + let v195: any = sink({ k: seed, i: 195 }); + let v196: any = sink({ k: seed, i: 196 }); + let v197: any = sink({ k: seed, i: 197 }); + let v198: any = sink({ k: seed, i: 198 }); + let v199: any = sink({ k: seed, i: 199 }); + sink(v0); + sink(v1); + sink(v2); + sink(v3); + sink(v4); + sink(v5); + sink(v6); + sink(v7); + sink(v8); + sink(v9); + sink(v10); + sink(v11); + sink(v12); + sink(v13); + sink(v14); + sink(v15); + sink(v16); + sink(v17); + sink(v18); + sink(v19); + sink(v20); + sink(v21); + sink(v22); + sink(v23); + sink(v24); + sink(v25); + sink(v26); + sink(v27); + sink(v28); + sink(v29); + sink(v30); + sink(v31); + sink(v32); + sink(v33); + sink(v34); + sink(v35); + sink(v36); + sink(v37); + sink(v38); + sink(v39); + sink(v40); + sink(v41); + sink(v42); + sink(v43); + sink(v44); + sink(v45); + sink(v46); + sink(v47); + sink(v48); + sink(v49); + sink(v50); + sink(v51); + sink(v52); + sink(v53); + sink(v54); + sink(v55); + sink(v56); + sink(v57); + sink(v58); + sink(v59); + sink(v60); + sink(v61); + sink(v62); + sink(v63); + sink(v64); + sink(v65); + sink(v66); + sink(v67); + sink(v68); + sink(v69); + sink(v70); + sink(v71); + sink(v72); + sink(v73); + sink(v74); + sink(v75); + sink(v76); + sink(v77); + sink(v78); + sink(v79); + sink(v80); + sink(v81); + sink(v82); + sink(v83); + sink(v84); + sink(v85); + sink(v86); + sink(v87); + sink(v88); + sink(v89); + sink(v90); + sink(v91); + sink(v92); + sink(v93); + sink(v94); + sink(v95); + sink(v96); + sink(v97); + sink(v98); + sink(v99); + sink(v100); + sink(v101); + sink(v102); + sink(v103); + sink(v104); + sink(v105); + sink(v106); + sink(v107); + sink(v108); + sink(v109); + sink(v110); + sink(v111); + sink(v112); + sink(v113); + sink(v114); + sink(v115); + sink(v116); + sink(v117); + sink(v118); + sink(v119); + sink(v120); + sink(v121); + sink(v122); + sink(v123); + sink(v124); + sink(v125); + sink(v126); + sink(v127); + sink(v128); + sink(v129); + sink(v130); + sink(v131); + sink(v132); + sink(v133); + sink(v134); + sink(v135); + sink(v136); + sink(v137); + sink(v138); + sink(v139); + sink(v140); + sink(v141); + sink(v142); + sink(v143); + sink(v144); + sink(v145); + sink(v146); + sink(v147); + sink(v148); + sink(v149); + sink(v150); + sink(v151); + sink(v152); + sink(v153); + sink(v154); + sink(v155); + sink(v156); + sink(v157); + sink(v158); + sink(v159); + sink(v160); + sink(v161); + sink(v162); + sink(v163); + sink(v164); + sink(v165); + sink(v166); + sink(v167); + sink(v168); + sink(v169); + sink(v170); + sink(v171); + sink(v172); + sink(v173); + sink(v174); + sink(v175); + sink(v176); + sink(v177); + sink(v178); + sink(v179); + sink(v180); + sink(v181); + sink(v182); + sink(v183); + sink(v184); + sink(v185); + sink(v186); + sink(v187); + sink(v188); + sink(v189); + sink(v190); + sink(v191); + sink(v192); + sink(v193); + sink(v194); + sink(v195); + sink(v196); + sink(v197); + sink(v198); + sink(v199); + if (seed === 0 && typeof gc === "function") gc(); + let sum = 0; + sum += v0.i; + sum += v1.i; + sum += v2.i; + sum += v3.i; + sum += v4.i; + sum += v5.i; + sum += v6.i; + sum += v7.i; + sum += v8.i; + sum += v9.i; + sum += v10.i; + sum += v11.i; + sum += v12.i; + sum += v13.i; + sum += v14.i; + sum += v15.i; + sum += v16.i; + sum += v17.i; + sum += v18.i; + sum += v19.i; + sum += v20.i; + sum += v21.i; + sum += v22.i; + sum += v23.i; + sum += v24.i; + sum += v25.i; + sum += v26.i; + sum += v27.i; + sum += v28.i; + sum += v29.i; + sum += v30.i; + sum += v31.i; + sum += v32.i; + sum += v33.i; + sum += v34.i; + sum += v35.i; + sum += v36.i; + sum += v37.i; + sum += v38.i; + sum += v39.i; + sum += v40.i; + sum += v41.i; + sum += v42.i; + sum += v43.i; + sum += v44.i; + sum += v45.i; + sum += v46.i; + sum += v47.i; + sum += v48.i; + sum += v49.i; + sum += v50.i; + sum += v51.i; + sum += v52.i; + sum += v53.i; + sum += v54.i; + sum += v55.i; + sum += v56.i; + sum += v57.i; + sum += v58.i; + sum += v59.i; + sum += v60.i; + sum += v61.i; + sum += v62.i; + sum += v63.i; + sum += v64.i; + sum += v65.i; + sum += v66.i; + sum += v67.i; + sum += v68.i; + sum += v69.i; + sum += v70.i; + sum += v71.i; + sum += v72.i; + sum += v73.i; + sum += v74.i; + sum += v75.i; + sum += v76.i; + sum += v77.i; + sum += v78.i; + sum += v79.i; + sum += v80.i; + sum += v81.i; + sum += v82.i; + sum += v83.i; + sum += v84.i; + sum += v85.i; + sum += v86.i; + sum += v87.i; + sum += v88.i; + sum += v89.i; + sum += v90.i; + sum += v91.i; + sum += v92.i; + sum += v93.i; + sum += v94.i; + sum += v95.i; + sum += v96.i; + sum += v97.i; + sum += v98.i; + sum += v99.i; + sum += v100.i; + sum += v101.i; + sum += v102.i; + sum += v103.i; + sum += v104.i; + sum += v105.i; + sum += v106.i; + sum += v107.i; + sum += v108.i; + sum += v109.i; + sum += v110.i; + sum += v111.i; + sum += v112.i; + sum += v113.i; + sum += v114.i; + sum += v115.i; + sum += v116.i; + sum += v117.i; + sum += v118.i; + sum += v119.i; + sum += v120.i; + sum += v121.i; + sum += v122.i; + sum += v123.i; + sum += v124.i; + sum += v125.i; + sum += v126.i; + sum += v127.i; + sum += v128.i; + sum += v129.i; + sum += v130.i; + sum += v131.i; + sum += v132.i; + sum += v133.i; + sum += v134.i; + sum += v135.i; + sum += v136.i; + sum += v137.i; + sum += v138.i; + sum += v139.i; + sum += v140.i; + sum += v141.i; + sum += v142.i; + sum += v143.i; + sum += v144.i; + sum += v145.i; + sum += v146.i; + sum += v147.i; + sum += v148.i; + sum += v149.i; + sum += v150.i; + sum += v151.i; + sum += v152.i; + sum += v153.i; + sum += v154.i; + sum += v155.i; + sum += v156.i; + sum += v157.i; + sum += v158.i; + sum += v159.i; + sum += v160.i; + sum += v161.i; + sum += v162.i; + sum += v163.i; + sum += v164.i; + sum += v165.i; + sum += v166.i; + sum += v167.i; + sum += v168.i; + sum += v169.i; + sum += v170.i; + sum += v171.i; + sum += v172.i; + sum += v173.i; + sum += v174.i; + sum += v175.i; + sum += v176.i; + sum += v177.i; + sum += v178.i; + sum += v179.i; + sum += v180.i; + sum += v181.i; + sum += v182.i; + sum += v183.i; + sum += v184.i; + sum += v185.i; + sum += v186.i; + sum += v187.i; + sum += v188.i; + sum += v189.i; + sum += v190.i; + sum += v191.i; + sum += v192.i; + sum += v193.i; + sum += v194.i; + sum += v195.i; + sum += v196.i; + sum += v197.i; + sum += v198.i; + sum += v199.i; + return sum; +} +let total = 0; +async function step(n: number): Promise { + for (let i = 0; i < 16; i++) { + const bytes = new Uint8Array(20000); + bytes[0] = n; + pressure.push(bytes); + if (pressure.length > 192) pressure.shift(); + } + await null; + total += big(n); + if (n + 1 < 1024) return step(n + 1); +} +step(0).then(() => console.log("stable:" + total)); diff --git a/test-files/test_gap_gc_generated_root_final_remark.ts b/test-files/test_gap_gc_generated_root_final_remark.ts new file mode 100644 index 0000000000..03fd4f02a1 --- /dev/null +++ b/test-files/test_gap_gc_generated_root_final_remark.ts @@ -0,0 +1,33 @@ +// Generated locals and module globals are mutable roots. Incremental GC's +// final root remark, rather than a per-store shading call, discovers their +// latest values; closure captures remain ordinary heap slots with barriers. + +let globalRoot: any = { i: -1, seed: "start", prior: null }; + +function makeWriter(seed: string) { + let captured: any = { i: -1, seed, prior: null }; + + return function writeMany(): string { + let local: any = { i: -1, seed, prior: null }; + const maybeGc = (globalThis as any).gc; + + for (let i = 0; i < 1024; i++) { + const next = { + i, + seed, + prior: (i & 1) === 0 ? local : globalRoot, + }; + local = next; + captured = next; + globalRoot = next; + + // Node normally has no exposed gc(); Perry does. Forced-evacuation runs + // use this to make the generated roots move and be rewritten repeatedly. + if ((i & 31) === 0 && typeof maybeGc === "function") maybeGc(); + } + + return `${local.i}:${captured.i}:${globalRoot.i}:${local.seed}`; + }; +} + +console.log(makeWriter("remark")()); diff --git a/test-parity/gc_repsel_corpus.txt b/test-parity/gc_repsel_corpus.txt index e513498566..4516e08458 100644 --- a/test-parity/gc_repsel_corpus.txt +++ b/test-parity/gc_repsel_corpus.txt @@ -988,3 +988,10 @@ test_gap_addkey_born_wide_11497 test_gap_gc_11931_function_attrs_incremental test_gap_gc_11931_function_attrs_old_minor test_gap_gc_11931_function_attrs_worker + +# #11929: generated mutable local/global roots use the collector's final +# root remark instead of per-store shading; closure captures keep heap barriers. +test_gap_gc_generated_root_final_remark +# #11929/#11960: root-heavy stable homes during budgeted marking and copying GC. +test_gap_gc_11929_stable_home_marking +test_gap_gc_11929_inline_births