diff --git a/crates/perry-ffi/src/lib.rs b/crates/perry-ffi/src/lib.rs index 83d9d16d4b..a37a6c0ed1 100644 --- a/crates/perry-ffi/src/lib.rs +++ b/crates/perry-ffi/src/lib.rs @@ -55,6 +55,12 @@ pub use async_runtime::{ pub mod turnloop_net; +/// Native payloads for binding crates: an ordinary JS object owning the +/// binding's Rust state, with an owner link for pump-delivered events +/// (#11919). A transport-owning family's payload is +/// `turnloop_net::TransportPayload`. +pub mod native_payload; + /// turnloop P10: run a job on the loop of the agent this thread acts for. /// /// What a binding uses INSTEAD of keeping an async runtime alive when it is a diff --git a/crates/perry-ffi/src/native_payload.rs b/crates/perry-ffi/src/native_payload.rs new file mode 100644 index 0000000000..4936dec2b0 --- /dev/null +++ b/crates/perry-ffi/src/native_payload.rs @@ -0,0 +1,629 @@ +//! Native payloads for binding crates: an ordinary JS object that owns a +//! `Box` of the binding's Rust state (#11919; the pattern and its rules are +//! `docs/native-payload-pattern.md` in the perry repo). +//! +//! The runtime's `native_payload` module is generic over `T`, which a +//! separately linked binding cannot instantiate. A binding declares its +//! family as a `static` [`PayloadFamily`] instead (class id, owner link, +//! constructor name, prototype installer, and the payload's size, alignment +//! and drop thunk, all derived from `T` by [`PayloadFamily::new`]) and calls +//! the functions here, which forward to the same runtime code the in-tree +//! families use: one cell, one lifecycle, one owner link. +//! +//! Rules for `T`, unchanged: plain Rust data only (no JS values, NaN-boxed +//! bits or GC pointers; keep JS values in [`js_state`]); `Drop for T` must not +//! allocate on the GC heap, call JS or touch thread-locals. + +use std::ffi::c_void; + +/// Revision of the payload ABI this file is written against. +const PAYLOAD_ABI_VERSION: u8 = 1; + +/// A family, declared once per binding as a `static`. +/// +/// Layout-checked against the runtime's copy: its first word is this crate's +/// layout digest, and the runtime refuses a descriptor whose digest differs. +#[repr(C)] +pub struct PayloadFamily { + abi: u64, + class_id: u32, + links_owner: u32, + constructor_length: u32, + _reserved: u32, + name: *const u8, + name_len: usize, + install_prototype: Option, + drop_payload: unsafe extern "C" fn(resource: *mut c_void, hint: *mut c_void), + payload_size: usize, + payload_align: usize, +} + +// SAFETY: every field is immutable static data (a name in `.rodata`, function +// pointers, integers); the descriptor is only ever read. +unsafe impl Sync for PayloadFamily {} + +unsafe extern "C" fn drop_box(resource: *mut c_void, _hint: *mut c_void) { + drop(Box::from_raw(resource as *mut T)); +} + +const fn layout_digest() -> u64 { + use std::mem::{offset_of, size_of}; + (size_of::() as u64) << 48 + | (offset_of!(PayloadFamily, class_id) as u64) << 40 + | (offset_of!(PayloadFamily, name) as u64) << 32 + | (offset_of!(PayloadFamily, install_prototype) as u64) << 24 + | (offset_of!(PayloadFamily, drop_payload) as u64) << 16 + | (offset_of!(PayloadFamily, payload_align) as u64) << 8 + | PAYLOAD_ABI_VERSION as u64 +} + +/// The prototype installer's signature: it receives the builder for the +/// duration of the call and installs methods through [`PayloadPrototype`]. +pub type InstallFn = unsafe extern "C" fn(proto: *mut c_void); + +impl PayloadFamily { + /// The descriptor of a family whose payload type is `T`. + /// + /// `class_id` is the family's id from the runtime's `native_class_ids.rs`; + /// `links_owner` gives the cell a traced edge to its owner (required for + /// [`owner_link`]: callbacks and pump-delivered events); + /// `constructor_length` is `x.constructor.length`. + pub const fn new( + class_id: u32, + name: &'static str, + links_owner: bool, + constructor_length: u32, + install_prototype: Option, + ) -> Self { + Self { + abi: layout_digest(), + class_id, + links_owner: links_owner as u32, + constructor_length, + _reserved: 0, + name: name.as_ptr(), + name_len: name.len(), + install_prototype, + drop_payload: drop_box::, + payload_size: std::mem::size_of::(), + payload_align: std::mem::align_of::(), + } + } + + fn holds(&self) -> bool { + self.payload_size == std::mem::size_of::() + && self.payload_align == std::mem::align_of::() + } +} + +/// Why a payload is not available. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadMiss { + /// Not an instance of the family. + Foreign, + /// An instance whose payload was released or finalized. + Closed, +} + +/// Result of an explicit close; the object remains a valid instance. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloseOutcome { + /// Released now. + Closed, + /// Released when the outermost native call returns. + Deferred, + /// It was already closed. + AlreadyClosed, + /// Not an instance of the family. + Foreign, +} + +/// The non-finalized cell states. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Lifecycle { + /// A payload is installed. + Open, + /// A close is deferred until the outermost native call returns. + Closing, + /// Released; [`attach`] may reopen it. + Closed, +} + +/// Why [`attach`] refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AttachMiss { + /// Not an instance of the family, or another payload type. + Foreign, + /// A payload is installed. + Open, + /// A deferred close is pending. + Closing, + /// The cell was finalized; the object can never reopen. + Finalized, +} + +/// A stable, inert token naming an instance's payload cell. `Copy + Send` as +/// plain data; dereferenced only on the creating thread, by the runtime. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct OwnerLink(usize); + +impl OwnerLink { + /// The raw token (the cell address), for a C callback's userdata. + pub fn raw(self) -> usize { + self.0 + } + + /// A link back from [`OwnerLink::raw`]. + /// + /// # Safety + /// `raw` came from [`OwnerLink::raw`] of a link of this process. + pub unsafe fn from_raw(raw: usize) -> Self { + Self(raw) + } +} + +/// One own enumerable property set at allocation. +#[repr(C)] +struct OwnProp { + key: *const u8, + key_len: usize, + value: f64, +} + +#[cfg(any(not(test), feature = "runtime-link"))] +extern "C" { + fn js_perry_payload_abi_layout() -> u64; + fn js_perry_payload_alloc( + family: *const PayloadFamily, + resource: *mut c_void, + external_bytes: usize, + own: *const OwnProp, + own_len: usize, + ) -> f64; + fn js_perry_payload_ptr( + value: f64, + family: *const PayloadFamily, + out_miss: *mut i32, + ) -> *mut c_void; + fn js_perry_payload_close(value: f64, family: *const PayloadFamily) -> i32; + fn js_perry_payload_attach( + value: f64, + family: *const PayloadFamily, + resource: *mut c_void, + external_bytes: usize, + ) -> i32; + fn js_perry_payload_lifecycle(value: f64, family: *const PayloadFamily) -> i32; + fn js_perry_payload_owner_link(value: f64, family: *const PayloadFamily) -> usize; + fn js_perry_payload_link_ref(link: usize); + fn js_perry_payload_link_unref(link: usize); + fn js_perry_payload_link_event_owner(link: usize, out: *mut f64) -> i32; + fn js_perry_payload_js_state(value: f64, family: *const PayloadFamily, create: i32) -> f64; + fn js_perry_payload_set_external_bytes(value: f64, family: *const PayloadFamily, bytes: usize); + fn js_perry_payload_proto_method( + proto: *mut c_void, + name: *const u8, + len: usize, + info: *const crate::JsFunctionInfo, + arity: u32, + ); + fn js_perry_payload_proto_getter( + proto: *mut c_void, + name: *const u8, + len: usize, + info: *const crate::JsFunctionInfo, + ); + fn js_perry_payload_proto_data( + proto: *mut c_void, + name: *const u8, + len: usize, + value: f64, + flags: u32, + ); + fn js_perry_payload_proto_inherit(proto: *mut c_void, parent: f64); +} + +// The runtime's result codes (`native_payload_abi.rs`). +#[cfg(any(not(test), feature = "runtime-link"))] +mod code { + pub const OK: i32 = 0; + pub const FOREIGN: i32 = -1; + pub const OPEN: i32 = 1; + pub const CLOSING: i32 = 2; + pub const IS_CLOSED: i32 = 3; + pub const DEFERRED: i32 = 4; + pub const ATTACH_OPEN: i32 = 5; + pub const ATTACH_CLOSING: i32 = 6; + pub const ATTACH_FINALIZED: i32 = 7; +} + +macro_rules! runtime_call { + ($body:block, $fallback:expr) => {{ + #[cfg(any(not(test), feature = "runtime-link"))] + { + $body + } + #[cfg(all(test, not(feature = "runtime-link")))] + { + $fallback + } + }}; +} + +fn undefined() -> f64 { + f64::from_bits(0x7FFC_0000_0000_0001) +} + +/// Whether the linked runtime reads [`PayloadFamily`] with this crate's +/// layout. Every call below is refused by the runtime when it does not. +pub fn abi_matches() -> bool { + runtime_call!( + { + // SAFETY: a plain getter in the linked runtime. + unsafe { js_perry_payload_abi_layout() == layout_digest() } + }, + false + ) +} + +/// Allocate an instance of `family` owning `payload`. `external_bytes` is the +/// native memory the payload really retains (GC pacing); `own` lists node's +/// own enumerable data properties in order. Returns `undefined` when the +/// runtime refuses the family (the payload is dropped). +/// +/// # Panics +/// If `T` is not the type `family` was declared over. +pub fn alloc( + family: &'static PayloadFamily, + payload: T, + external_bytes: usize, + own: &[(&[u8], f64)], +) -> f64 { + assert!(family.holds::(), "a family has one payload type"); + let resource = Box::into_raw(Box::new(payload)) as *mut c_void; + alloc_with(family, resource, external_bytes, own) +} + +/// Allocate an instance with its permanent cell but no payload (CLOSED); the +/// first [`attach`] installs one. +pub fn alloc_closed(family: &'static PayloadFamily, own: &[(&[u8], f64)]) -> f64 { + alloc_with(family, std::ptr::null_mut(), 0, own) +} + +fn alloc_with( + family: &'static PayloadFamily, + resource: *mut c_void, + external_bytes: usize, + own: &[(&[u8], f64)], +) -> f64 { + let props: Vec = own + .iter() + .map(|&(key, value)| OwnProp { + key: key.as_ptr(), + key_len: key.len(), + value, + }) + .collect(); + runtime_call!( + { + // SAFETY: `family` is static; `resource` is null or a boxed `T` + // the family's thunk frees; `props` lives across the call. + unsafe { + js_perry_payload_alloc( + family, + resource, + external_bytes, + props.as_ptr(), + props.len(), + ) + } + }, + { + let _ = (external_bytes, props); + if !resource.is_null() { + // SAFETY: the box allocated above. + unsafe { (family.drop_payload)(resource, std::ptr::null_mut()) }; + } + undefined() + } + ) +} + +/// The live payload of `value`: the receiver check and the borrow in one. +/// +/// # Safety +/// As the runtime's `payload_mut`: do not hold the reference across a close +/// or attach of the same object, across a call that can re-enter the family +/// on it, or across an allocation or JS call while `value` is unrooted. +pub unsafe fn payload_mut<'a, T: 'static>( + value: f64, + family: &'static PayloadFamily, +) -> Result<&'a mut T, PayloadMiss> { + if !family.holds::() { + return Err(PayloadMiss::Foreign); + } + runtime_call!( + { + let mut miss = 0i32; + let ptr = js_perry_payload_ptr(value, family, &mut miss); + if ptr.is_null() { + Err(if miss == code::FOREIGN { + PayloadMiss::Foreign + } else { + PayloadMiss::Closed + }) + } else { + Ok(&mut *(ptr as *mut T)) + } + }, + { + let _ = value; + Err(PayloadMiss::Foreign) + } + ) +} + +/// Explicit close: release the payload now (or after the outermost native +/// call). The object stays a valid instance and its cell stays alive. +pub fn close(value: f64, family: &'static PayloadFamily) -> CloseOutcome { + runtime_call!( + { + // SAFETY: `family` is static. + match unsafe { js_perry_payload_close(value, family) } { + code::OK => CloseOutcome::Closed, + code::DEFERRED => CloseOutcome::Deferred, + code::IS_CLOSED => CloseOutcome::AlreadyClosed, + _ => CloseOutcome::Foreign, + } + }, + { + let _ = (value, family); + CloseOutcome::Foreign + } + ) +} + +/// Reopen a CLOSED instance in its own cell: identity, properties and links +/// are kept. On refusal the payload is dropped. +pub fn attach( + value: f64, + family: &'static PayloadFamily, + payload: T, + external_bytes: usize, +) -> Result<(), AttachMiss> { + if !family.holds::() { + return Err(AttachMiss::Foreign); + } + let resource = Box::into_raw(Box::new(payload)) as *mut c_void; + runtime_call!( + { + // SAFETY: `family` is static; `resource` is a boxed `T` the + // runtime consumes on every outcome. + match unsafe { js_perry_payload_attach(value, family, resource, external_bytes) } { + code::OK => Ok(()), + code::ATTACH_OPEN => Err(AttachMiss::Open), + code::ATTACH_CLOSING => Err(AttachMiss::Closing), + code::ATTACH_FINALIZED => Err(AttachMiss::Finalized), + _ => Err(AttachMiss::Foreign), + } + }, + { + let _ = (value, external_bytes); + // SAFETY: the box allocated above. + unsafe { (family.drop_payload)(resource, std::ptr::null_mut()) }; + Err(AttachMiss::Foreign) + } + ) +} + +/// The instance's state; finalized and wrong-thread cells answer `Closed`. +pub fn lifecycle(value: f64, family: &'static PayloadFamily) -> Result { + runtime_call!( + { + // SAFETY: `family` is static. + match unsafe { js_perry_payload_lifecycle(value, family) } { + code::OPEN => Ok(Lifecycle::Open), + code::CLOSING => Ok(Lifecycle::Closing), + code::IS_CLOSED => Ok(Lifecycle::Closed), + code::FOREIGN => Err(PayloadMiss::Foreign), + _ => Err(PayloadMiss::Closed), + } + }, + { + let _ = (value, family); + Err(PayloadMiss::Foreign) + } + ) +} + +/// The instance's owner link. Requires a family declared with `links_owner`. +pub fn owner_link(value: f64, family: &'static PayloadFamily) -> Result { + runtime_call!( + { + // SAFETY: `family` is static. + match unsafe { js_perry_payload_owner_link(value, family) } { + 0 => Err(if lifecycle(value, family) == Err(PayloadMiss::Foreign) { + PayloadMiss::Foreign + } else { + PayloadMiss::Closed + }), + link => Ok(OwnerLink(link)), + } + }, + { + let _ = (value, family); + Err(PayloadMiss::Foreign) + } + ) +} + +/// Take one ref for an outstanding item that will be delivered through this +/// link: while any ref is held the cell is pinned and its owner lives, with +/// no JS reference. +/// +/// # Safety +/// On the creating thread, with a live cell; match it with [`link_unref`]. +pub unsafe fn link_ref(link: OwnerLink) { + runtime_call!({ js_perry_payload_link_ref(link.0) }, { + let _ = link; + }) +} + +/// Drop a ref taken by [`link_ref`], after the item was delivered. +/// +/// # Safety +/// As [`link_ref`]. +pub unsafe fn link_unref(link: OwnerLink) { + runtime_call!({ js_perry_payload_link_unref(link.0) }, { + let _ = link; + }) +} + +/// The owner, for a pump-delivered event (OPEN, CLOSING or CLOSED; `None` +/// once finalized). Never throws or allocates. Root it before allocating. +/// +/// # Safety +/// The cell is alive: a ref is held for the item being delivered. +pub unsafe fn link_event_owner(link: OwnerLink) -> Option { + runtime_call!( + { + let mut owner = 0f64; + (js_perry_payload_link_event_owner(link.0, &mut owner) != 0).then_some(owner) + }, + { + let _ = link; + None + } + ) +} + +/// The instance's hidden JS-state object, created on first use when +/// `create`; `undefined` otherwise when absent. Every JS value a family keeps +/// for an instance lives here (or in an own property node shows). +pub fn js_state(value: f64, family: &'static PayloadFamily, create: bool) -> f64 { + runtime_call!( + { + // SAFETY: `family` is static. + unsafe { js_perry_payload_js_state(value, family, i32::from(create)) } + }, + { + let _ = (value, family, create); + undefined() + } + ) +} + +/// Re-state the native bytes a live payload retains (GC pacing). +pub fn set_external_bytes(value: f64, family: &'static PayloadFamily, bytes: usize) { + runtime_call!( + { + // SAFETY: `family` is static. + unsafe { js_perry_payload_set_external_bytes(value, family, bytes) } + }, + { + let _ = (value, family, bytes); + } + ) +} + +/// The prototype under construction, handed to a family's installer. +pub struct PayloadPrototype(*mut c_void); + +impl PayloadPrototype { + /// Wrap the pointer an installer receives. + /// + /// # Safety + /// `raw` is the installer's argument, used only during that call. + pub unsafe fn from_raw(raw: *mut c_void) -> Self { + Self(raw) + } + + /// Install a builtin method (writable, non-enumerable, configurable). + pub fn method(&mut self, name: &str, info: &'static crate::JsFunctionInfo, arity: u32) { + runtime_call!( + { + // SAFETY: the builder is live for the installer call. + unsafe { + js_perry_payload_proto_method(self.0, name.as_ptr(), name.len(), info, arity) + } + }, + { + let _ = (name, info, arity); + } + ) + } + + /// Install a non-enumerable, configurable getter. + pub fn getter(&mut self, name: &str, info: &'static crate::JsFunctionInfo) { + runtime_call!( + { + // SAFETY: the builder is live for the installer call. + unsafe { js_perry_payload_proto_getter(self.0, name.as_ptr(), name.len(), info) } + }, + { + let _ = (name, info); + } + ) + } + + /// Install an ordinary data property. + pub fn data( + &mut self, + name: &str, + value: f64, + writable: bool, + enumerable: bool, + configurable: bool, + ) { + let flags = u32::from(writable) | u32::from(enumerable) << 1 | u32::from(configurable) << 2; + runtime_call!( + { + // SAFETY: the builder is live for the installer call. + unsafe { + js_perry_payload_proto_data(self.0, name.as_ptr(), name.len(), value, flags) + } + }, + { + let _ = (name, value, flags); + } + ) + } + + /// Set the prototype's own `[[Prototype]]`. + pub fn inherit(&mut self, parent: f64) { + runtime_call!( + { + // SAFETY: the builder is live for the installer call. + unsafe { js_perry_payload_proto_inherit(self.0, parent) } + }, + { + let _ = parent; + } + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Probe(#[allow(dead_code)] [u64; 3]); + + #[test] + fn a_descriptor_records_its_payload_type_and_layout() { + let family = PayloadFamily::new::(0xFFFF_2412, "Probe", true, 0, None); + assert_eq!(family.payload_size, 24); + assert_eq!(family.payload_align, 8); + assert_eq!(family.abi, layout_digest()); + assert!(family.holds::()); + assert!(!family.holds::()); + assert_eq!(family.abi as u8, PAYLOAD_ABI_VERSION); + } + + /// The runtime reads the descriptor with the same layout (linked runs). + #[cfg(feature = "runtime-link")] + #[test] + fn the_runtime_reads_descriptors_with_this_layout() { + assert!(abi_matches()); + } +} diff --git a/crates/perry-ffi/src/turnloop_net.rs b/crates/perry-ffi/src/turnloop_net.rs index dffd32465b..d1110a86e4 100644 --- a/crates/perry-ffi/src/turnloop_net.rs +++ b/crates/perry-ffi/src/turnloop_net.rs @@ -46,6 +46,8 @@ pub const NET_CLOSED: i32 = 7; pub const NET_ERROR: i32 = 8; /// Completion kind: a subsystem-owned deadline expired (P5); `id` names it. pub const NET_TIMER: i32 = 9; +/// `NetCompletion::flags`: a link route's completion (`id` is an owner link). +pub const NET_FLAG_LINK: i32 = 1; /// This module's view of the runtime's completion record. /// @@ -60,11 +62,14 @@ pub struct NetCompletion { pub errno: i32, /// Nonzero when the operation that produced this will produce no more. pub terminal: i32, - /// Layout padding; see the runtime's definition. - pub _reserved: i32, - /// The socket or listener this concerns. + /// [`NET_FLAG_LINK`] on a link route's completions; zero otherwise. + pub flags: i32, + /// The socket or listener this concerns. On a link route: its owner link + /// ([`NetCompletion::link`]). pub id: i64, - /// For [`NET_ACCEPT`], the accepted connection's id; else zero. + /// For [`NET_ACCEPT`], the accepted connection's id; else zero. On a link + /// route: an install slot for [`link_install_accepted`], valid during the + /// sink call only. pub conn: i64, /// The write/end completion token the caller supplied; zero if none. pub user: u64, @@ -85,6 +90,14 @@ pub struct NetCompletion { } impl NetCompletion { + /// The payload this completion is for, on a link route. + pub fn link(&self) -> Option { + (self.flags & NET_FLAG_LINK != 0 && self.id != 0).then(|| { + // SAFETY: the runtime put a cell address from `owner_link` here. + unsafe { crate::native_payload::OwnerLink::from_raw(self.id as usize) } + }) + } + /// Borrow the read payload. /// /// # Safety @@ -306,16 +319,24 @@ const OK: i32 = 0; const ENOLOOP: i32 = -2; /// Revision of the ABI this file is written against; must match the runtime's. -const ABI_VERSION: u8 = 2; +const ABI_VERSION: u8 = 3; fn layout_digest() -> u64 { + layout_digest_for(TRANSPORT_CORE_WORDS) +} + +/// The runtime's `net_abi_layout` expression, over this crate's own structs +/// and block size. +const fn layout_digest_for(core_words: usize) -> u64 { use std::mem::{align_of, offset_of, size_of}; - (size_of::() as u64) << 48 - | (offset_of!(NetCompletion, id) as u64) << 40 - | (offset_of!(NetCompletion, data) as u64) << 32 - | (offset_of!(NetCompletion, code) as u64) << 24 - | (offset_of!(NetCompletion, syscall) as u64) << 16 - | (align_of::() as u64) << 8 + (size_of::() as u64 & 0xFFF) << 52 + | (offset_of!(NetCompletion, id) as u64 & 0x3F) << 46 + | (offset_of!(NetCompletion, data) as u64 & 0x7F) << 39 + | (offset_of!(NetCompletion, code) as u64 & 0x7F) << 32 + | (offset_of!(NetCompletion, syscall) as u64 & 0x7F) << 25 + | (align_of::() as u64 & 0xF) << 21 + | (core_words as u64 & 0x1FF) << 12 + | (align_of::() as u64 & 0xF) << 8 | ABI_VERSION as u64 } @@ -918,6 +939,650 @@ pub fn peer_address(id: i64) -> Option { }) } +// ── Link routes (NET-TRANSPORT-DESIGN P0) ─────────────────────────────────── +// +// A converted binding owns each socket in a native payload +// (`crate::native_payload`): its payload type is `TransportPayload`, the +// runtime's transport state first and the binding's own fields after it. Every +// call names the socket by `(&mut payload.core, owner_link)` instead of an id; +// every completion's `id` is that owner link ([`NetCompletion::link`]). While +// the driver owes a terminal completion (a handle's close, a resolve, a +// deadline) the payload's cell is pinned, so a listening server or open socket +// with no JS reference lives until it is closed, as in Node. + +/// Words of the opaque block that holds the runtime's transport state. Part of +/// the ABI digest: a runtime with another block size refuses this binding. +#[cfg(not(windows))] +pub const TRANSPORT_CORE_WORDS: usize = 39; +/// See the non-Windows definition. +#[cfg(windows)] +pub const TRANSPORT_CORE_WORDS: usize = 48; + +/// The runtime's transport state for one socket or listener, held inline in +/// the binding's payload (one allocation per socket). Opaque: only the +/// runtime reads it. Dropping it frees memory only. +#[repr(C, align(8))] +pub struct TransportCore { + words: [u64; TRANSPORT_CORE_WORDS], +} + +impl TransportCore { + /// An idle core whose completions go to the link sink `route`. + pub fn new(route: u8) -> Self { + let mut core = std::mem::MaybeUninit::::uninit(); + runtime_call!( + { + // SAFETY: `core` is writable for the whole block and 8-aligned. + let rc = + unsafe { js_perry_net_core_init(core.as_mut_ptr().cast(), i32::from(route)) }; + assert_eq!(rc, OK, "link route {route} is outside the sink table"); + // SAFETY: initialized by the runtime above. + unsafe { core.assume_init() } + }, + { + let _ = route; + // SAFETY: an all-zero block; the fallback build never reads it. + unsafe { + core.as_mut_ptr().write_bytes(0, 1); + core.assume_init() + } + } + ) + } + + fn raw(&mut self) -> *mut std::ffi::c_void { + (self as *mut Self).cast() + } +} + +impl Drop for TransportCore { + fn drop(&mut self) { + runtime_call!( + { + // SAFETY: initialized by `new`, dropped exactly once. + unsafe { js_perry_net_core_drop(self.raw()) } + }, + {} + ) + } +} + +/// A family payload that owns a transport: the core first (the runtime reads +/// it at offset 0 of the payload a link names), then the binding's fields. +#[repr(C)] +pub struct TransportPayload { + /// The runtime's transport state. + pub core: TransportCore, + /// The binding's own fields. + pub ext: E, +} + +impl TransportPayload { + /// An idle transport for the link sink `route`, plus the binding's fields. + pub fn new(route: u8, ext: E) -> Self { + Self { + core: TransportCore::new(route), + ext, + } + } +} + +use crate::native_payload::OwnerLink; + +#[cfg(any(not(test), feature = "runtime-link"))] +extern "C" { + fn js_perry_net_register_link_sink(subsystem: i32, sink: SinkFn) -> i32; + fn js_perry_net_core_init(core: *mut std::ffi::c_void, route: i32) -> i32; + fn js_perry_net_core_drop(core: *mut std::ffi::c_void); + fn js_perry_net_link_tcp_listen( + core: *mut std::ffi::c_void, + link: usize, + host: *const u8, + host_len: usize, + port: u16, + backlog: u32, + reuse_port: i32, + nodelay: i32, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_pipe_listen( + core: *mut std::ffi::c_void, + link: usize, + path: *const u8, + path_len: usize, + backlog: u32, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_accept_start( + core: *mut std::ffi::c_void, + link: usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_install_accepted( + core: *mut std::ffi::c_void, + link: usize, + completion: *const NetCompletion, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_tcp_connect( + core: *mut std::ffi::c_void, + link: usize, + host: *const u8, + host_len: usize, + port: u16, + nodelay: i32, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_pipe_connect( + core: *mut std::ffi::c_void, + link: usize, + path: *const u8, + path_len: usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_adopt_stream( + core: *mut std::ffi::c_void, + link: usize, + socket: i64, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_read_start( + core: *mut std::ffi::c_void, + link: usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_write( + core: *mut std::ffi::c_void, + link: usize, + bytes: *const u8, + len: usize, + user: u64, + out_queued: *mut usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_shutdown( + core: *mut std::ffi::c_void, + link: usize, + user: u64, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_close( + core: *mut std::ffi::c_void, + link: usize, + out_closed: *mut i32, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_set_ref(core: *mut std::ffi::c_void, link: usize, referenced: i32) -> i32; + fn js_perry_net_link_queued_bytes(core: *mut std::ffi::c_void, link: usize) -> usize; + fn js_perry_net_link_deadline_arm( + core: *mut std::ffi::c_void, + link: usize, + delay_ms: u64, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_deadline_park( + core: *mut std::ffi::c_void, + link: usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_deadline_cancel( + core: *mut std::ffi::c_void, + link: usize, + err: *mut RawNetError, + ) -> i32; + fn js_perry_net_link_local_address( + core: *mut std::ffi::c_void, + link: usize, + out: *mut u8, + cap: usize, + out_len: *mut usize, + out_port: *mut u16, + out_family: *mut i32, + ) -> i32; + fn js_perry_net_link_peer_address( + core: *mut std::ffi::c_void, + link: usize, + out: *mut u8, + cap: usize, + out_len: *mut usize, + out_port: *mut u16, + out_family: *mut i32, + ) -> i32; +} + +/// Install a link-routed binding's completion sink (no id allocator: the sink +/// allocates the payload of an accepted connection itself and calls +/// [`link_install_accepted`]). Refused, leaving the route unusable, when the +/// runtime's completion or core layout differs from this crate's. +pub fn register_link_sink(subsystem: u8, sink: SinkFn) -> bool { + runtime_call!( + { + // SAFETY: plain registration calls in the linked runtime. + let ok = unsafe { + if js_perry_net_abi_layout() != layout_digest() + || !crate::native_payload::abi_matches() + { + return false; + } + js_perry_net_register_link_sink(i32::from(subsystem), sink) != 0 + }; + if ok { + REGISTERED.store(true, Ordering::Release); + } + ok + }, + { + let _ = (subsystem, sink, layout_digest()); + false + } + ) +} + +/// Run one link call with an error out-param. +#[cfg(any(not(test), feature = "runtime-link"))] +fn link_call(f: impl FnOnce(*mut RawNetError) -> i32) -> Result<(), NetError> { + let mut raw = RawNetError::blank(); + let rc = f(&mut raw); + check(rc, raw) +} + +/// Bind and listen on `host:port` (link form of [`tcp_listen`]). The handle +/// pins the payload until its close completes. +pub fn link_tcp_listen( + core: &mut TransportCore, + link: OwnerLink, + host: &str, + port: u16, + backlog: u32, + reuse_port: bool, + nodelay: bool, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_tcp_listen( + core.raw(), + link.raw(), + host.as_ptr(), + host.len(), + port, + backlog, + i32::from(reuse_port), + i32::from(nodelay), + err, + ) + }) + }, + { + let _ = (core, link, host, port, backlog, reuse_port, nodelay); + Err(unavailable()) + } + ) +} + +/// Listen on a Unix-domain socket path or a Windows named pipe. Unlink the +/// path yourself, synchronously, at close. +pub fn link_pipe_listen( + core: &mut TransportCore, + link: OwnerLink, + path: &str, + backlog: u32, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_pipe_listen( + core.raw(), + link.raw(), + path.as_ptr(), + path.len(), + backlog, + err, + ) + }) + }, + { + let _ = (core, link, path, backlog); + Err(unavailable()) + } + ) +} + +/// Start the listener's multishot accept. +pub fn link_accept_start(core: &mut TransportCore, link: OwnerLink) -> Result<(), NetError> { + runtime_call!( + { link_call(|err| unsafe { js_perry_net_link_accept_start(core.raw(), link.raw(), err) }) }, + { + let _ = (core, link); + Err(unavailable()) + } + ) +} + +/// Inside the sink, for a [`NET_ACCEPT`] completion: install the accepted +/// connection into a fresh payload's core. A connection not installed before +/// the sink returns is closed by the runtime. +pub fn link_install_accepted( + core: &mut TransportCore, + link: OwnerLink, + completion: &NetCompletion, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_install_accepted(core.raw(), link.raw(), completion, err) + }) + }, + { + let _ = (core, link, completion); + Err(unavailable()) + } + ) +} + +/// Connect a TCP client (link form of [`tcp_connect`]). +pub fn link_tcp_connect( + core: &mut TransportCore, + link: OwnerLink, + host: &str, + port: u16, + nodelay: bool, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_tcp_connect( + core.raw(), + link.raw(), + host.as_ptr(), + host.len(), + port, + i32::from(nodelay), + err, + ) + }) + }, + { + let _ = (core, link, host, port, nodelay); + Err(unavailable()) + } + ) +} + +/// Connect to a Unix-domain socket or a Windows named pipe. +pub fn link_pipe_connect( + core: &mut TransportCore, + link: OwnerLink, + path: &str, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_pipe_connect( + core.raw(), + link.raw(), + path.as_ptr(), + path.len(), + err, + ) + }) + }, + { + let _ = (core, link, path); + Err(unavailable()) + } + ) +} + +/// Adopt an already-connected stream socket; consumed on every outcome. +#[cfg(any(unix, windows))] +pub fn link_adopt_stream( + core: &mut TransportCore, + link: OwnerLink, + socket: AdoptedSocket, +) -> Result<(), NetError> { + runtime_call!( + { + #[cfg(unix)] + let raw_socket = { + use std::os::fd::IntoRawFd; + i64::from(socket.into_raw_fd()) + }; + #[cfg(windows)] + let raw_socket = { + use std::os::windows::io::IntoRawSocket; + socket.into_raw_socket() as i64 + }; + link_call(|err| unsafe { + js_perry_net_link_adopt_stream(core.raw(), link.raw(), raw_socket, err) + }) + }, + { + // Consumed on every outcome: dropping it closes it. + drop(socket); + let _ = (core, link); + Err(unavailable()) + } + ) +} + +/// Start the multishot read. +pub fn link_read_start(core: &mut TransportCore, link: OwnerLink) -> Result<(), NetError> { + runtime_call!( + { link_call(|err| unsafe { js_perry_net_link_read_start(core.raw(), link.raw(), err) }) }, + { + let _ = (core, link); + Err(unavailable()) + } + ) +} + +/// Queue bytes (copied); returns the socket's total queued bytes. +pub fn link_write( + core: &mut TransportCore, + link: OwnerLink, + bytes: &[u8], + user: u64, +) -> Result { + runtime_call!( + { + let mut queued = 0usize; + link_call(|err| unsafe { + js_perry_net_link_write( + core.raw(), + link.raw(), + bytes.as_ptr(), + bytes.len(), + user, + &mut queued, + err, + ) + }) + .map(|()| queued) + }, + { + let _ = (core, link, bytes, user); + Err(unavailable()) + } + ) +} + +/// `end()`: shut the write side down behind the queued writes. +pub fn link_shutdown(core: &mut TransportCore, link: OwnerLink, user: u64) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_shutdown(core.raw(), link.raw(), user, err) + }) + }, + { + let _ = (core, link, user); + Err(unavailable()) + } + ) +} + +/// Release the transport: the handle moves into the driver's close, the +/// deadline and a pending resolve go with it. `Ok(true)`: a [`NET_CLOSED`] +/// for this link follows; `Ok(false)`: there was no handle, so a binding that +/// owes a `close` event schedules it itself. Then release the payload +/// (`native_payload::close`); never close from `Drop`. +pub fn link_close(core: &mut TransportCore, link: OwnerLink) -> Result { + runtime_call!( + { + let mut closed = 0i32; + link_call(|err| unsafe { + js_perry_net_link_close(core.raw(), link.raw(), &mut closed, err) + }) + .map(|()| closed != 0) + }, + { + let _ = (core, link); + Err(unavailable()) + } + ) +} + +/// Node's `ref()`/`unref()`; remembered for handles installed later. +pub fn link_set_ref(core: &mut TransportCore, link: OwnerLink, referenced: bool) -> bool { + runtime_call!( + { + // SAFETY: `core`/`link` name one payload of this thread. + unsafe { + js_perry_net_link_set_ref(core.raw(), link.raw(), i32::from(referenced)) == OK + } + }, + { + let _ = (core, link, referenced); + false + } + ) +} + +/// Bytes accepted and not yet reported written (`writableLength`). +pub fn link_queued_bytes(core: &mut TransportCore, link: OwnerLink) -> usize { + runtime_call!( + { + // SAFETY: `core`/`link` name one payload of this thread. + unsafe { js_perry_net_link_queued_bytes(core.raw(), link.raw()) } + }, + { + let _ = (core, link); + 0 + } + ) +} + +/// Arm or move the socket's deadline; its expiry arrives as [`NET_TIMER`]. +pub fn link_deadline_arm( + core: &mut TransportCore, + link: OwnerLink, + delay_ms: u64, +) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_deadline_arm(core.raw(), link.raw(), delay_ms, err) + }) + }, + { + let _ = (core, link, delay_ms); + Err(unavailable()) + } + ) +} + +/// Disarm the deadline, keeping its handle for a cheap re-arm. +pub fn link_deadline_park(core: &mut TransportCore, link: OwnerLink) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { js_perry_net_link_deadline_park(core.raw(), link.raw(), err) }) + }, + { + let _ = (core, link); + Err(unavailable()) + } + ) +} + +/// Cancel the deadline. Idempotent. +pub fn link_deadline_cancel(core: &mut TransportCore, link: OwnerLink) -> Result<(), NetError> { + runtime_call!( + { + link_call(|err| unsafe { + js_perry_net_link_deadline_cancel(core.raw(), link.raw(), err) + }) + }, + { + let _ = (core, link); + Err(unavailable()) + } + ) +} + +#[cfg(any(not(test), feature = "runtime-link"))] +fn link_endpoint( + core: &mut TransportCore, + link: OwnerLink, + f: unsafe extern "C" fn( + *mut std::ffi::c_void, + usize, + *mut u8, + usize, + *mut usize, + *mut u16, + *mut i32, + ) -> i32, +) -> Option { + let mut buf = [0u8; 64]; + let mut len: usize = 0; + let mut port: u16 = 0; + let mut family: i32 = 4; + // SAFETY: `buf` is writable for its own length and every out-param is a + // live local. + let rc = unsafe { + f( + core.raw(), + link.raw(), + buf.as_mut_ptr(), + buf.len(), + &mut len, + &mut port, + &mut family, + ) + }; + if rc != OK { + return None; + } + Some(Endpoint { + address: String::from_utf8_lossy(&buf[..len]).into_owned(), + port, + family, + }) +} + +/// The local endpoint (`server.address()`, `socket.localAddress`). +pub fn link_local_address(core: &mut TransportCore, link: OwnerLink) -> Option { + runtime_call!( + { link_endpoint(core, link, js_perry_net_link_local_address) }, + { + let _ = (core, link); + None + } + ) +} + +/// The peer endpoint (`socket.remoteAddress`). +pub fn link_peer_address(core: &mut TransportCore, link: OwnerLink) -> Option { + runtime_call!( + { link_endpoint(core, link, js_perry_net_link_peer_address) }, + { + let _ = (core, link); + None + } + ) +} + #[cfg(test)] mod tests { use super::*; @@ -931,13 +1596,26 @@ mod tests { let digest = layout_digest(); assert_eq!(digest as u8, ABI_VERSION); assert_eq!( - (digest >> 48) as usize, + (digest >> 52) as usize, std::mem::size_of::() ); assert_eq!( - ((digest >> 32) & 0xff) as usize, + ((digest >> 39) & 0x7f) as usize, std::mem::offset_of!(NetCompletion, data) ); + // The opaque core block is part of the contract: a binding built + // against another block size computes another digest. + assert_eq!(((digest >> 12) & 0x1ff) as usize, TRANSPORT_CORE_WORDS); + assert_ne!(digest, layout_digest_for(TRANSPORT_CORE_WORDS + 1)); + } + + /// The runtime and this crate agree on the completion and the core block + /// (linked runs). Sabotage: change `TRANSPORT_CORE_WORDS` here by one. + #[cfg(feature = "runtime-link")] + #[test] + fn the_runtime_agrees_on_the_completion_and_core_layout() { + // SAFETY: a plain getter in the linked runtime. + assert_eq!(unsafe { js_perry_net_abi_layout() }, layout_digest()); } #[test] diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 748ad562a3..fc2077c828 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -67,6 +67,8 @@ mod mark_slot_hoists; mod minor_fixed_cost; mod native_payload; mod native_payload_callbacks; +#[cfg(not(target_arch = "wasm32"))] +mod net_transport; mod noncollecting_root_lock; mod object_create; mod old_free_intrusive; diff --git a/crates/perry-runtime/src/gc/tests/net_transport.rs b/crates/perry-runtime/src/gc/tests/net_transport.rs new file mode 100644 index 0000000000..59a47acf68 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/net_transport.rs @@ -0,0 +1,609 @@ +//! NET-TRANSPORT-DESIGN P0 witnesses: a turnloop transport owned by a native +//! payload, on the real driver, under real collections. Each test asserts its +//! subject ran (a listener bound, a completion dispatched, a move happened), +//! and `every_net_transport_sabotage_makes_its_witness_red` runs each one +//! against the defect it exists to catch. +use super::super::*; +use super::support::*; +use crate::native_payload::{self as np, CloseOutcome, NativePayloadFamily, OwnerLink}; +use crate::turnloop_net::transport::{self, TransportCore, TransportPayload}; +use crate::turnloop_net::{NetCompletion, NET_ACCEPT, NET_CLOSED, NET_FLAG_LINK}; +use std::cell::{Cell, RefCell}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +static DROPS: AtomicUsize = AtomicUsize::new(0); + +/// The family's own fields; its drop is counted. +struct Ext; +impl Drop for Ext { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } +} +type Payload = TransportPayload; + +fn install(_: &mut np::PayloadPrototype) {} +static FAMILY: NativePayloadFamily = NativePayloadFamily { + class_id: crate::native_class_ids::CRYPTO_HASH, + name: "TransportProbe", + constructor_export: None, + constructor_length: 0, + links_owner: true, + install_prototype: install, +}; + +/// An unclaimed sink slot (the id-route tests hold 3). +const ROUTE: u8 = 14; + +#[derive(Clone, Debug)] +struct Event { + kind: i32, + link: usize, + /// `link_event_owner` as the sink saw it. + owner: Option, +} + +thread_local! { + static EVENTS: RefCell> = const { RefCell::new(Vec::new()) }; + /// When set, the sink installs an accepted connection into a fresh + /// payload instead of refusing it. + static INSTALL_ACCEPTED: Cell = const { Cell::new(false) }; + static CHILD: Cell = const { Cell::new(0) }; +} + +extern "C" fn link_sink(completion: *const NetCompletion) { + // SAFETY: dispatch passes a live completion for the duration of the call. + let c = unsafe { &*completion }; + assert_ne!( + c.flags & NET_FLAG_LINK, + 0, + "a link route marks its completions" + ); + let link = OwnerLink(c.id as usize); + // SAFETY: the dispatched completion's ref keeps the cell alive. + let owner = unsafe { np::link_event_owner(link) }.map(f64::to_bits); + if c.kind == NET_ACCEPT && INSTALL_ACCEPTED.with(Cell::get) { + let scope = RuntimeHandleScope::new(); + let child = scope.root_nanbox_f64(fresh()); + let child_link = np::owner_link(child.get_nanbox_f64(), &FAMILY).unwrap(); + // SAFETY: the child is OPEN on this thread; the completion is ours. + unsafe { + transport::install_accepted(core(child.get_nanbox_f64()), child_link, completion) + .expect("install the accepted connection"); + } + CHILD.with(|slot| slot.set(child_link.0)); + } + EVENTS.with(|events| { + events.borrow_mut().push(Event { + kind: c.kind, + link: link.0, + owner, + }) + }); +} + +fn fresh() -> f64 { + np::alloc( + &FAMILY, + Payload { + core: TransportCore::new(ROUTE), + ext: Ext, + }, + 0, + &[], + ) +} + +/// The payload's core: offset 0 of the payload (`repr(C)`, core first). +fn core(value: f64) -> *mut TransportCore { + let payload = unsafe { np::payload_mut::(value, &FAMILY) }.expect("an OPEN payload"); + &mut payload.core as *mut TransportCore +} + +fn cell(link: OwnerLink) -> *mut crate::native_handle::NativeHandleHeader { + link.0 as *mut crate::native_handle::NativeHandleHeader +} + +fn refs(link: OwnerLink) -> u32 { + unsafe { (*cell(link)).refs } +} + +fn finalized() -> usize { + crate::native_handle::PAYLOAD_FINALIZED.load(Ordering::SeqCst) +} + +fn full() { + let before = crate::gc::block_persist_force_mark_count(); + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Manual)); + assert_eq!(crate::gc::block_persist_force_mark_count(), before); +} + +fn count(kind: i32, link: OwnerLink) -> usize { + EVENTS.with(|events| { + events + .borrow() + .iter() + .filter(|e| e.kind == kind && e.link == link.0) + .count() + }) +} + +fn last(kind: i32, link: OwnerLink) -> Option { + EVENTS.with(|events| { + events + .borrow() + .iter() + .rev() + .find(|e| e.kind == kind && e.link == link.0) + .cloned() + }) +} + +fn pump_until(want: impl Fn() -> bool) -> bool { + let limit = Instant::now() + Duration::from_secs(5); + loop { + if want() { + return true; + } + if Instant::now() >= limit { + return false; + } + crate::event_pump::pump_net_for_test(Duration::from_millis(5)); + } +} + +/// Root scanners, counters and the loop, as the payload tests set them up. +struct Fixture; +impl Fixture { + fn start() -> Self { + np::reset_payload_prototypes_for_tests(); + gc_register_mutable_root_scanner(np::scan_payload_prototype_roots_mut); + register_runtime_handle_root_scanner_for_tests(); + gc_register_named_mutable_root_scanner( + "shape_table", + crate::object::shapes::scan_shape_table_rekey_mut, + ); + gc_register_named_mutable_root_scanner( + "pinned", + crate::gc::pin::scan_pinned_object_roots_mut, + ); + DROPS.store(0, Ordering::SeqCst); + EVENTS.with(|events| events.borrow_mut().clear()); + INSTALL_ACCEPTED.with(|f| f.set(false)); + CHILD.with(|c| c.set(0)); + assert!( + crate::event_pump::install_net_loop_for_test(), + "the host must provide a turnloop loop" + ); + assert!( + crate::turnloop_net::register_link_sink(ROUTE, link_sink), + "link sink registration must succeed, or every assertion is vacuous" + ); + Fixture + } +} +impl Drop for Fixture { + fn drop(&mut self) { + crate::event_pump::reset_net_loop_for_test(); + EVENTS.with(|events| events.borrow_mut().clear()); + np::reset_payload_prototypes_for_tests(); + } +} + +/// Close the transport and release the payload (rule 3); the `Closed` stays +/// owed. Returns whether a `NET_CLOSED` will follow. +fn destroy(link: OwnerLink) -> bool { + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(unsafe { np::link_event_owner(link) }.unwrap()); + let closed = unsafe { transport::close(core(value.get_nanbox_f64()), link) }.unwrap(); + assert_eq!( + np::close(value.get_nanbox_f64(), &FAMILY), + CloseOutcome::Closed + ); + closed +} + +/// N1 (runtime half) + rule 2: a payload holding a driver handle, with no JS +/// reference, survives full and moving collections until the handle's +/// terminal `Closed` is dispatched; the accept on the way reaches the same +/// owner and installs into a child that is itself kept by its handle. +#[test] +fn n1_a_handle_holding_payload_lives_until_its_closed_is_dispatched() { + let _guard = CopyingNurseryTestGuard::new(0); + let _fixture = Fixture::start(); + let _no_stack = ConservativeScanDisabledGuard::new(); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(fresh()); + let link = np::owner_link(value.get_nanbox_f64(), &FAMILY).unwrap(); + let addr = unsafe { + let c = core(value.get_nanbox_f64()); + let addr = transport::tcp_listen(c, link, "127.0.0.1:0".parse().unwrap(), 16, false, true) + .expect("listen"); + transport::accept_start(c, link).expect("accept"); + addr + }; + assert!(crate::turnloop_net::live_handles() >= 1); + drop(scope); + // Prove marking reaches the unreferenced owner through the pinned cell: + // after a sweep, a stale-but-not-reused owner would otherwise look alive. + let owner_addr = + (unsafe { np::link_event_owner(link) }.unwrap().to_bits() & POINTER_MASK) as usize; + clear_marks(); + clear_mark_seeds(); + let valid_ptrs = build_valid_pointer_set(); + mark_mutable_registered_roots(&valid_ptrs); + drain_incremental_mark_barrier_seeds(&valid_ptrs); + assert_ne!( + unsafe { (*header_from_user_ptr(owner_addr as *const u8)).gc_flags } & GC_FLAG_MARKED, + 0, + "the pinned cell must mark its otherwise unreferenced owner" + ); + clear_marks(); + clear_mark_seeds(); + let trace = collect_minor_trace(GcTriggerKind::MallocCount); + assert!(trace.copying_nursery.eligible); + full(); + full(); + assert_eq!( + DROPS.load(Ordering::SeqCst), + 0, + "a payload holding a driver handle must survive every collection" + ); + assert_eq!(refs(link), 1, "the installed handle holds exactly one ref"); + let before_finalized = finalized(); + + INSTALL_ACCEPTED.with(|f| f.set(true)); + let _client = std::net::TcpStream::connect(addr).expect("connect to the listener"); + assert!( + pump_until(|| count(NET_ACCEPT, link) == 1), + "the accept must reach the unreferenced listener" + ); + let accept = last(NET_ACCEPT, link).unwrap(); + assert_eq!( + accept.owner, + unsafe { np::link_event_owner(link) }.map(f64::to_bits) + ); + let child = OwnerLink(CHILD.with(Cell::get)); + assert_ne!(child.0, 0, "the sink installed the connection"); + assert_eq!(refs(child), 1, "the accepted handle pins its own payload"); + full(); + assert_eq!(DROPS.load(Ordering::SeqCst), 0); + + // Rule 3: close moves each handle into the driver's close, and the + // payload is released at once; the cell stays while the Closed is owed. + assert!(destroy(link)); + assert!(destroy(child)); + assert_eq!(DROPS.load(Ordering::SeqCst), 2, "release drops T now"); + assert_eq!(refs(link), 1, "the owed Closed keeps its ref after release"); + full(); + assert_eq!(finalized(), before_finalized, "no cell dies while owed"); + assert!( + pump_until(|| count(NET_CLOSED, link) == 1 && count(NET_CLOSED, child) == 1), + "both Closed completions must be dispatched" + ); + assert!(last(NET_CLOSED, link).unwrap().owner.is_some()); + assert_eq!(refs(link), 0); + assert_eq!(refs(child), 0); + full(); + assert_eq!( + finalized(), + before_finalized + 2, + "after the last terminal completion the cycles are garbage" + ); + assert_eq!( + DROPS.load(Ordering::SeqCst), + 2, + "the drop ran exactly once each" + ); +} + +/// The cb-net-2 blocker witness (N5): `close(); listen()` reopens the same +/// object; the first listener's `Closed`, dispatched after the reopen, is +/// delivered once as that close and leaves the new listener untouched. +#[test] +fn n5_a_stale_closed_after_reopen_leaves_the_new_listener_alone() { + let _guard = CopyingNurseryTestGuard::new(0); + let _fixture = Fixture::start(); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(fresh()); + let link = np::owner_link(value.get_nanbox_f64(), &FAMILY).unwrap(); + unsafe { + let c = core(value.get_nanbox_f64()); + transport::tcp_listen(c, link, "127.0.0.1:0".parse().unwrap(), 16, false, true).unwrap(); + transport::accept_start(c, link).unwrap(); + } + assert!(destroy(link)); + let reopened = np::attach( + value.get_nanbox_f64(), + &FAMILY, + Payload { + core: TransportCore::new(ROUTE), + ext: Ext, + }, + 0, + ); + assert_eq!(reopened, Ok(())); + let addr = unsafe { + let c = core(value.get_nanbox_f64()); + let addr = transport::tcp_listen(c, link, "127.0.0.1:0".parse().unwrap(), 16, false, true) + .unwrap(); + transport::accept_start(c, link).unwrap(); + addr + }; + assert_eq!(refs(link), 2, "the owed Closed and the new handle"); + let handle = unsafe { (*core(value.get_nanbox_f64())).handle() }; + assert!(handle.is_some()); + assert!( + pump_until(|| count(NET_CLOSED, link) == 1), + "the first listener's Closed must be dispatched" + ); + assert_eq!( + unsafe { (*core(value.get_nanbox_f64())).handle() }, + handle, + "a stale Closed must not touch the reopened listener" + ); + assert_eq!(refs(link), 1); + let _client = std::net::TcpStream::connect(addr).unwrap(); + assert!( + pump_until(|| count(NET_ACCEPT, link) == 1), + "the reopened listener still accepts" + ); + assert!(destroy(link)); + assert!(pump_until(|| count(NET_CLOSED, link) == 2)); + assert_eq!(refs(link), 0); +} + +/// N11: the pipe server path. `close()`, a synchronous unlink (the binding's +/// job), then `listen(samePath)`: the old `Closed` neither disturbs the new +/// listener nor its socket file. +#[cfg(unix)] +#[test] +fn n11_a_pipe_server_relistens_on_its_path_across_a_stale_closed() { + let _guard = CopyingNurseryTestGuard::new(0); + let _fixture = Fixture::start(); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let dir = std::env::temp_dir().join(format!("perry-netp0-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("n11.sock"); + let _ = std::fs::remove_file(&path); + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(fresh()); + let link = np::owner_link(value.get_nanbox_f64(), &FAMILY).unwrap(); + unsafe { + let c = core(value.get_nanbox_f64()); + transport::pipe_listen(c, link, &path, 16).unwrap(); + transport::accept_start(c, link).unwrap(); + } + assert!(destroy(link)); + std::fs::remove_file(&path).expect("the binding unlinks at close, synchronously"); + np::attach( + value.get_nanbox_f64(), + &FAMILY, + Payload { + core: TransportCore::new(ROUTE), + ext: Ext, + }, + 0, + ) + .unwrap(); + unsafe { + let c = core(value.get_nanbox_f64()); + transport::pipe_listen(c, link, &path, 16).unwrap(); + transport::accept_start(c, link).unwrap(); + } + assert!(pump_until(|| count(NET_CLOSED, link) == 1)); + assert!( + path.exists(), + "the old Closed must not remove the new socket" + ); + assert!(unsafe { (*core(value.get_nanbox_f64())).handle() }.is_some()); + let _client = std::os::unix::net::UnixStream::connect(&path).expect("connect to the new path"); + assert!(pump_until(|| count(NET_ACCEPT, link) == 1)); + assert!(destroy(link)); + assert!(pump_until(|| count(NET_CLOSED, link) == 2)); + let _ = std::fs::remove_dir_all(&dir); +} + +/// N7: dropping a payload never reaches the loop, from the sweep or from an +/// explicit release (the cb-net blocker's "record Drop releases listener" +/// witness becomes this guard). +#[test] +fn n7_dropping_a_transport_payload_never_reaches_the_driver() { + let _guard = CopyingNurseryTestGuard::new(0); + let _fixture = Fixture::start(); + let _no_stack = ConservativeScanDisabledGuard::new(); + let before = finalized(); + { + let scope = RuntimeHandleScope::new(); + let _never_connected = scope.root_nanbox_f64(fresh()); + let released = scope.root_nanbox_f64(fresh()); + let _forbid = transport::ForbidDriver::new(); + assert_eq!( + np::close(released.get_nanbox_f64(), &FAMILY), + CloseOutcome::Closed + ); + } + assert_eq!(DROPS.load(Ordering::SeqCst), 1); + { + let _forbid = transport::ForbidDriver::new(); + full(); + } + assert_eq!( + DROPS.load(Ordering::SeqCst), + 2, + "the sweep dropped the other" + ); + assert_eq!(finalized(), before + 2); +} + +/// N10: a moving collection between submission and completion moves the +/// owner; the token still names the cell, and the event reaches the moved +/// object (the pinned cell's owner edge is rewritten). +#[test] +fn n10_a_moved_owner_still_receives_its_events() { + let _guard = CopyingNurseryTestGuard::new(1); + let _fixture = Fixture::start(); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(fresh()); + let link = np::owner_link(value.get_nanbox_f64(), &FAMILY).unwrap(); + let addr = unsafe { + let c = core(value.get_nanbox_f64()); + let addr = transport::tcp_listen(c, link, "127.0.0.1:0".parse().unwrap(), 16, false, true) + .unwrap(); + transport::accept_start(c, link).unwrap(); + addr + }; + let before = value.get_nanbox_f64().to_bits(); + let trace = collect_minor_trace(GcTriggerKind::MallocCount); + assert!(trace.copying_nursery.eligible); + let moved = value.get_nanbox_f64().to_bits(); + assert_ne!(before, moved, "the fixture must move its owner"); + let _client = std::net::TcpStream::connect(addr).unwrap(); + assert!(pump_until(|| count(NET_ACCEPT, link) == 1)); + assert_eq!( + last(NET_ACCEPT, link).unwrap().owner, + Some(value.get_nanbox_f64().to_bits()), + "the event must reach the moved owner" + ); + assert!(destroy(link)); + assert!(pump_until(|| count(NET_CLOSED, link) == 1)); +} + +/// N6 / L8 (runtime half): a worker agent going away with a `Closed` still +/// owed runs no JS for it and dereferences no token; its heap teardown then +/// finalizes the cell despite the ref. +#[test] +fn n6_teardown_discards_owed_completions_without_dispatch() { + let _guard = CopyingNurseryTestGuard::new(0); + let before = finalized(); + let (dispatched, discarded) = std::thread::spawn(|| { + let _fixture = Fixture::start(); + let scope = RuntimeHandleScope::new(); + let value = scope.root_nanbox_f64(fresh()); + let link = np::owner_link(value.get_nanbox_f64(), &FAMILY).unwrap(); + unsafe { + let c = core(value.get_nanbox_f64()); + transport::tcp_listen(c, link, "127.0.0.1:0".parse().unwrap(), 16, false, true) + .unwrap(); + } + assert!(destroy(link)); + drop(scope); + // The agent's teardown sets the discard rule before its last turns. + transport::begin_teardown_for_test(); + assert!( + pump_until(|| transport::discarded_for_test() + count(NET_CLOSED, link) >= 1), + "the owed Closed must arrive during teardown, or the witness is vacuous" + ); + crate::event_pump::shutdown_agent_loop(); + (count(NET_CLOSED, link), transport::discarded_for_test()) + }) + .join() + .unwrap(); + assert_eq!(dispatched, 0, "teardown must discard, never dispatch"); + assert!(discarded >= 1); + assert_eq!( + finalized(), + before + 1, + "the worker heap teardown finalizes the pinned cell" + ); + assert_eq!(DROPS.load(Ordering::SeqCst), 1); +} + +/// The ABI digest folds the opaque core block: a binding built against +/// another block size computes another digest and is refused at +/// registration (perry-ffi's `register_*_sink`). The block is tight on the +/// platforms this test runs on, so the struct and the block move together. +#[test] +fn the_net_abi_digest_folds_the_transport_core_block() { + use crate::turnloop_net::abi::{js_perry_net_abi_layout, net_abi_layout}; + use crate::turnloop_net::TRANSPORT_CORE_WORDS; + assert_eq!( + js_perry_net_abi_layout(), + net_abi_layout(TRANSPORT_CORE_WORDS) + ); + assert_ne!( + net_abi_layout(TRANSPORT_CORE_WORDS), + net_abi_layout(TRANSPORT_CORE_WORDS + 1) + ); + assert_ne!( + net_abi_layout(TRANSPORT_CORE_WORDS), + net_abi_layout(TRANSPORT_CORE_WORDS - 1) + ); + let size = std::mem::size_of::(); + eprintln!("TransportCore: {size} bytes, block {TRANSPORT_CORE_WORDS} words"); + #[cfg(not(windows))] + assert_eq!( + size.div_ceil(8), + TRANSPORT_CORE_WORDS, + "the reserved block must match the core exactly on this platform" + ); + assert_eq!(std::mem::offset_of!(Payload, core), 0, "core first"); +} + +#[test] +fn every_net_transport_sabotage_makes_its_witness_red() { + let exe = std::env::current_exe().unwrap(); + for (var, fault, witness) in [ + ( + "PERRY_TEST_NET_SABOTAGE", + "skip_ref", + "n1_a_handle_holding_payload_lives_until_its_closed_is_dispatched", + ), + ( + "PERRY_TEST_CALLBACK_SABOTAGE", + "mark", + "n1_a_handle_holding_payload_lives_until_its_closed_is_dispatched", + ), + ( + "PERRY_TEST_NET_SABOTAGE", + "handle_check", + "n5_a_stale_closed_after_reopen_leaves_the_new_listener_alone", + ), + ( + "PERRY_TEST_NET_SABOTAGE", + "handle_check", + "n11_a_pipe_server_relistens_on_its_path_across_a_stale_closed", + ), + ( + "PERRY_TEST_NET_SABOTAGE", + "drop_closes", + "n7_dropping_a_transport_payload_never_reaches_the_driver", + ), + ( + "PERRY_TEST_CALLBACK_SABOTAGE", + "rewrite", + "n10_a_moved_owner_still_receives_its_events", + ), + ( + "PERRY_TEST_NET_SABOTAGE", + "teardown_dispatch", + "n6_teardown_discards_owed_completions_without_dispatch", + ), + ] { + if cfg!(not(unix)) && witness.starts_with("n11") { + continue; + } + let name = format!("gc::tests::net_transport::{witness}"); + let output = std::process::Command::new(&exe) + .args(["--exact", &name, "--nocapture", "--test-threads=1"]) + .env(var, fault) + .output() + .unwrap(); + assert!( + String::from_utf8_lossy(&output.stdout).contains("running 1 test"), + "{witness} must exist" + ); + assert!( + !output.status.success(), + "{var}={fault} must make {witness} RED" + ); + eprintln!( + "net transport sabotage {var}={fault}: {witness} RED ({})", + output.status + ); + } +} diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index df311ef42b..fd497abffe 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -146,6 +146,7 @@ pub mod native_abi; pub mod native_arena; pub mod native_handle; pub mod native_payload; +pub mod native_payload_abi; #[cfg(target_os = "linux")] mod native_stack; pub mod native_value_profile; diff --git a/crates/perry-runtime/src/native_handle.rs b/crates/perry-runtime/src/native_handle.rs index a9b7a17afb..c442b8845b 100644 --- a/crates/perry-runtime/src/native_handle.rs +++ b/crates/perry-runtime/src/native_handle.rs @@ -28,7 +28,7 @@ pub(crate) static PAYLOAD_FINALIZED: std::sync::atomic::AtomicUsize = static MAIN_THREAD_ID: AtomicU64 = AtomicU64::new(0); -type NativeHandleFinalizer = unsafe extern "C" fn(*mut c_void, *mut c_void); +pub(crate) type NativeHandleFinalizer = unsafe extern "C" fn(*mut c_void, *mut c_void); /// GC payload for a Perry native handle. #[repr(C)] diff --git a/crates/perry-runtime/src/native_payload.rs b/crates/perry-runtime/src/native_payload.rs index 32f2de35a5..8cf38c079b 100644 --- a/crates/perry-runtime/src/native_payload.rs +++ b/crates/perry-runtime/src/native_payload.rs @@ -215,6 +215,13 @@ fn slot_index(class_id: u32) -> usize { index } +/// Whether `class_id` names a slot in the payload-family prototype table: +/// the check a C-ABI family descriptor must pass before its id is used. +#[inline] +pub(crate) fn is_payload_class_id(class_id: u32) -> bool { + slot_index_if_payload(class_id).is_some() +} + #[inline] fn slot_index_if_payload(class_id: u32) -> Option { if matches!( @@ -239,7 +246,25 @@ pub(crate) fn scan_payload_prototype_roots_mut(visitor: &mut crate::gc::RuntimeR } fn family_prototype(family: &NativePayloadFamily) -> *mut ObjectHeader { - let index = slot_index(family.class_id); + family_prototype_with( + family.class_id, + family.name, + family.constructor_export, + family.constructor_length, + &|builder| (family.install_prototype)(builder), + ) +} + +/// The per-realm prototype of the family `class_id`, built on first use with +/// `install`. Shared by Rust families and C-ABI families. +pub(crate) fn family_prototype_with( + class_id: u32, + name: &str, + constructor_export: Option<(&'static str, &'static str)>, + constructor_length: u32, + install: &dyn Fn(&mut PayloadPrototype), +) -> *mut ObjectHeader { + let index = slot_index(class_id); let existing = PAYLOAD_PROTOTYPES.with(|slots| slots[index].load(Ordering::Acquire)); if existing != 0 { return existing as *mut ObjectHeader; @@ -252,8 +277,8 @@ fn family_prototype(family: &NativePayloadFamily) -> *mut ObjectHeader { return proto; } let mut builder = PayloadPrototype { proto }; - (family.install_prototype)(&mut builder); - let constructor = match family.constructor_export { + install(&mut builder); + let constructor = match constructor_export { Some((module, export)) => crate::object::bound_native_callable_export_value(module, export), None => { let closure = crate::closure::js_closure_alloc( @@ -263,10 +288,10 @@ fn family_prototype(family: &NativePayloadFamily) -> *mut ObjectHeader { if closure.is_null() { f64::from_bits(crate::value::TAG_UNDEFINED) } else { - crate::object::native_module::set_bound_native_closure_name(closure, family.name); + crate::object::native_module::set_bound_native_closure_name(closure, name); crate::object::native_module::set_builtin_closure_length( closure as usize, - family.constructor_length, + constructor_length, ); crate::value::js_nanbox_pointer(closure as i64) } @@ -320,15 +345,62 @@ extern "C" fn payload_ctor_thunk( /// layout half catches a family that reads its payload as the wrong type. #[inline(always)] const fn type_tag(class_id: u32) -> u64 { - (class_id as u64) - | ((std::mem::size_of::() as u64 & 0xFF_FFFF) << 32) - | ((std::mem::align_of::() as u64 & 0xFF) << 56) + type_tag_of( + class_id, + std::mem::size_of::(), + std::mem::align_of::(), + ) +} + +/// [`type_tag`] from a payload's size and alignment, for a family declared +/// over the C ABI (`native_payload_abi.rs`), whose `T` the runtime never sees. +/// perry-ffi computes the same value from its own `size_of::()`. +#[inline(always)] +pub(crate) const fn type_tag_of(class_id: u32, size: usize, align: usize) -> u64 { + (class_id as u64) | ((size as u64 & 0xFF_FFFF) << 32) | ((align as u64 & 0xFF) << 56) } unsafe extern "C" fn drop_payload(resource: *mut c_void, _hint: *mut c_void) { drop(Box::from_raw(resource as *mut T)); } +/// What the cell knows about a family's payload type: its tag and the drop +/// thunk that frees it. Monomorphized for an in-tree family; supplied by the +/// family descriptor for an ext crate's family (`native_payload_abi.rs`). +#[derive(Clone, Copy)] +pub(crate) struct PayloadType { + pub(crate) tag: u64, + pub(crate) drop: crate::native_handle::NativeHandleFinalizer, +} + +impl PayloadType { + #[inline(always)] + fn of(class_id: u32) -> Self { + Self { + tag: type_tag::(class_id), + drop: drop_payload::, + } + } +} + +/// The family facts the allocation and attach paths read, for a Rust family +/// and a C-ABI family alike. +pub(crate) struct FamilyParts<'a> { + pub(crate) class_id: u32, + pub(crate) links_owner: bool, + pub(crate) name: &'a str, +} + +impl NativePayloadFamily { + fn parts(&self) -> FamilyParts<'_> { + FamilyParts { + class_id: self.class_id, + links_owner: self.links_owner, + name: self.name, + } + } +} + /// Is `word` (an `ObjectMeta.native_state`) a payload cell reference? The /// one predicate the meta record's GC arm uses to decide whether the word is /// an edge. Only payload families store a POINTER_TAG-boxed word there. @@ -362,6 +434,21 @@ pub fn alloc_closed(family: &'static NativePayloadFamily, own: &[(&[u8], f64)]) alloc_cell::<()>(family, None, 0, own, proto) } +/// [`alloc`] / [`alloc_closed`] for a family whose payload type the runtime +/// does not know (the C ABI). `resource` is a boxed payload the family's +/// `ty.drop` frees, or null for a CLOSED instance. On failure the resource is +/// dropped here, so the caller never owns it after this call. +pub(crate) fn alloc_raw( + family: &FamilyParts<'_>, + resource: *mut c_void, + ty: PayloadType, + external_bytes: usize, + own: &[(&[u8], f64)], + proto: *mut ObjectHeader, +) -> f64 { + alloc_cell_raw(family, resource, ty, external_bytes, own, proto) +} + /// Allocate a payload-family instance linked to an already-materialized /// constructor prototype, including AsyncLocalStorage and AsyncResource. pub fn alloc_with_prototype( @@ -380,6 +467,28 @@ fn alloc_cell( external_bytes: usize, own: &[(&[u8], f64)], proto: *mut ObjectHeader, +) -> f64 { + let ty = PayloadType::of::(family.class_id); + let resource = payload.map_or(std::ptr::null_mut(), |p| { + Box::into_raw(Box::new(p)) as *mut c_void + }); + alloc_cell_raw(&family.parts(), resource, ty, external_bytes, own, proto) +} + +/// Free a boxed payload the cell never took (a refused allocation or attach). +unsafe fn drop_untaken(resource: *mut c_void, ty: PayloadType) { + if !resource.is_null() { + (ty.drop)(resource, std::ptr::null_mut()); + } +} + +fn alloc_cell_raw( + family: &FamilyParts<'_>, + resource: *mut c_void, + ty: PayloadType, + external_bytes: usize, + own: &[(&[u8], f64)], + proto: *mut ObjectHeader, ) -> f64 { let scope = crate::gc::RuntimeHandleScope::new(); let own_roots: Vec<_> = own @@ -387,17 +496,19 @@ fn alloc_cell( .map(|&(key, value)| (key, scope.root_nanbox_f64(value))) .collect(); if proto.is_null() { + unsafe { drop_untaken(resource, ty) }; return f64::from_bits(crate::value::TAG_UNDEFINED); } let obj = unsafe { born_instance(family.class_id, proto, own.len() as u32) }; if obj.is_null() { + unsafe { drop_untaken(resource, ty) }; return f64::from_bits(crate::value::TAG_UNDEFINED); } let obj = scope.root_raw_mut_ptr(obj); for (key, value) in &own_roots { set_own(&scope, &obj, key, value.get_nanbox_f64()); } - attach_rooted(&obj, family, payload, external_bytes); + attach_rooted(&obj, family, resource, ty, external_bytes); obj.with_mut_ptr::(|obj| crate::value::js_nanbox_pointer(obj as i64)) } @@ -411,7 +522,10 @@ pub fn attach_to_object( payload: T, external_bytes: usize, ) -> bool { + let ty = PayloadType::of::(family.class_id); + let resource = Box::into_raw(Box::new(payload)) as *mut c_void; let Some(obj) = any_object(value) else { + unsafe { drop_untaken(resource, ty) }; return false; }; let scope = crate::gc::RuntimeHandleScope::new(); @@ -419,48 +533,49 @@ pub fn attach_to_object( let meta = obj .with_mut_ptr::(|obj| unsafe { crate::object::object_meta_ensure(obj) }); if meta.is_null() { + unsafe { drop_untaken(resource, ty) }; return false; } let previous = unsafe { (*meta).native_state }; if is_payload_state_word(previous) { let cell = (previous & crate::value::POINTER_MASK) as *mut NativeHandleHeader; - return attach_cell( + return attach_cell_raw( obj.with_mut_ptr::(|obj| crate::value::js_nanbox_pointer(obj as i64)), cell, - family, - payload, + family.class_id, + resource, + ty, external_bytes, ) .is_ok(); } - attach_rooted(&obj, family, Some(payload), external_bytes); + attach_rooted(&obj, &family.parts(), resource, ty, external_bytes); true } -fn attach_rooted( +fn attach_rooted( obj: &crate::gc::RuntimeHandle<'_>, - family: &'static NativePayloadFamily, - payload: Option, + family: &FamilyParts<'_>, + resource: *mut c_void, + ty: PayloadType, external_bytes: usize, ) { let meta = obj .with_mut_ptr::(|obj| unsafe { crate::object::object_meta_ensure(obj) }); if meta.is_null() { + unsafe { drop_untaken(resource, ty) }; return; } // The cell allocation may collect; re-read meta through the rooted object. - let resource = payload.map_or(std::ptr::null_mut(), |p| { - Box::into_raw(Box::new(p)) as *mut c_void - }); let cell = unsafe { crate::native_handle::native_handle_new_rust_payload( resource, if resource.is_null() { family.class_id as u64 } else { - type_tag::(family.class_id) + ty.tag }, - drop_payload::, + ty.drop, family.name, ) }; @@ -627,6 +742,25 @@ pub unsafe fn payload_mut<'a, T: 'static>( Ok(&mut *(resource as *mut T)) } +/// [`payload_mut`] for a C-ABI family: the payload's address, checked against +/// the family's class id and its payload tag (`type_tag_of`). +/// +/// # Safety +/// As [`payload_mut`]. +#[inline] +pub(crate) unsafe fn payload_ptr_raw( + value: f64, + class_id: u32, + tag: u64, +) -> Result<*mut c_void, PayloadMiss> { + let cell = payload_cell(value, class_id)?; + let resource = crate::native_handle::native_handle_rust_payload_ptr(cell, tag); + if resource.is_null() { + return Err(PayloadMiss::Closed); + } + Ok(resource) +} + /// The live payload attached to any ordinary object, including a subclass /// instance whose own class id differs from the native base's id. pub unsafe fn payload_mut_attached<'a, T: 'static>( @@ -671,7 +805,11 @@ pub enum Lifecycle { } pub fn lifecycle(value: f64, family: &NativePayloadFamily) -> Result { - let cell = payload_cell(value, family.class_id)?; + lifecycle_class(value, family.class_id) +} + +pub(crate) fn lifecycle_class(value: f64, class_id: u32) -> Result { + let cell = payload_cell(value, class_id)?; unsafe { if (*cell).finalized != 0 || (*cell).creator_thread_id != crate::native_handle::current_thread_id() @@ -705,45 +843,51 @@ pub fn attach( payload: T, external_bytes: usize, ) -> Result<(), AttachMiss> { - let cell = payload_cell(value, family.class_id).map_err(|_| AttachMiss::Foreign)?; - attach_cell(value, cell, family, payload, external_bytes) + let ty = PayloadType::of::(family.class_id); + let resource = Box::into_raw(Box::new(payload)) as *mut c_void; + let Ok(cell) = payload_cell(value, family.class_id) else { + unsafe { drop_untaken(resource, ty) }; + return Err(AttachMiss::Foreign); + }; + attach_cell_raw(value, cell, family.class_id, resource, ty, external_bytes) +} + +/// [`attach`] for a C-ABI family: `resource` is the boxed payload, freed by +/// `ty.drop` here when the attach is refused. +pub(crate) fn attach_raw( + value: f64, + class_id: u32, + resource: *mut c_void, + ty: PayloadType, + external_bytes: usize, +) -> Result<(), AttachMiss> { + let Ok(cell) = payload_cell(value, class_id) else { + unsafe { drop_untaken(resource, ty) }; + return Err(AttachMiss::Foreign); + }; + attach_cell_raw(value, cell, class_id, resource, ty, external_bytes) } -fn attach_cell( +fn attach_cell_raw( value: f64, cell: *mut NativeHandleHeader, - family: &'static NativePayloadFamily, - payload: T, + class_id: u32, + resource: *mut c_void, + ty: PayloadType, external_bytes: usize, ) -> Result<(), AttachMiss> { + let refused = unsafe { attach_refusal(cell, class_id, ty) }; + if let Some(miss) = refused { + unsafe { drop_untaken(resource, ty) }; + return Err(miss); + } unsafe { - if (*cell).magic != crate::native_handle::NATIVE_HANDLE_MAGIC - || (*cell).creator_thread_id != crate::native_handle::current_thread_id() - { - return Err(AttachMiss::Foreign); - } - if (*cell).finalized != 0 && !lifecycle_sabotage("attach_finalized") { - return Err(AttachMiss::Finalized); - } - if (*cell).flags & CLOSING != 0 || (*cell).busy != 0 { - return Err(AttachMiss::Closing); - } - if !(*cell).resource_ptr.is_null() { - return Err(AttachMiss::Open); - } - let tag = type_tag::(family.class_id); - if (*cell).type_id != family.class_id as u64 - && ((*cell).type_id != tag || (*cell).finalizer != drop_payload:: as *mut c_void) - { - return Err(AttachMiss::Foreign); - } // alloc_closed cannot know T. Establish the layout/thunk on first // attach; subsequent opens keep both unchanged. - if (*cell).type_id == family.class_id as u64 { - (*cell).type_id = tag; - (*cell).finalizer = drop_payload:: as *mut c_void; + if (*cell).type_id == class_id as u64 { + (*cell).type_id = ty.tag; + (*cell).finalizer = ty.drop as *mut c_void; } - let resource = Box::into_raw(Box::new(payload)) as *mut c_void; crate::native_handle::native_handle_attach_rust_payload(cell, resource); // Root the owner before reporting bytes: pacing can collect. The // payload is already installed and reachable through the owner. @@ -754,6 +898,34 @@ fn attach_cell( Ok(()) } +/// Why `cell` cannot take a payload of type `ty` now, if it cannot. +unsafe fn attach_refusal( + cell: *mut NativeHandleHeader, + class_id: u32, + ty: PayloadType, +) -> Option { + if (*cell).magic != crate::native_handle::NATIVE_HANDLE_MAGIC + || (*cell).creator_thread_id != crate::native_handle::current_thread_id() + { + return Some(AttachMiss::Foreign); + } + if (*cell).finalized != 0 && !lifecycle_sabotage("attach_finalized") { + return Some(AttachMiss::Finalized); + } + if (*cell).flags & CLOSING != 0 || (*cell).busy != 0 { + return Some(AttachMiss::Closing); + } + if !(*cell).resource_ptr.is_null() { + return Some(AttachMiss::Open); + } + if (*cell).type_id != class_id as u64 + && ((*cell).type_id != ty.tag || (*cell).finalizer != ty.drop as *mut c_void) + { + return Some(AttachMiss::Foreign); + } + None +} + /// A resource incarnation. Store it in reopenable payloads, children and data /// completions, and compare once before using the current resource. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -768,7 +940,11 @@ pub fn next_open_serial() -> OpenSerial { /// Close immediately, or defer native destruction until the outer C call ends. pub fn close(value: f64, family: &NativePayloadFamily) -> CloseOutcome { - let Ok(cell) = payload_cell(value, family.class_id) else { + close_class(value, family.class_id) +} + +pub(crate) fn close_class(value: f64, class_id: u32) -> CloseOutcome { + let Ok(cell) = payload_cell(value, class_id) else { return CloseOutcome::Foreign; }; unsafe { @@ -807,10 +983,28 @@ pub struct OwnerLink(pub(crate) usize); /// Obtain a link in any non-finalized state with the traced owner edge enabled. pub fn owner_link(value: f64, family: &NativePayloadFamily) -> Result { - if !family.links_owner { + owner_link_class(value, family.class_id, family.links_owner) +} + +/// The largest cell address a link may carry: a transport token packs +/// `cell >> 3` into 54 bits (`turnloop_net::transport`), and user-space +/// addresses stay below 2^56 on every 64-bit target perry ships (5-level +/// paging included). +pub(crate) const LINK_ADDRESS_LIMIT: u64 = 1 << 56; + +pub(crate) fn owner_link_class( + value: f64, + class_id: u32, + links_owner: bool, +) -> Result { + if !links_owner { return Err(PayloadMiss::Foreign); } - let cell = payload_cell(value, family.class_id)?; + let cell = payload_cell(value, class_id)?; + assert!( + (cell as u64) < LINK_ADDRESS_LIMIT && cell as usize & 7 == 0, + "a payload cell must be 8-aligned below 2^56 to travel in a link token" + ); unsafe { if (*cell).magic != crate::native_handle::NATIVE_HANDLE_MAGIC || (*cell).finalized != 0 @@ -862,6 +1056,17 @@ pub unsafe fn link_event_owner(link: OwnerLink) -> Option { ((*cell).owner != 0).then(|| f64::from_bits((*cell).owner)) } +/// The OPEN payload behind `link`, or null (CLOSING, CLOSED, finalized or +/// another thread). Never throws or allocates. The transport dispatch reads a +/// payload's `TransportCore` through this, after `link_event_owner`. +/// +/// # Safety +/// As [`link_event_owner`]. +#[inline] +pub(crate) unsafe fn link_open_payload(link: OwnerLink) -> *mut c_void { + crate::native_handle::rust_payload_ptr_on_owner_thread(link.0 as *mut NativeHandleHeader) +} + #[repr(C)] pub struct CallbackSite { pub link: OwnerLink, @@ -1259,7 +1464,11 @@ pub fn close_attached(value: f64, family: &NativePayloadFamily) -> b /// Re-state the native bytes a live payload retains (after a buffer grew or /// was released). No-op for a foreign or closed value. pub fn set_external_bytes(value: f64, family: &NativePayloadFamily, bytes: usize) { - if let Ok(cell) = payload_cell(value, family.class_id) { + set_external_bytes_class(value, family.class_id, bytes) +} + +pub(crate) fn set_external_bytes_class(value: f64, class_id: u32, bytes: usize) { + if let Ok(cell) = payload_cell(value, class_id) { unsafe { crate::native_handle::native_handle_set_external_bytes(cell, bytes) }; } } @@ -1370,7 +1579,7 @@ pub fn js_state(value: f64, family: &NativePayloadFamily, create: bool) -> f64 { js_state_for_class(value, family.class_id, create) } -fn js_state_for_class(value: f64, class_id: u32, create: bool) -> f64 { +pub(crate) fn js_state_for_class(value: f64, class_id: u32, create: bool) -> f64 { let undefined = undefined(); let Some(obj) = instance_of(value, class_id) else { return undefined; diff --git a/crates/perry-runtime/src/native_payload_abi.rs b/crates/perry-runtime/src/native_payload_abi.rs new file mode 100644 index 0000000000..8239906d23 --- /dev/null +++ b/crates/perry-runtime/src/native_payload_abi.rs @@ -0,0 +1,508 @@ +//! The C ABI of the native-payload pattern, for families that live in a +//! separately linked binding (#11919, NET-TRANSPORT-DESIGN P0). +//! +//! An ext crate (`perry-ext-net`, ...) links only `perry-ffi`, so it cannot +//! name `NativePayloadFamily` or call the generic `native_payload::alloc::`. +//! It describes its family with a `#[repr(C)]` [`PerryPayloadFamily`] instead: +//! the class id, whether the cell links its owner, the constructor name, a +//! prototype installer, and the payload's size, alignment and drop thunk. The +//! payload itself is a `Box` the binding allocates and hands over as a raw +//! pointer; from then on it is an ordinary payload cell, with every rule of +//! `docs/native-payload-pattern.md`. +//! +//! Every entry point here forwards to the same code the in-tree families use +//! (`native_payload.rs`), so there is one implementation of the cell, its +//! lifecycle and its owner link. The descriptor carries a layout digest in its +//! first word ([`js_perry_payload_abi_layout`]); a descriptor written against +//! another layout is refused rather than misread. + +use std::ffi::c_void; + +use crate::native_payload::{self as np, FamilyParts, PayloadMiss, PayloadType}; +use crate::object::ObjectHeader; + +/// Revision of this ABI. Bumped with any signature or descriptor change. +pub const PERRY_PAYLOAD_ABI_VERSION: u8 = 1; + +/// A family, as a binding declares it. Lives in a `static` of the binding. +/// +/// `abi` must stay the first field: it is read before any other field, so a +/// descriptor of a different layout is recognised before it is misread. +#[repr(C)] +pub struct PerryPayloadFamily { + /// [`js_perry_payload_abi_layout`] as the binding computed it. + pub abi: u64, + /// The family's id from `native_class_ids.rs`. + pub class_id: u32, + /// Nonzero: the cell keeps a traced edge to its owner (`owner_link`). + pub links_owner: u32, + /// `x.constructor.length` of the stand-in constructor. + pub constructor_length: u32, + /// Padding, so the layout does not depend on how a compiler packs it. + pub _reserved: u32, + /// The constructor name node reports, UTF-8, not NUL-terminated, static. + pub name: *const u8, + /// Length of `name`. + pub name_len: usize, + /// Installs the prototype's methods, once per realm. May be absent. + pub install_prototype: Option, + /// Frees one boxed payload. Runs at release, sweep or thread teardown; it + /// must not allocate on the GC heap, call JS or touch thread-locals. + pub drop_payload: unsafe extern "C" fn(resource: *mut c_void, hint: *mut c_void), + /// `size_of::()` of the payload. + pub payload_size: usize, + /// `align_of::()` of the payload. + pub payload_align: usize, +} + +/// One own enumerable property set at allocation (node's own fields). +#[repr(C)] +pub struct PerryPayloadOwnProp { + /// Property name, ASCII, not NUL-terminated. + pub key: *const u8, + /// Length of `key`. + pub key_len: usize, + /// NaN-boxed value; rooted by the runtime for the allocation. + pub value: f64, +} + +/// Digest of [`PerryPayloadFamily`]'s layout plus the ABI revision. perry-ffi +/// computes the same expression over its own copy of the struct. +#[no_mangle] +pub extern "C" fn js_perry_payload_abi_layout() -> u64 { + payload_abi_layout() +} + +pub(crate) const fn payload_abi_layout() -> u64 { + use std::mem::{offset_of, size_of}; + (size_of::() as u64) << 48 + | (offset_of!(PerryPayloadFamily, class_id) as u64) << 40 + | (offset_of!(PerryPayloadFamily, name) as u64) << 32 + | (offset_of!(PerryPayloadFamily, install_prototype) as u64) << 24 + | (offset_of!(PerryPayloadFamily, drop_payload) as u64) << 16 + | (offset_of!(PerryPayloadFamily, payload_align) as u64) << 8 + | PERRY_PAYLOAD_ABI_VERSION as u64 +} + +/// `PayloadMiss` / `Lifecycle` / `CloseOutcome` / `AttachMiss` codes. +pub const PERRY_PAYLOAD_OK: i32 = 0; +/// Not an instance of the family (or a refused descriptor). +pub const PERRY_PAYLOAD_FOREIGN: i32 = -1; +/// An instance whose payload is released or finalized. +pub const PERRY_PAYLOAD_CLOSED: i32 = -2; +/// `lifecycle`: OPEN. +pub const PERRY_PAYLOAD_OPEN: i32 = 1; +/// `lifecycle`: CLOSING (a deferred close is pending). +pub const PERRY_PAYLOAD_CLOSING: i32 = 2; +/// `lifecycle` / `close`: CLOSED (`close`: it was already closed). +pub const PERRY_PAYLOAD_IS_CLOSED: i32 = 3; +/// `close`: deferred until the outermost native call returns. +pub const PERRY_PAYLOAD_DEFERRED: i32 = 4; +/// `attach`: the cell is OPEN. +pub const PERRY_PAYLOAD_ATTACH_OPEN: i32 = 5; +/// `attach`: the cell is CLOSING. +pub const PERRY_PAYLOAD_ATTACH_CLOSING: i32 = 6; +/// `attach`: the cell was finalized (sweep or teardown). +pub const PERRY_PAYLOAD_ATTACH_FINALIZED: i32 = 7; + +/// A validated descriptor. +struct Family<'a> { + parts: FamilyParts<'a>, + raw: &'a PerryPayloadFamily, +} + +impl Family<'_> { + fn ty(&self) -> PayloadType { + PayloadType { + tag: np::type_tag_of( + self.parts.class_id, + self.raw.payload_size, + self.raw.payload_align, + ), + drop: self.raw.drop_payload, + } + } +} + +/// The descriptor behind `family`, or `None` when it was written against +/// another layout, names a class id outside the payload block, or has a +/// name that is not UTF-8. +/// +/// # Safety +/// `family` is null or points at a descriptor that outlives the program +/// (a binding's `static`). +unsafe fn family<'a>(family: *const PerryPayloadFamily) -> Option> { + let raw = family.as_ref()?; + if raw.abi != payload_abi_layout() || !np::is_payload_class_id(raw.class_id) { + return None; + } + let name = if raw.name.is_null() || raw.name_len == 0 { + "" + } else { + std::str::from_utf8(std::slice::from_raw_parts(raw.name, raw.name_len)).ok()? + }; + Some(Family { + parts: FamilyParts { + class_id: raw.class_id, + links_owner: raw.links_owner != 0, + name, + }, + raw, + }) +} + +fn undefined() -> f64 { + f64::from_bits(crate::value::TAG_UNDEFINED) +} + +/// # Safety +/// `own` points at `own_len` readable entries whose keys are readable. +unsafe fn own_props<'a>(own: *const PerryPayloadOwnProp, own_len: usize) -> Vec<(&'a [u8], f64)> { + if own.is_null() || own_len == 0 { + return Vec::new(); + } + std::slice::from_raw_parts(own, own_len) + .iter() + .map(|p| { + let key = if p.key.is_null() { + &[][..] + } else { + std::slice::from_raw_parts(p.key, p.key_len) + }; + (key, p.value) + }) + .collect() +} + +fn prototype(family: &Family<'_>) -> *mut ObjectHeader { + let install = family.raw.install_prototype; + np::family_prototype_with( + family.parts.class_id, + family.parts.name, + None, + family.raw.constructor_length, + &|builder| { + if let Some(install) = install { + // SAFETY: the binding's installer receives the builder for the + // duration of this call only. + unsafe { install(builder as *mut np::PayloadPrototype as *mut c_void) } + } + }, + ) +} + +/// Allocate an instance owning `resource` (a boxed payload, or null for a +/// CLOSED instance). The resource is consumed on every outcome: a refused +/// descriptor or failed allocation frees it with the descriptor's thunk. +/// +/// # Safety +/// `family` is a static descriptor; `resource` is null or a payload of the +/// descriptor's size and alignment that `drop_payload` frees; `own` as +/// [`own_props`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_alloc( + family_ptr: *const PerryPayloadFamily, + resource: *mut c_void, + external_bytes: usize, + own: *const PerryPayloadOwnProp, + own_len: usize, +) -> f64 { + let Some(family) = family(family_ptr) else { + if !resource.is_null() && !family_ptr.is_null() { + ((*family_ptr).drop_payload)(resource, std::ptr::null_mut()); + } + return undefined(); + }; + let own = own_props(own, own_len); + let proto = prototype(&family); + np::alloc_raw( + &family.parts, + resource, + family.ty(), + external_bytes, + &own, + proto, + ) +} + +/// The payload of `value` (`payload_mut`), or null with `*out_miss` set to +/// [`PERRY_PAYLOAD_FOREIGN`] or [`PERRY_PAYLOAD_CLOSED`]. +/// +/// # Safety +/// `family` is a static descriptor; `out_miss` is null or writable. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_ptr( + value: f64, + family_ptr: *const PerryPayloadFamily, + out_miss: *mut i32, +) -> *mut c_void { + let result = match family(family_ptr) { + Some(family) => np::payload_ptr_raw(value, family.parts.class_id, family.ty().tag), + None => Err(PayloadMiss::Foreign), + }; + match result { + Ok(ptr) => ptr, + Err(miss) => { + if !out_miss.is_null() { + // GC_STORE_AUDIT(POINTER_FREE): an i32 code into a caller out-param. + std::ptr::write( + out_miss, + match miss { + PayloadMiss::Foreign => PERRY_PAYLOAD_FOREIGN, + PayloadMiss::Closed => PERRY_PAYLOAD_CLOSED, + }, + ); + } + std::ptr::null_mut() + } + } +} + +/// Explicit close (release). Returns [`PERRY_PAYLOAD_OK`] when released now, +/// [`PERRY_PAYLOAD_DEFERRED`], [`PERRY_PAYLOAD_IS_CLOSED`] or +/// [`PERRY_PAYLOAD_FOREIGN`]. +/// +/// # Safety +/// `family` is a static descriptor. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_close( + value: f64, + family_ptr: *const PerryPayloadFamily, +) -> i32 { + let Some(family) = family(family_ptr) else { + return PERRY_PAYLOAD_FOREIGN; + }; + match np::close_class(value, family.parts.class_id) { + np::CloseOutcome::Closed => PERRY_PAYLOAD_OK, + np::CloseOutcome::Deferred => PERRY_PAYLOAD_DEFERRED, + np::CloseOutcome::AlreadyClosed => PERRY_PAYLOAD_IS_CLOSED, + np::CloseOutcome::Foreign => PERRY_PAYLOAD_FOREIGN, + } +} + +/// Reopen a CLOSED instance in its own cell (`attach`). `resource` is consumed +/// on every outcome. +/// +/// # Safety +/// As [`js_perry_payload_alloc`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_attach( + value: f64, + family_ptr: *const PerryPayloadFamily, + resource: *mut c_void, + external_bytes: usize, +) -> i32 { + let Some(family) = family(family_ptr) else { + if !resource.is_null() && !family_ptr.is_null() { + ((*family_ptr).drop_payload)(resource, std::ptr::null_mut()); + } + return PERRY_PAYLOAD_FOREIGN; + }; + if resource.is_null() { + return PERRY_PAYLOAD_FOREIGN; + } + match np::attach_raw( + value, + family.parts.class_id, + resource, + family.ty(), + external_bytes, + ) { + Ok(()) => PERRY_PAYLOAD_OK, + Err(np::AttachMiss::Foreign) => PERRY_PAYLOAD_FOREIGN, + Err(np::AttachMiss::Open) => PERRY_PAYLOAD_ATTACH_OPEN, + Err(np::AttachMiss::Closing) => PERRY_PAYLOAD_ATTACH_CLOSING, + Err(np::AttachMiss::Finalized) => PERRY_PAYLOAD_ATTACH_FINALIZED, + } +} + +/// [`PERRY_PAYLOAD_OPEN`], [`PERRY_PAYLOAD_CLOSING`], [`PERRY_PAYLOAD_IS_CLOSED`], +/// or a miss code (finalized and wrong-thread cells answer CLOSED). +/// +/// # Safety +/// `family` is a static descriptor. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_lifecycle( + value: f64, + family_ptr: *const PerryPayloadFamily, +) -> i32 { + let Some(family) = family(family_ptr) else { + return PERRY_PAYLOAD_FOREIGN; + }; + match np::lifecycle_class(value, family.parts.class_id) { + Ok(np::Lifecycle::Open) => PERRY_PAYLOAD_OPEN, + Ok(np::Lifecycle::Closing) => PERRY_PAYLOAD_CLOSING, + Ok(np::Lifecycle::Closed) => PERRY_PAYLOAD_IS_CLOSED, + Err(PayloadMiss::Foreign) => PERRY_PAYLOAD_FOREIGN, + Err(PayloadMiss::Closed) => PERRY_PAYLOAD_CLOSED, + } +} + +/// The instance's owner link (the cell address), or 0 when it has none (a +/// family without `links_owner`, a foreign value, a finalized cell). +/// +/// # Safety +/// `family` is a static descriptor. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_owner_link( + value: f64, + family_ptr: *const PerryPayloadFamily, +) -> usize { + let Some(family) = family(family_ptr) else { + return 0; + }; + np::owner_link_class(value, family.parts.class_id, family.parts.links_owner) + .map_or(0, |link| link.0) +} + +/// One ref per outstanding item the link names (`native_payload::link_ref`). +/// +/// # Safety +/// `link` came from [`js_perry_payload_owner_link`] on this thread and its +/// cell is alive. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_link_ref(link: usize) { + np::link_ref(np::OwnerLink(link)); +} + +/// Match a [`js_perry_payload_link_ref`]. +/// +/// # Safety +/// As [`js_perry_payload_link_ref`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_link_unref(link: usize) { + np::link_unref(np::OwnerLink(link)); +} + +/// The owner for a pump event (OPEN, CLOSING or CLOSED; not finalized). Writes +/// it to `*out` and returns 1, or returns 0. Never throws or allocates. +/// +/// # Safety +/// As [`js_perry_payload_link_ref`]; `out` is writable. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_link_event_owner(link: usize, out: *mut f64) -> i32 { + if link == 0 || out.is_null() { + return 0; + } + match np::link_event_owner(np::OwnerLink(link)) { + Some(owner) => { + // GC_STORE_AUDIT(POINTER_FREE): the caller's out-param on its own + // stack, which the conservative scan covers until it roots it. + std::ptr::write(out, owner); + 1 + } + None => 0, + } +} + +/// The instance's hidden JS-state object (`native_payload::js_state`). +/// +/// # Safety +/// `family` is a static descriptor. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_js_state( + value: f64, + family_ptr: *const PerryPayloadFamily, + create: i32, +) -> f64 { + let Some(family) = family(family_ptr) else { + return undefined(); + }; + np::js_state_for_class(value, family.parts.class_id, create != 0) +} + +/// Re-state the native bytes a live payload retains. +/// +/// # Safety +/// `family` is a static descriptor. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_set_external_bytes( + value: f64, + family_ptr: *const PerryPayloadFamily, + bytes: usize, +) { + if let Some(family) = family(family_ptr) { + np::set_external_bytes_class(value, family.parts.class_id, bytes); + } +} + +/// # Safety +/// `ptr`/`len` describe readable UTF-8 bytes. +unsafe fn name_arg<'a>(ptr: *const u8, len: usize) -> Option<&'a str> { + if ptr.is_null() || len == 0 { + return None; + } + std::str::from_utf8(std::slice::from_raw_parts(ptr, len)).ok() +} + +/// `PayloadPrototype::method` for an installer called through the C ABI. +/// +/// # Safety +/// `proto` is the builder handed to the installer; `info` is a static +/// function info; `name`/`len` as [`name_arg`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_proto_method( + proto: *mut c_void, + name: *const u8, + len: usize, + info: *const crate::closure::JsFunctionInfo, + arity: u32, +) { + if let (Some(builder), Some(name)) = ( + (proto as *mut np::PayloadPrototype).as_mut(), + name_arg(name, len), + ) { + builder.method(name, info, arity); + } +} + +/// `PayloadPrototype::getter` for an installer called through the C ABI. +/// +/// # Safety +/// As [`js_perry_payload_proto_method`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_proto_getter( + proto: *mut c_void, + name: *const u8, + len: usize, + info: *const crate::closure::JsFunctionInfo, +) { + if let (Some(builder), Some(name)) = ( + (proto as *mut np::PayloadPrototype).as_mut(), + name_arg(name, len), + ) { + builder.getter(name, info); + } +} + +/// `PayloadPrototype::data` for an installer called through the C ABI. +/// `flags` bit 0 writable, bit 1 enumerable, bit 2 configurable. +/// +/// # Safety +/// As [`js_perry_payload_proto_method`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_proto_data( + proto: *mut c_void, + name: *const u8, + len: usize, + value: f64, + flags: u32, +) { + if let (Some(builder), Some(name)) = ( + (proto as *mut np::PayloadPrototype).as_mut(), + name_arg(name, len), + ) { + builder.data(name, value, flags & 1 != 0, flags & 2 != 0, flags & 4 != 0); + } +} + +/// `PayloadPrototype::inherit` for an installer called through the C ABI. +/// +/// # Safety +/// As [`js_perry_payload_proto_method`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_payload_proto_inherit(proto: *mut c_void, parent: f64) { + if let Some(builder) = (proto as *mut np::PayloadPrototype).as_mut() { + builder.inherit(parent); + } +} diff --git a/crates/perry-runtime/src/turnloop_net/abi.rs b/crates/perry-runtime/src/turnloop_net/abi.rs index 02b764aba1..b4b6d0cd1a 100644 --- a/crates/perry-runtime/src/turnloop_net/abi.rs +++ b/crates/perry-runtime/src/turnloop_net/abi.rs @@ -12,6 +12,7 @@ //! the P1 coexistence contract: a worker agent has no loop until P3/P4, so its //! binding keeps the tokio transport, and the two paths never share a socket. +use std::ffi::c_void; use std::net::SocketAddr; use std::path::PathBuf; @@ -94,9 +95,14 @@ unsafe fn str_arg<'a>(ptr: *const u8, len: usize) -> &'a str { /// Revision of this ABI. Bumped whenever a signature or a struct field /// changes; a binding compiled against a different revision is refused rather /// than allowed to misread a completion. -pub const PERRY_NET_ABI_VERSION: u8 = 2; +/// +/// 3: link routes (NET-TRANSPORT-DESIGN P0): `NetCompletion::flags`, the +/// `js_perry_net_link_*` family and the opaque `TransportCore` block. +pub const PERRY_NET_ABI_VERSION: u8 = 3; -/// A digest of [`NetCompletion`]'s layout plus [`PERRY_NET_ABI_VERSION`]. +/// A digest of [`NetCompletion`]'s layout, the opaque `TransportCore` block +/// a binding reserves ([`super::TRANSPORT_CORE_WORDS`]) and +/// [`PERRY_NET_ABI_VERSION`]. /// /// A binding declares its own `#[repr(C)]` copy of the completion struct — it /// has no Cargo edge to this crate — so the two definitions can drift apart @@ -106,13 +112,23 @@ pub const PERRY_NET_ABI_VERSION: u8 = 2; /// registration instead of a corrupt read. #[no_mangle] pub extern "C" fn js_perry_net_abi_layout() -> u64 { + net_abi_layout(super::TRANSPORT_CORE_WORDS) +} + +/// The digest for a given core block size; perry-ffi computes the same +/// expression from its own struct and its own block constant. Fields, in bit +/// ranges: completion size 12 | `id` 6 | `data` 7 | `code` 7 | `syscall` 7 | +/// completion align 4 | core words 9 | core align 4 | version 8. +pub(crate) const fn net_abi_layout(core_words: usize) -> u64 { use std::mem::{align_of, offset_of, size_of}; - (size_of::() as u64) << 48 - | (offset_of!(NetCompletion, id) as u64) << 40 - | (offset_of!(NetCompletion, data) as u64) << 32 - | (offset_of!(NetCompletion, code) as u64) << 24 - | (offset_of!(NetCompletion, syscall) as u64) << 16 - | (align_of::() as u64) << 8 + (size_of::() as u64 & 0xFFF) << 52 + | (offset_of!(NetCompletion, id) as u64 & 0x3F) << 46 + | (offset_of!(NetCompletion, data) as u64 & 0x7F) << 39 + | (offset_of!(NetCompletion, code) as u64 & 0x7F) << 32 + | (offset_of!(NetCompletion, syscall) as u64 & 0x7F) << 25 + | (align_of::() as u64 & 0xF) << 21 + | (core_words as u64 & 0x1FF) << 12 + | (align_of::() as u64 & 0xF) << 8 | PERRY_NET_ABI_VERSION as u64 } @@ -679,3 +695,477 @@ pub unsafe extern "C" fn js_perry_net_completion_bytes( } c.data } + +// ── Link routes (NET-TRANSPORT-DESIGN P0) ─────────────────────────────────── +// +// Every call takes the payload's `TransportCore` (offset 0 of the binding's +// `TransportPayload`) and its owner link (`js_perry_payload_owner_link`) +// instead of an id. The runtime checks that the link's payload is OPEN on this +// thread and that `core` is its first field before touching either. + +use super::transport::{self, TransportCore}; +use crate::native_payload::OwnerLink; + +/// Install a link-routed binding's completion sink (no id allocator: the sink +/// installs accepted connections itself). Returns nonzero on success. +#[no_mangle] +pub extern "C" fn js_perry_net_register_link_sink(subsystem: i32, sink: SinkFn) -> i32 { + if subsystem < 0 || subsystem as usize >= super::MAX_SUBSYSTEMS { + return 0; + } + i32::from(super::register_link_sink(subsystem as u8, sink)) +} + +/// Construct an idle core for link route `route` in the binding's opaque +/// block. Returns [`PERRY_NET_ERR`] (and writes nothing) for a route outside +/// the sink table. +/// +/// # Safety +/// `core` is writable for `TRANSPORT_CORE_WORDS` words and 8-aligned. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_core_init(core: *mut c_void, route: i32) -> i32 { + if core.is_null() || route < 0 || route as usize >= super::MAX_SUBSYSTEMS { + return PERRY_NET_ERR; + } + // SAFETY: forwarded contract. + unsafe { transport::core_init(core.cast::(), route as u8) }; + PERRY_NET_OK +} + +/// Drop a core in place. Frees memory only (it runs in a collection, at +/// release or at thread teardown): it never reaches the loop. +/// +/// # Safety +/// `core` was initialized by [`js_perry_net_core_init`] and is dead after. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_core_drop(core: *mut c_void) { + if !core.is_null() { + // SAFETY: forwarded contract. + unsafe { transport::core_drop(core.cast::()) }; + } +} + +fn link_arg(link: usize) -> OwnerLink { + OwnerLink(link) +} + +/// Link form of [`js_perry_net_tcp_listen`]. +/// +/// # Safety +/// `core`/`link` name one OPEN payload of this thread; `host` as there. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_tcp_listen( + core: *mut c_void, + link: usize, + host: *const u8, + host_len: usize, + port: u16, + backlog: u32, + reuse_port: i32, + nodelay: i32, + err: *mut PerryNetError, +) -> i32 { + // SAFETY: forwarded contract from this function's own safety note. + let host = unsafe { str_arg(host, host_len) }; + let host = if host.is_empty() { "0.0.0.0" } else { host }; + let Ok(addr) = parse_bind_addr(host, port) else { + PerryNetError::write( + err, + NodeError { + code: "EINVAL", + errno: 0, + syscall: "listen", + }, + ); + return PERRY_NET_ERR; + }; + // SAFETY: forwarded contract. + let result = unsafe { + transport::tcp_listen( + core.cast(), + link_arg(link), + addr, + backlog, + reuse_port != 0, + nodelay != 0, + ) + }; + finish(result.map(|_| ()), err) +} + +/// Link form of [`js_perry_net_pipe_listen`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_pipe_listen( + core: *mut c_void, + link: usize, + path: *const u8, + path_len: usize, + backlog: u32, + err: *mut PerryNetError, +) -> i32 { + // SAFETY: forwarded contract. + let path = unsafe { str_arg(path, path_len) }; + finish( + unsafe { + transport::pipe_listen(core.cast(), link_arg(link), &PathBuf::from(path), backlog) + }, + err, + ) +} + +/// Link form of [`js_perry_net_accept_start`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_accept_start( + core: *mut c_void, + link: usize, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::accept_start(core.cast(), link_arg(link)) }, + err, + ) +} + +/// Install the connection of the accept completion the sink is handling into +/// a fresh payload's core (+1 ref on that payload's cell). A connection the +/// sink does not install is closed by the runtime when the sink returns. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]; `completion` is the sink's argument +/// and the sink call is still on the stack. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_install_accepted( + core: *mut c_void, + link: usize, + completion: *const NetCompletion, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::install_accepted(core.cast(), link_arg(link), completion) }, + err, + ) +} + +/// Link form of [`js_perry_net_tcp_connect`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_tcp_connect( + core: *mut c_void, + link: usize, + host: *const u8, + host_len: usize, + port: u16, + nodelay: i32, + err: *mut PerryNetError, +) -> i32 { + // SAFETY: forwarded contract. + let host = unsafe { str_arg(host, host_len) }; + let host = if host.is_empty() { "127.0.0.1" } else { host }; + finish( + unsafe { transport::tcp_connect(core.cast(), link_arg(link), host, port, nodelay != 0) }, + err, + ) +} + +/// Link form of [`js_perry_net_pipe_connect`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_pipe_connect( + core: *mut c_void, + link: usize, + path: *const u8, + path_len: usize, + err: *mut PerryNetError, +) -> i32 { + // SAFETY: forwarded contract. + let path = unsafe { str_arg(path, path_len) }; + finish( + unsafe { transport::pipe_connect(core.cast(), link_arg(link), &PathBuf::from(path)) }, + err, + ) +} + +/// Link form of [`js_perry_net_adopt_stream`]: `socket` is consumed on every +/// outcome. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`] and [`js_perry_net_adopt_stream`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_adopt_stream( + core: *mut c_void, + link: usize, + socket: i64, + err: *mut PerryNetError, +) -> i32 { + if socket < 0 { + return finish( + Err(super::map_error( + turnloop::Error::new(turnloop::ErrorKind::InvalidInput), + "adopt", + )), + err, + ); + } + #[cfg(unix)] + { + use std::os::fd::FromRawFd; + // SAFETY: the caller transfers ownership of an open descriptor. + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(socket as i32) }; + finish( + unsafe { transport::adopt_stream(core.cast(), link_arg(link), fd) }, + err, + ) + } + #[cfg(windows)] + { + use std::os::windows::io::FromRawSocket; + // SAFETY: the caller transfers ownership of an open socket. + let sock = unsafe { std::os::windows::io::OwnedSocket::from_raw_socket(socket as u64) }; + finish( + unsafe { transport::adopt_stream(core.cast(), link_arg(link), sock) }, + err, + ) + } + #[cfg(not(any(unix, windows)))] + { + let _ = (core, link); + finish( + Err(super::map_error( + turnloop::Error::new(turnloop::ErrorKind::Unsupported), + "adopt", + )), + err, + ) + } +} + +/// Link form of [`js_perry_net_read_start`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_read_start( + core: *mut c_void, + link: usize, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::read_start(core.cast(), link_arg(link)) }, + err, + ) +} + +/// Link form of [`js_perry_net_write`] (the bytes are copied). +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`] and [`js_perry_net_write`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_write( + core: *mut c_void, + link: usize, + bytes: *const u8, + len: usize, + user: u64, + out_queued: *mut usize, + err: *mut PerryNetError, +) -> i32 { + let owned = if bytes.is_null() || len == 0 { + Vec::new() + } else { + // SAFETY: the caller promises a readable range for `len` bytes. + unsafe { std::slice::from_raw_parts(bytes, len) }.to_vec() + }; + match unsafe { transport::write(core.cast(), link_arg(link), owned, user) } { + Ok(queued) => { + if !out_queued.is_null() { + // SAFETY: the caller supplies a writable `usize`. + // GC_STORE_AUDIT(POINTER_FREE): a queued-byte count into a caller out-param. + unsafe { std::ptr::write(out_queued, queued) }; + } + PERRY_NET_OK + } + Err(e) => finish(Err(e), err), + } +} + +/// Link form of [`js_perry_net_shutdown`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_shutdown( + core: *mut c_void, + link: usize, + user: u64, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::shutdown(core.cast(), link_arg(link), user) }, + err, + ) +} + +/// Release the core's driver resources (the handle moves into the driver's +/// close). `*out_closed` is 1 when a `NET_CLOSED` completion for this link +/// will follow, 0 when there was no handle. The binding then releases the +/// payload. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]; `out_closed` null or writable. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_close( + core: *mut c_void, + link: usize, + out_closed: *mut i32, + err: *mut PerryNetError, +) -> i32 { + match unsafe { transport::close(core.cast(), link_arg(link)) } { + Ok(closed) => { + if !out_closed.is_null() { + // SAFETY: caller-supplied writable `i32`. + // GC_STORE_AUDIT(POINTER_FREE): a flag into a caller out-param. + unsafe { std::ptr::write(out_closed, i32::from(closed)) }; + } + PERRY_NET_OK + } + Err(e) => finish(Err(e), err), + } +} + +/// Link form of [`js_perry_net_set_ref`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_set_ref( + core: *mut c_void, + link: usize, + referenced: i32, +) -> i32 { + match unsafe { transport::set_ref(core.cast(), link_arg(link), referenced != 0) } { + Ok(()) => PERRY_NET_OK, + Err(_) => PERRY_NET_ERR, + } +} + +/// Link form of [`js_perry_net_queued_bytes`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_queued_bytes(core: *mut c_void, link: usize) -> usize { + unsafe { transport::queued_bytes(core.cast(), link_arg(link)) } +} + +/// Arm or move the socket's deadline (link form of [`js_perry_net_timer_arm`]). +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_deadline_arm( + core: *mut c_void, + link: usize, + delay_ms: u64, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::deadline_arm(core.cast(), link_arg(link), delay_ms) }, + err, + ) +} + +/// Park the socket's deadline (link form of [`js_perry_net_timer_park`]). +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_deadline_park( + core: *mut c_void, + link: usize, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::deadline_park(core.cast(), link_arg(link)) }, + err, + ) +} + +/// Cancel the socket's deadline (link form of [`js_perry_net_timer_cancel`]). +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_deadline_cancel( + core: *mut c_void, + link: usize, + err: *mut PerryNetError, +) -> i32 { + finish( + unsafe { transport::deadline_cancel(core.cast(), link_arg(link)) }, + err, + ) +} + +/// Link form of [`js_perry_net_local_address`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`] and [`write_addr`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_local_address( + core: *mut c_void, + link: usize, + out: *mut u8, + cap: usize, + out_len: *mut usize, + out_port: *mut u16, + out_family: *mut i32, +) -> i32 { + // SAFETY: forwarded contract. + unsafe { + write_addr( + transport::local_addr(core.cast(), link_arg(link)), + out, + cap, + out_len, + out_port, + out_family, + ) + } +} + +/// Link form of [`js_perry_net_peer_address`]. +/// +/// # Safety +/// As [`js_perry_net_link_tcp_listen`] and [`write_addr`]. +#[no_mangle] +pub unsafe extern "C" fn js_perry_net_link_peer_address( + core: *mut c_void, + link: usize, + out: *mut u8, + cap: usize, + out_len: *mut usize, + out_port: *mut u16, + out_family: *mut i32, +) -> i32 { + // SAFETY: forwarded contract. + unsafe { + write_addr( + transport::peer_addr(core.cast(), link_arg(link)), + out, + cap, + out_len, + out_port, + out_family, + ) + } +} diff --git a/crates/perry-runtime/src/turnloop_net/mod.rs b/crates/perry-runtime/src/turnloop_net/mod.rs index b8ba084d23..5f8c0ba999 100644 --- a/crates/perry-runtime/src/turnloop_net/mod.rs +++ b/crates/perry-runtime/src/turnloop_net/mod.rs @@ -65,6 +65,7 @@ pub mod abi; pub mod census; pub(crate) mod errors; mod sink; +pub mod transport; #[cfg(windows)] mod windows_pipe; mod write_queue; @@ -74,14 +75,17 @@ mod write_queue; mod tests; pub use errors::{map_error, NodeError}; -pub use sink::{register_sink, sink_installed, NetCompletion, SinkFn, MAX_SUBSYSTEMS}; +pub use sink::{ + register_link_sink, register_sink, sink_installed, NetCompletion, SinkFn, MAX_SUBSYSTEMS, +}; +pub use transport::{TransportCore, TransportPayload, TRANSPORT_CORE_WORDS}; // P6: the completion kinds, for an in-tree subsystem. A separately linked // binding reads them through `perry_ffi::turnloop_net`, which declares its // own copy; perry-stdlib has a Cargo edge to this crate and must not need a // second declaration to keep in step with. pub use sink::{ - NET_ACCEPT, NET_CLOSED, NET_CONNECT, NET_DATA, NET_EOF, NET_ERROR, NET_SHUTDOWN, NET_TIMER, - NET_WROTE, + NET_ACCEPT, NET_CLOSED, NET_CONNECT, NET_DATA, NET_EOF, NET_ERROR, NET_FLAG_LINK, NET_SHUTDOWN, + NET_TIMER, NET_WROTE, }; // ── Operation classes, carried in the top 8 bits of every submission token ── @@ -112,6 +116,26 @@ fn token_parts(t: Token) -> (u64, i64) { ((t.0 >> ID_BITS), (t.0 & ID_MASK) as i64) } +/// How a socket's submissions are named in their tokens: by the binding's id +/// (the id route, phases P1-P7), or by its payload cell (a link route, +/// NET-TRANSPORT-DESIGN). The write queue and the Windows pipe helpers serve +/// both, so they take this rather than an id. +#[derive(Clone, Copy, Debug)] +pub(super) enum Name { + Id(i64), + Link { route: u8, cell: usize }, +} + +impl Name { + #[inline] + fn token(self, op: u64) -> Token { + match self { + Name::Id(id) => token(op, id), + Name::Link { route, cell } => transport::link_token(route, op, cell), + } + } +} + /// The `syscall` string Node reports for a failure of each operation class. fn syscall_for(op: u64) -> &'static str { match op { @@ -279,6 +303,11 @@ struct ConnectPlan { retrying: bool, /// The most recent failure, reported if the list runs out. last_error: Option, + /// The pending resolve of a link-routed plan: its incarnation. A resolve + /// completion whose op is not this one belongs to an earlier connect of + /// the same socket and must not start an attempt. Always `None` on the id + /// route, whose plan map already forgets a superseded plan. + op: Option, } /// A subsystem-owned one-shot deadline (P5). @@ -323,7 +352,7 @@ pub fn live_handles() -> usize { NET.with(|net| { let net = net.borrow(); net.entries.len() + net.plans.len() + net.timers.len() - }) + }) + transport::outstanding() } /// Whether this thread can take the turnloop net path at all. @@ -355,6 +384,8 @@ fn not_found(syscall: &'static str) -> NodeError { /// Run `f` against this agent's driver, creating a net-sized loop first. fn with_driver(f: impl FnOnce(&mut turnloop::Loop) -> R) -> Option { + #[cfg(test)] + transport::assert_driver_access_allowed(); crate::event_pump::with_net_driver(f) } @@ -549,6 +580,7 @@ pub fn tcp_connect_host( remaining: std::collections::VecDeque::new(), retrying: false, last_error: None, + op: None, }, ) }); @@ -819,7 +851,7 @@ pub fn close(id: i64) -> NetResult<()> { entry.closing = true; #[cfg(windows)] { - windows_pipe::cancel_eof(driver, id, entry); + windows_pipe::cancel_eof(driver, Name::Id(id), entry); if let Some(op) = entry.pipe_drain.take() { driver.cancel(op); } @@ -890,6 +922,12 @@ pub fn is_live(id: i64) -> bool { /// never calls host code), so a sink is free to run JS, allocate, collect, and /// submit new operations on the same loop. pub(crate) fn dispatch(completion: Completion) { + // A link token names its payload cell, not an id: route it before the + // id lookups below (the transitional fork; phase D deletes the id route). + if transport::owns(completion.token) { + transport::dispatch(completion); + return; + } let (op_class, id) = token_parts(completion.token); census::note_completion(op_class); let Completion { @@ -979,7 +1017,7 @@ pub(crate) fn dispatch(completion: Completion) { .entries .get_mut(&id) .ok_or_else(|| Error::new(ErrorKind::InvalidInput))?; - windows_pipe::arm_eof(driver, id, entry) + windows_pipe::arm_eof(driver, Name::Id(id), entry).map(|_| ()) }) }); if !matches!(armed, Some(Ok(()))) { @@ -1050,7 +1088,7 @@ pub(crate) fn dispatch(completion: Completion) { NET.with(|net| { if let Some(entry) = net.borrow_mut().entries.get_mut(&id) { if entry.pipe_eof_timer.is_some() { - let _ = windows_pipe::arm_eof(driver, id, entry); + let _ = windows_pipe::arm_eof(driver, Name::Id(id), entry); } } }) @@ -1067,7 +1105,7 @@ pub(crate) fn dispatch(completion: Completion) { with_driver(|driver| { NET.with(|net| { if let Some(entry) = net.borrow_mut().entries.get_mut(&id) { - windows_pipe::cancel_eof(driver, id, entry); + windows_pipe::cancel_eof(driver, Name::Id(id), entry); } }) }); @@ -1305,6 +1343,7 @@ fn accept_connection(subsystem: u8, server: i64, conn: Handle, peer: Option = NET.with(|net| net.borrow().entries.keys().copied().collect()); for id in ids { let _ = close(id); diff --git a/crates/perry-runtime/src/turnloop_net/sink.rs b/crates/perry-runtime/src/turnloop_net/sink.rs index 53c1e7b575..f1d5eb5fed 100644 --- a/crates/perry-runtime/src/turnloop_net/sink.rs +++ b/crates/perry-runtime/src/turnloop_net/sink.rs @@ -20,7 +20,7 @@ //! allocate JS values. use std::net::SocketAddr; -use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use super::NodeError; @@ -84,6 +84,9 @@ pub const NET_SHUTDOWN: i32 = 6; pub const NET_CLOSED: i32 = 7; /// An operation failed; `code`/`errno`/`syscall` carry Node's triple. pub const NET_ERROR: i32 = 8; +/// `NetCompletion::flags`: the completion comes from a link route, so `id` is +/// the payload's owner link and an accept's `conn` is its install slot. +pub const NET_FLAG_LINK: i32 = 1; /// A subsystem-owned deadline expired (P5). `id` names the deadline, which is /// the caller's own id — a connection's, not a socket handle's. pub const NET_TIMER: i32 = 9; @@ -107,12 +110,17 @@ pub struct NetCompletion { /// terminal, and Node does not tear the server down for one — the tokio /// accept loop deliberately did not break on an accept error either. pub terminal: i32, - /// Padding, so the struct's layout is identical on both sides of the ABI - /// without depending on how the compiler packs two trailing i32s. - pub _reserved: i32, - /// The Perry-side id of the socket or listener this concerns. + /// [`NET_FLAG_LINK`] when this completion comes from a link route; zero + /// otherwise. Also keeps the layout independent of how a compiler packs + /// two trailing i32s. + pub flags: i32, + /// The Perry-side id of the socket or listener this concerns. On a link + /// route ([`NET_FLAG_LINK`]): the payload's owner link (its cell address). pub id: i64, - /// For [`NET_ACCEPT`], the newly allocated connection id; else zero. + /// For [`NET_ACCEPT`], the newly allocated connection id; else zero. On a + /// link route: the accepted connection's slot, which the sink hands to + /// `install_accepted` while this call is on the stack (or leaves alone, + /// and the runtime closes the connection). pub conn: i64, /// The caller's write/end completion token, echoed back; zero if none. pub user: u64, @@ -138,7 +146,7 @@ impl NetCompletion { kind, errno: 0, terminal: 0, - _reserved: 0, + flags: 0, id, conn: 0, user: 0, @@ -152,6 +160,12 @@ impl NetCompletion { } } + /// Mark a completion as coming from a link route. + pub(super) fn linked(mut self) -> Self { + self.flags |= NET_FLAG_LINK; + self + } + pub(super) fn connect(id: i64) -> Self { Self::blank(NET_CONNECT, id) } @@ -258,6 +272,10 @@ static SINKS: [AtomicPtr<()>; MAX_SUBSYSTEMS] = [const { AtomicPtr::new(std::ptr::null_mut()) }; MAX_SUBSYSTEMS]; static ALLOCS: [AtomicPtr<()>; MAX_SUBSYSTEMS] = [const { AtomicPtr::new(std::ptr::null_mut()) }; MAX_SUBSYSTEMS]; +/// Whether each slot's sink declared `links`: its sockets are transport cores +/// in payloads, named in tokens by their cell (`transport.rs`), and its +/// completions carry that cell in `id`. Set once, at registration. +static LINKS: [AtomicBool; MAX_SUBSYSTEMS] = [const { AtomicBool::new(false) }; MAX_SUBSYSTEMS]; /// Install a binding's completion sink and accepted-connection id allocator. /// @@ -281,7 +299,7 @@ pub fn register_sink(subsystem: u8, sink: SinkFn, alloc: AllocFn) -> bool { // Re-registering the same sink stays idempotent, which is the documented // contract and what a binding whose module is initialised twice relies on. let held = SINKS[slot].load(Ordering::Acquire); - if !held.is_null() && held != sink as *mut () { + if !held.is_null() && (held != sink as *mut () || LINKS[slot].load(Ordering::Acquire)) { return false; } // Publish the allocator first: an accept completion needs it, and a sink @@ -291,6 +309,34 @@ pub fn register_sink(subsystem: u8, sink: SinkFn, alloc: AllocFn) -> bool { true } +/// Install a link-routed binding's completion sink (NET-TRANSPORT-DESIGN P0). +/// +/// The binding declares `links`: its sockets are [`super::TransportCore`]s +/// inside native payloads, every completion's `id` is the payload's owner link +/// (the cell address), and an accepted connection is installed by the sink +/// into a payload it allocates (`transport::install_accepted`), so there is no +/// id allocator. Refused, like [`register_sink`], for a slot another sink +/// holds, and for a slot already registered without `links`. +pub fn register_link_sink(subsystem: u8, sink: SinkFn) -> bool { + let slot = subsystem as usize; + if slot >= MAX_SUBSYSTEMS { + return false; + } + let held = SINKS[slot].load(Ordering::Acquire); + if !held.is_null() && (held != sink as *mut () || !LINKS[slot].load(Ordering::Acquire)) { + return false; + } + LINKS[slot].store(true, Ordering::Release); + SINKS[slot].store(sink as *mut (), Ordering::Release); + true +} + +/// Whether `subsystem`'s sink was registered with `links`. +#[inline] +pub(super) fn is_link_route(subsystem: u8) -> bool { + (subsystem as usize) < MAX_SUBSYSTEMS && LINKS[subsystem as usize].load(Ordering::Acquire) +} + pub(super) fn emit(subsystem: u8, completion: NetCompletion) { let slot = subsystem as usize; if slot >= MAX_SUBSYSTEMS { diff --git a/crates/perry-runtime/src/turnloop_net/tests.rs b/crates/perry-runtime/src/turnloop_net/tests.rs index 2417b78138..8cbffffe2a 100644 --- a/crates/perry-runtime/src/turnloop_net/tests.rs +++ b/crates/perry-runtime/src/turnloop_net/tests.rs @@ -923,6 +923,7 @@ fn writes_between_connect_attempts_reach_the_attempt_that_succeeds() { remaining: [dead, live].into_iter().collect(), retrying: false, last_error: None, + op: None, }, ) }); diff --git a/crates/perry-runtime/src/turnloop_net/transport.rs b/crates/perry-runtime/src/turnloop_net/transport.rs new file mode 100644 index 0000000000..061122cc29 --- /dev/null +++ b/crates/perry-runtime/src/turnloop_net/transport.rs @@ -0,0 +1,1575 @@ +//! Transport cores: turnloop sockets owned by native payloads +//! (NET-TRANSPORT-DESIGN P0, #11919). +//! +//! A link-routed socket keeps no entry in this module's id maps. Its state is a +//! [`TransportCore`] at offset 0 of the family's payload ([`TransportPayload`]), +//! and every submission's token names the payload *cell*, so a completion +//! finds its socket with no lookup: +//! +//! ```text +//! token = 0b11 (link tag) | route:4 | op:4 | (cell >> 3):54 +//! dispatch: link = cell; owner = link_event_owner(link)?; core = link_open_payload(link) +//! ``` +//! +//! The cell is malloc'd, never moves and outlives its payload, so the token +//! stays valid across every collection. Three rules make that safe: +//! +//! 1. **The token is the route.** It carries the cell, the operation class and +//! the sink route; no map is consulted. +//! 2. **A live driver resource pins its cell.** Each terminal completion the +//! driver still owes holds one `link_ref` from submission until it is +//! dispatched: a socket or listener handle's `Closed`, a resolve's result, a +//! deadline timer handle's `Closed`. Reads, accepts and writes need no ref: +//! they finish before their handle's `Closed` (DESIGN D4). So the sweep never +//! finalizes a payload that holds a driver handle, `Drop for T` only frees +//! memory, and no completion can reach a freed or reused cell. +//! 3. **Release moves the handle into the close.** [`close`] takes the handle +//! out of the core and submits the driver's close under the cell's token; +//! the binding then releases the payload (`native_payload::close`). The +//! driver owns the descriptor until its `Closed`, which is dispatched +//! through `link_event_owner` (it answers for a CLOSED cell) and unrefs. +//! +//! **Incarnation** is the driver's generational handle: a completion mutates +//! the core only when `core.entry.handle == completion.handle` (the plan's +//! resolve op for a resolve, `core.deadline` for a timer). A stale `Closed` +//! still delivers the owner-level `close` and its unref; any other stale +//! result is dropped, and a stale accept closes the new connection. +//! +//! The id route (`super::dispatch`'s maps) stays as it is until each binding +//! converts; [`owns`] is the transitional fork. + +use std::cell::Cell; +use std::ffi::c_void; +use std::net::SocketAddr; +use std::path::Path; + +use super::*; +use crate::native_payload::{self as np, OwnerLink}; + +/// Bits 63..62 set: a link token. The id route's op classes are 1..=10, the +/// process, pool and posted-job spaces sit at 0x10..=0x3F in the top byte and +/// the JS timer is `u64::MAX`, which no link token equals (its cell bits would +/// name an address at 2^57 - 8, above `LINK_ADDRESS_LIMIT`). +const LINK_TAG: u64 = 0b11 << 62; +const ROUTE_SHIFT: u32 = 58; +const OP_SHIFT: u32 = 54; +const CELL_MASK: u64 = (1 << OP_SHIFT) - 1; +/// A link token naming no cell: for a close whose `Closed` nobody waits on (a +/// refused accepted connection, an expired timer's handle has its own token). +pub(super) const LINK_NULL: Token = Token(LINK_TAG); + +/// Words of the opaque block perry-ffi reserves for a [`TransportCore`] +/// (`perry_ffi::turnloop_net::TransportCore`). Folded into the ABI digest +/// (`js_perry_net_abi_layout`), so a binding built against another block size +/// is refused at registration; the assertion below keeps the core inside it. +#[cfg(not(windows))] +pub const TRANSPORT_CORE_WORDS: usize = 39; +/// Windows' `Entry` carries the pipe drain and EOF-timer fields (48 bytes) +/// and a wider `PathBuf` (8): 46 words by construction, reserved with a +/// two-word margin because no Windows build checked it exactly; the +/// assertion below fails that build if the estimate is short. +#[cfg(windows)] +pub const TRANSPORT_CORE_WORDS: usize = 48; + +const _: () = assert!( + std::mem::size_of::() <= TRANSPORT_CORE_WORDS * 8 + && std::mem::align_of::() <= 8, + "TransportCore outgrew the opaque block perry-ffi reserves: raise TRANSPORT_CORE_WORDS \ + here AND in perry-ffi (the ABI digest then refuses bindings built against the old size)" +); + +/// One socket or listener's transport state, owned by its payload. +/// +/// Plain Rust data: no JS value, no GC pointer, no thread-local, so dropping +/// it only frees memory (the payload rule). A driver `Handle` is a plain +/// generational id; while one is installed the cell is pinned (rule 2), so the +/// sweep never drops a core that names a live descriptor. Worker teardown may +/// drop one that still names a handle: by then `Loop::drop` released it. +pub struct TransportCore { + /// The installed handle and its stream or listener state. `None` while + /// idle, resolving, or after [`close`] took the handle. + entry: Option, + /// Writes and an `end()` waiting for a connect or for the write in flight. + backlog: Backlog, + /// A client connect still choosing an address; `plan.op` is the pending + /// resolve's incarnation. + plan: Option>, + /// The socket's one-shot deadline (server timeouts, `setTimeout`). + deadline: Option, + /// The sink this socket's completions go to. + route: u8, + /// Node's `ref()` state, applied to every handle this core installs. + referenced: bool, +} + +impl TransportCore { + /// An idle core whose completions go to the link sink `route`. + pub fn new(route: u8) -> Self { + Self { + entry: None, + backlog: Backlog::default(), + plan: None, + deadline: None, + route, + referenced: true, + } + } + + /// The handle this core holds now (its incarnation), if any. + pub fn handle(&self) -> Option { + self.entry.as_ref().map(|e| e.handle) + } + + /// The sink route. + pub fn route(&self) -> u8 { + self.route + } + + /// Move the socket to another link sink (an HTTP upgrade hands the + /// connection to `net`). Outstanding operations keep their tokens; every + /// completion is delivered to the route current at dispatch. + pub fn set_route(&mut self, route: u8) { + self.route = route; + } +} + +/// A family payload that owns a transport: the core first, so the runtime +/// reads it at offset 0 of the payload a link names. +#[repr(C)] +pub struct TransportPayload { + pub core: TransportCore, + pub ext: E, +} + +/// An accepted connection in flight to the sink (`NetCompletion::conn`). +pub(super) struct AcceptSlot { + conn: Option, + peer: Option, +} + +crate::perry_thread_local! { + /// Set by agent teardown: link completions are discarded, never + /// dereferenced (LIFECYCLE-DESIGN L8). The thread's loop is dropped next + /// and its heap teardown finalizes the cells. + static TEARDOWN: Cell = const { Cell::new(false) }; + /// Driver resources this thread's link routes still hold a ref for: + /// installed handles, pending resolves, deadline timers and pipe jobs. + static LINK_RESOURCES: Cell = const { Cell::new(0) }; +} + +/// Whether `t` is a link token (the transitional fork in `super::dispatch`). +#[inline] +pub(super) fn owns(t: Token) -> bool { + t.0 & LINK_TAG == LINK_TAG +} + +#[inline] +pub(super) fn link_token(route: u8, op: u64, cell: usize) -> Token { + debug_assert!(route < 16 && op < 16 && cell & 7 == 0); + debug_assert!((cell as u64) < np::LINK_ADDRESS_LIMIT); + Token(LINK_TAG | (route as u64) << ROUTE_SHIFT | op << OP_SHIFT | (cell as u64) >> 3) +} + +#[inline] +fn link_parts(t: Token) -> (u8, u64, usize) { + ( + ((t.0 >> ROUTE_SHIFT) & 0xF) as u8, + (t.0 >> OP_SHIFT) & 0xF, + ((t.0 & CELL_MASK) << 3) as usize, + ) +} + +/// Driver resources held by link routes on this thread (`live_handles`). +pub(super) fn outstanding() -> usize { + LINK_RESOURCES.with(Cell::get) +} + +pub(super) fn begin_teardown() { + TEARDOWN.with(|t| t.set(true)); +} + +#[cfg(test)] +pub(super) fn reset_for_test() { + TEARDOWN.with(|t| t.set(false)); + LINK_RESOURCES.with(|n| n.set(0)); +} + +// Test-only: a guard under which any driver access panics. A payload `Drop` +// runs inside a collection or at thread teardown and must never reach the +// loop (N7); the witness drops payloads under this guard. +#[cfg(test)] +crate::perry_thread_local! { + static DRIVER_FORBIDDEN: Cell = const { Cell::new(false) }; + /// Link completions the teardown rule discarded: the witness asserts its + /// subject ran. + static DISCARDED: Cell = const { Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn discarded_for_test() -> usize { + DISCARDED.with(Cell::get) +} + +#[cfg(test)] +pub(crate) fn begin_teardown_for_test() { + begin_teardown(); +} + +#[cfg(test)] +pub(super) fn assert_driver_access_allowed() { + assert!( + !DRIVER_FORBIDDEN.with(Cell::get), + "the turnloop driver was reached where it is forbidden (a payload Drop)" + ); +} + +#[cfg(test)] +pub(crate) struct ForbidDriver; + +#[cfg(test)] +impl ForbidDriver { + pub(crate) fn new() -> Self { + DRIVER_FORBIDDEN.with(|f| f.set(true)); + ForbidDriver + } +} + +#[cfg(test)] +impl Drop for ForbidDriver { + fn drop(&mut self) { + DRIVER_FORBIDDEN.with(|f| f.set(false)); + } +} + +/// Sabotage hooks for the P0 witnesses, compiled only into test binaries. +#[inline] +pub(super) fn sabotage(fault: &str) -> bool { + #[cfg(test)] + { + std::env::var("PERRY_TEST_NET_SABOTAGE").as_deref() == Ok(fault) + } + #[cfg(not(test))] + { + let _ = fault; + false + } +} + +/// Take one ref for a driver resource that will end in one terminal +/// completion on this cell. +unsafe fn hold(link: OwnerLink) { + if sabotage("skip_ref") { + return; + } + np::link_ref(link); + LINK_RESOURCES.with(|n| n.set(n.get() + 1)); +} + +/// Drop the ref of a resource whose terminal completion was just dispatched, +/// or that will never produce one. +unsafe fn release(link: OwnerLink) { + np::link_unref(link); + LINK_RESOURCES.with(|n| n.set(n.get().saturating_sub(1))); +} + +fn bad(syscall: &'static str) -> NodeError { + map_error(Error::new(ErrorKind::InvalidInput), syscall) +} + +/// Validate a submission's `(core, link)` pair: the link's payload is OPEN on +/// this thread and `core` is its first field, and the core's route is a link +/// sink. +/// +/// # Safety +/// `link` came from `owner_link` on this thread and its cell is alive (the +/// caller holds the owner). +unsafe fn bind<'a>( + core: *mut TransportCore, + link: OwnerLink, + syscall: &'static str, +) -> NetResult<(&'a mut TransportCore, Name)> { + if core.is_null() || link.0 == 0 || np::link_open_payload(link) != core.cast::() { + return Err(bad(syscall)); + } + let c = &mut *core; + if !sink::is_link_route(c.route) { + return Err(bad(syscall)); + } + let name = Name::Link { + route: c.route, + cell: link.0, + }; + Ok((c, name)) +} + +/// The OPEN core behind `link`, for dispatch. +/// +/// # Safety +/// The cell is alive (a ref is held for the completion being dispatched). +unsafe fn core_of<'a>(link: OwnerLink) -> Option<&'a mut TransportCore> { + let p = np::link_open_payload(link); + (!p.is_null()).then(|| &mut *(p as *mut TransportCore)) +} + +/// The OPEN core behind `link` when it still holds `handle`: the incarnation +/// that submitted the operation now completing. +unsafe fn current<'a>(link: OwnerLink, handle: Option) -> Option<&'a mut TransportCore> { + let c = core_of(link)?; + let matches = c.entry.as_ref().is_some_and(|e| Some(e.handle) == handle); + (matches || (c.entry.is_some() && sabotage("handle_check"))).then_some(c) +} + +/// Install a handle the driver just created for this core (+1 ref). +unsafe fn install( + driver: &mut turnloop::Loop, + c: &mut TransportCore, + link: OwnerLink, + mut entry: Entry, +) { + if !c.referenced { + let _ = driver.set_ref(entry.handle, false); + entry.referenced = false; + } + c.entry = Some(entry); + hold(link); +} + +fn idle_or(c: &TransportCore, syscall: &'static str) -> NetResult<()> { + if c.entry.is_some() || c.plan.is_some() { + return Err(bad(syscall)); + } + Ok(()) +} + +// ── Submission ────────────────────────────────────────────────────────────── + +/// Bind and listen on a TCP address; returns the actual local address. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn tcp_listen( + core: *mut TransportCore, + link: OwnerLink, + addr: SocketAddr, + backlog: u32, + reuse_port: bool, + nodelay: bool, +) -> NetResult { + let (c, _) = bind(core, link, "listen")?; + idle_or(c, "listen")?; + with_driver(|driver| { + let opts = listen_opts(backlog, reuse_port, nodelay); + let handle = driver + .tcp_listen(addr, &opts) + .map_err(|e| map_error(e, "listen"))?; + let local = driver.local_addr(handle).unwrap_or(addr); + let mut entry = Entry::new(handle, c.route, true); + entry.local = Some(local); + install(driver, c, link, entry); + Ok(local) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Bind and listen on a Unix-domain socket path or a Windows named pipe. +/// Unlinking a stale or closed socket file is the binding's job, done +/// synchronously at close (a deferred unlink would delete a reopened +/// listener's path). +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn pipe_listen( + core: *mut TransportCore, + link: OwnerLink, + path: &Path, + backlog: u32, +) -> NetResult<()> { + let (c, _) = bind(core, link, "listen")?; + idle_or(c, "listen")?; + with_driver(|driver| { + let opts = ListenOpts { + reuse_port: ReusePort::No, + backlog, + ..ListenOpts::default() + }; + let handle = driver + .pipe_listen(&PipeName(path.to_path_buf()), &opts) + .map_err(|e| map_error(e, "listen"))?; + let mut entry = Entry::new(handle, c.route, true); + entry.path = Some(path.to_path_buf()); + install(driver, c, link, entry); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Start the listener's multishot accept. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn accept_start(core: *mut TransportCore, link: OwnerLink) -> NetResult<()> { + let (c, name) = bind(core, link, "accept")?; + with_driver(|driver| { + let entry = c + .entry + .as_mut() + .filter(|e| e.listener && !e.closing) + .ok_or_else(|| not_found("accept"))?; + if entry.accept_op.is_some() { + return Ok(()); + } + let op = driver + .accept_start(entry.handle, name.token(OP_ACCEPT)) + .map_err(|e| map_error(e, "accept"))?; + entry.accept_op = Some(op); + census::note_submit(OP_ACCEPT); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Install the connection an accept completion carries into a fresh payload's +/// core. Valid only inside the sink call that received `completion`; a +/// connection the sink does not install is closed by the runtime after the +/// sink returns. +/// +/// # Safety +/// See [`bind`]; `completion` is the pointer the sink was called with. +pub unsafe fn install_accepted( + core: *mut TransportCore, + link: OwnerLink, + completion: *const NetCompletion, +) -> NetResult<()> { + let (c, _) = bind(core, link, "accept")?; + idle_or(c, "accept")?; + let Some(completion) = completion.as_ref() else { + return Err(bad("accept")); + }; + if completion.kind != sink::NET_ACCEPT + || completion.flags & sink::NET_FLAG_LINK == 0 + || completion.conn == 0 + { + return Err(bad("accept")); + } + let slot = &mut *(completion.conn as *mut AcceptSlot); + let Some(conn) = slot.conn.take() else { + return Err(bad("accept")); + }; + let peer = slot.peer; + with_driver(|driver| { + let mut entry = Entry::new(conn, c.route, false); + entry.local = driver.local_addr(conn).ok(); + entry.peer = peer; + install(driver, c, link, entry); + }) + .ok_or_else(no_loop) +} + +/// Connect a TCP client to `host:port`, resolving a hostname off the loop +/// thread first (a resolve holds its own ref until its result is dispatched). +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn tcp_connect( + core: *mut TransportCore, + link: OwnerLink, + host: &str, + port: u16, + nodelay: bool, +) -> NetResult<()> { + let (c, name) = bind(core, link, "connect")?; + idle_or(c, "connect")?; + if let Ok(ip) = host.parse::() { + return with_driver(|driver| { + connect_addr(driver, c, link, SocketAddr::new(ip, port), nodelay) + }) + .unwrap_or_else(|| Err(no_loop())); + } + with_driver(|driver| { + let request = turnloop::DnsRequest { + host: host.to_string(), + port, + }; + let op = driver + .resolve(request, name.token(OP_RESOLVE)) + .map_err(|e| map_error(e, "getaddrinfo"))?; + census::note_submit(OP_RESOLVE); + c.plan = Some(Box::new(ConnectPlan { + subsystem: c.route, + nodelay, + remaining: std::collections::VecDeque::new(), + retrying: false, + last_error: None, + op: Some(op), + })); + hold(link); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// One connect attempt: a fresh handle, installed (+1 ref). +unsafe fn connect_addr( + driver: &mut turnloop::Loop, + c: &mut TransportCore, + link: OwnerLink, + addr: SocketAddr, + nodelay: bool, +) -> NetResult<()> { + let name = Name::Link { + route: c.route, + cell: link.0, + }; + let handle = driver + .tcp_connect( + addr, + &TcpOpts { + nodelay, + ..TcpOpts::default() + }, + name.token(OP_CONNECT), + ) + .map_err(|e| map_error(e, "connect"))?; + let mut entry = Entry::new(handle, c.route, false); + entry.peer = Some(addr); + entry.connecting = true; + install(driver, c, link, entry); + Ok(()) +} + +/// Connect to a Unix-domain socket or a Windows named pipe. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn pipe_connect( + core: *mut TransportCore, + link: OwnerLink, + path: &Path, +) -> NetResult<()> { + let (c, name) = bind(core, link, "connect")?; + idle_or(c, "connect")?; + with_driver(|driver| { + let handle = driver + .pipe_connect(&PipeName(path.to_path_buf()), name.token(OP_CONNECT)) + .map_err(|e| map_error(e, "connect"))?; + let mut entry = Entry::new(handle, c.route, false); + entry.path = Some(path.to_path_buf()); + entry.connecting = true; + install(driver, c, link, entry); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Adopt an already-connected stream socket. `socket` is consumed on every +/// outcome, as on the id route. +/// +/// # Safety +/// See [`bind`]. +#[cfg(any(unix, windows))] +pub unsafe fn adopt_stream( + core: *mut TransportCore, + link: OwnerLink, + socket: AdoptedSocket, +) -> NetResult<()> { + let (c, name) = bind(core, link, "adopt")?; + idle_or(c, "adopt")?; + let (local, peer) = { + let sock = socket2::SockRef::from(&socket); + ( + sock.local_addr().ok().and_then(|a| a.as_socket()), + sock.peer_addr().ok().and_then(|a| a.as_socket()), + ) + }; + with_driver(move |driver| { + #[cfg(unix)] + let detached = turnloop::Detached::from_fd(socket); + #[cfg(windows)] + let detached = turnloop::Detached::from_socket(socket); + let detached = detached.map_err(|e| map_error(e, "adopt"))?; + let handle = driver + .attach(detached, name.token(OP_READ)) + .map_err(|e| map_error(e, "adopt"))?; + let mut entry = Entry::new(handle, c.route, false); + entry.local = local; + entry.peer = peer; + install(driver, c, link, entry); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Start the multishot read. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn read_start(core: *mut TransportCore, link: OwnerLink) -> NetResult<()> { + let (c, name) = bind(core, link, "read")?; + with_driver(|driver| { + let entry = c.entry.as_mut().ok_or_else(|| not_found("read"))?; + if entry.read_op.is_some() || entry.closing { + return Ok(()); + } + let op = driver + .read_start(entry.handle, name.token(OP_READ)) + .map_err(|e| map_error(e, "read"))?; + entry.read_op = Some(op); + census::note_submit(OP_READ); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Queue `bytes`; returns the socket's total queued bytes (`writableLength`). +/// The same queue discipline as the id route (`write_queue`): one driver write +/// in flight, the rest coalesced in the backlog, writes before `Connected` +/// held for the attempt that succeeds. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn write( + core: *mut TransportCore, + link: OwnerLink, + bytes: Vec, + user: u64, +) -> NetResult { + let (c, name) = bind(core, link, "write")?; + with_driver(|driver| accept_write(driver, c, name, bytes, user)) + .unwrap_or_else(|| Err(no_loop())) +} + +/// `end()`: shut the write side down behind every write accepted before it. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn shutdown(core: *mut TransportCore, link: OwnerLink, user: u64) -> NetResult<()> { + let (c, name) = bind(core, link, "shutdown")?; + with_driver(|driver| accept_shutdown(driver, c, name, link, user)) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Release the core's driver resources (rule 3). Takes the handle out and +/// submits its close under the cell's token, closes the deadline, cancels a +/// pending resolve and drops the backlog. Returns whether a `Closed` (a +/// `NET_CLOSED` completion for this link) will follow; when it is `false` +/// there was no handle, and a binding that owes a `close` event emits it +/// itself. The binding releases the payload afterwards. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn close(core: *mut TransportCore, link: OwnerLink) -> NetResult { + let (c, name) = bind(core, link, "close")?; + with_driver(|driver| { + if let Some(timer) = c.deadline.take() { + if driver.close(timer, name.token(OP_TIMER)).is_err() { + release(link); + } + } + if let Some(plan) = c.plan.take() { + if let Some(op) = plan.op { + // Its terminal (`Cancelled`) result carries the resolve's ref. + driver.cancel(op); + } + } + c.backlog = Backlog::default(); + let Some(entry) = c.entry.take() else { + return Ok(false); + }; + #[cfg(windows)] + let mut entry = entry; + #[cfg(windows)] + { + windows_pipe::cancel_eof(driver, name, &mut entry); + if let Some(op) = entry.pipe_drain.take() { + driver.cancel(op); + } + } + if entry.closing { + // A failed connect attempt's close is already in flight; its + // `Closed` no longer matches the core and is delivered as this + // socket's close. + return Ok(true); + } + census::note_submit(OP_CLOSE); + match driver.close(entry.handle, name.token(OP_CLOSE)) { + Ok(()) => Ok(true), + Err(e) => { + // The handle is still live: keep it (and its ref) so no + // completion can outlive the cell. + c.entry = Some(entry); + Err(map_error(e, "close")) + } + } + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Node's `ref()`/`unref()`: remembered on the core, so a handle installed +/// later inherits it. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn set_ref( + core: *mut TransportCore, + link: OwnerLink, + referenced: bool, +) -> NetResult<()> { + let (c, _) = bind(core, link, "")?; + c.referenced = referenced; + let Some(entry) = c.entry.as_mut() else { + return Ok(()); + }; + if entry.referenced == referenced { + return Ok(()); + } + entry.referenced = referenced; + let handle = entry.handle; + with_driver(|driver| { + driver + .set_ref(handle, referenced) + .map_err(|e| map_error(e, "")) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Bytes accepted and not yet reported written. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn queued_bytes(core: *mut TransportCore, link: OwnerLink) -> usize { + bind(core, link, "").map_or(0, |(c, _)| total_queued(c)) +} + +/// The local endpoint (`server.address()`, `socket.localAddress`). +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn local_addr(core: *mut TransportCore, link: OwnerLink) -> Option { + let (c, _) = bind(core, link, "").ok()?; + c.entry.as_ref().and_then(|e| e.local) +} + +/// The peer endpoint (`socket.remoteAddress`). +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn peer_addr(core: *mut TransportCore, link: OwnerLink) -> Option { + let (c, _) = bind(core, link, "").ok()?; + c.entry.as_ref().and_then(|e| e.peer) +} + +/// Arm, or move, the socket's one-shot deadline `delay_ms` from now. Never +/// keeps the loop alive on its own. Its expiry is delivered as `NET_TIMER`. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn deadline_arm( + core: *mut TransportCore, + link: OwnerLink, + delay_ms: u64, +) -> NetResult<()> { + let (c, name) = bind(core, link, "timer")?; + with_driver(|driver| { + let at = driver.now() + std::time::Duration::from_millis(delay_ms); + if let Some(timer) = c.deadline { + if driver.timer_reset(timer, at) { + census::note_timer_reset(); + return Ok(()); + } + // Fired (its expiry not yet dispatched) or closing: replace it. + // Its `Closed` releases its ref; a pending expiry no longer + // matches `deadline` and is dropped. + c.deadline = None; + if driver.close(timer, name.token(OP_TIMER)).is_err() { + release(link); + } + } + let timer = driver + .timer(at, None, name.token(OP_TIMER)) + .map_err(|e| map_error(e, "timer"))?; + census::note_timer_create(); + let _ = driver.set_ref(timer, false); + c.deadline = Some(timer); + hold(link); + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Disarm the deadline but keep its handle (see the id route's `timer_park`). +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn deadline_park(core: *mut TransportCore, link: OwnerLink) -> NetResult<()> { + let (c, name) = bind(core, link, "timer")?; + let Some(timer) = c.deadline else { + return Ok(()); + }; + with_driver(|driver| { + if let Some(at) = driver.now().checked_add(PARK_AHEAD) { + if driver.timer_reset(timer, at) { + census::note_timer_park(); + return Ok(()); + } + } + c.deadline = None; + census::note_submit(OP_TIMER); + if driver.close(timer, name.token(OP_TIMER)).is_err() { + release(link); + } + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +/// Cancel the deadline. Idempotent. +/// +/// # Safety +/// See [`bind`]. +pub unsafe fn deadline_cancel(core: *mut TransportCore, link: OwnerLink) -> NetResult<()> { + let (c, name) = bind(core, link, "timer")?; + let Some(timer) = c.deadline.take() else { + return Ok(()); + }; + with_driver(|driver| { + census::note_submit(OP_TIMER); + if driver.close(timer, name.token(OP_TIMER)).is_err() { + release(link); + } + Ok(()) + }) + .unwrap_or_else(|| Err(no_loop())) +} + +// ── Write queue on a core (the id route's `write_queue`, per socket) ──────── + +fn total_queued(c: &TransportCore) -> usize { + c.entry.as_ref().map_or(0, |e| e.queued) + c.backlog.bytes.len() +} + +/// Caller writes a backlog flush could not hand to the driver. +struct Failure { + error: NodeError, + users: Vec, +} + +fn accept_write( + driver: &mut turnloop::Loop, + c: &mut TransportCore, + name: Name, + bytes: Vec, + user: u64, +) -> NetResult { + let retrying = c.plan.as_ref().is_some_and(|p| p.retrying); + match c.entry.as_mut() { + Some(entry) if !retrying => { + if entry.listener || entry.closing { + return Err(bad("write")); + } + let waiting = !c.backlog.is_idle(); + if !entry.connecting && !waiting && entry.inflight < MAX_INFLIGHT_WRITES { + let len = bytes.len(); + write_queue::submit_bytes(driver, name, entry, bytes) + .map_err(|e| map_error(e, "write"))?; + entry.writes.push_back(PendingWrite { + user, + len, + last: true, + }); + } else { + if c.backlog.shutdown.is_some() { + return Err(bad("write")); + } + c.backlog.push(bytes, user); + } + } + // Resolving, or between two attempts of a plan. + _ if c.plan.is_some() => { + if c.backlog.shutdown.is_some() { + return Err(bad("write")); + } + c.backlog.push(bytes, user); + } + _ => return Err(not_found("write")), + } + Ok(total_queued(c)) +} + +unsafe fn accept_shutdown( + driver: &mut turnloop::Loop, + c: &mut TransportCore, + name: Name, + link: OwnerLink, + user: u64, +) -> NetResult<()> { + let retrying = c.plan.as_ref().is_some_and(|p| p.retrying); + let deferred = match c.entry.as_ref() { + Some(_) if retrying => true, + Some(entry) if entry.listener || entry.closing => return Err(bad("shutdown")), + Some(entry) => entry.connecting, + None if c.plan.is_some() => true, + None => return Err(not_found("shutdown")), + }; + if c.backlog.shutdown.is_some() { + return Err(bad("shutdown")); + } + c.backlog.shutdown = Some(user); + if deferred { + return Ok(()); + } + match flush(driver, c, name, link, true) { + Some(failure) => Err(failure.error), + None => Ok(()), + } +} + +/// Hand the backlog (and a deferred shutdown) to the driver if the socket can +/// take it now; `force` ignores the in-flight cap. +unsafe fn flush( + driver: &mut turnloop::Loop, + c: &mut TransportCore, + name: Name, + link: OwnerLink, + force: bool, +) -> Option { + let entry = c.entry.as_mut()?; + if entry.connecting || entry.closing { + return None; + } + let backlog = &mut c.backlog; + let mut failure = None; + if !backlog.writes.is_empty() { + if !force && entry.inflight >= MAX_INFLIGHT_WRITES { + return None; + } + let bytes = std::mem::take(&mut backlog.bytes); + let mut writes = std::mem::take(&mut backlog.writes); + match write_queue::submit_bytes(driver, name, entry, bytes) { + Ok(()) => { + if let Some(tail) = writes.last_mut() { + tail.last = true; + } + entry.writes.extend(writes); + } + Err(e) => { + let mut users: Vec = writes.iter().map(|w| w.user).collect(); + users.extend(backlog.shutdown.take()); + failure = Some(Failure { + error: map_error(e, "write"), + users, + }); + } + } + } + #[cfg(windows)] + if failure.is_none() + && backlog.shutdown.is_some() + && entry.inflight > 0 + && windows_pipe::is_pipe(driver, entry) + { + return None; + } + if failure.is_none() { + if let Some(user) = backlog.shutdown.take() { + match write_queue::submit_shutdown(driver, name, entry, user) { + // A Windows pipe drain is a job, not a handle operation: it + // holds its own ref until its terminal result. + Ok(true) => hold(link), + Ok(false) => {} + Err(e) => { + failure = Some(Failure { + error: map_error(e, "shutdown"), + users: vec![user], + }) + } + } + } + } + failure +} + +/// [`flush`] from a completion; a refused flush is reported like a failed +/// write. +unsafe fn flush_after(link: OwnerLink, force: bool) { + let Some(c) = core_of(link) else { + return; + }; + let route = c.route; + let name = Name::Link { + route, + cell: link.0, + }; + let failure = with_driver(|driver| flush(driver, c, name, link, force)).flatten(); + if let Some(failure) = failure { + let queued = core_of(link).map_or(0, |c| total_queued(c)); + report_error(route, link, &failure.users, queued, failure.error, false); + } +} + +/// Retire the caller writes one driver write (or shutdown) covered. +fn retire(c: &mut TransportCore, op_class: u64) -> Vec<(u64, usize, usize)> { + let waiting = c.backlog.bytes.len(); + let Some(entry) = c.entry.as_mut() else { + return Vec::new(); + }; + if op_class == OP_WRITE { + entry.inflight = entry.inflight.saturating_sub(1); + } + let mut retired = Vec::new(); + while let Some(w) = entry.writes.pop_front() { + entry.queued = entry.queued.saturating_sub(w.len); + retired.push((w.user, w.len, entry.queued + waiting)); + if w.last { + break; + } + } + retired +} + +fn report_error( + route: u8, + link: OwnerLink, + users: &[u64], + queued: usize, + error: NodeError, + terminal: bool, +) { + let id = link.0 as i64; + let mut reported = false; + for &user in users.iter().filter(|&&user| user != 0) { + sink::emit( + route, + NetCompletion::error(id, user, queued, error, terminal).linked(), + ); + reported = true; + } + if !reported { + sink::emit( + route, + NetCompletion::error(id, 0, queued, error, terminal).linked(), + ); + } +} + +// ── Dispatch ──────────────────────────────────────────────────────────────── + +/// Route one link-token completion. Called by `super::dispatch` after the +/// turn, outside every loop borrow, so a sink may run JS and submit. +pub(super) fn dispatch(completion: Completion) { + let (route, op, cell) = link_parts(completion.token); + census::note_completion(op); + if cell == 0 { + // LINK_NULL: a close nobody waits on. + return; + } + if TEARDOWN.with(Cell::get) && !sabotage("teardown_dispatch") { + // Never dereference a token once the thread is going away. + #[cfg(test)] + DISCARDED.with(|n| n.set(n.get() + 1)); + return; + } + let link = OwnerLink(cell); + // SAFETY: a ref held for this completion's resource keeps the cell alive + // (rule 2); a finalized cell answers None and is never touched again. + if unsafe { np::link_event_owner(link) }.is_none() { + census::note_no_entry(); + return; + } + let Completion { + result, + terminal, + op: op_id, + handle, + .. + } = completion; + // SAFETY: as above, for every helper below. + unsafe { + match op { + OP_TIMER => dispatch_timer(link, handle, result), + OP_RESOLVE => dispatch_resolve(link, op_id, terminal, result), + #[cfg(windows)] + OP_PIPE_EOF => dispatch_pipe_eof(link, handle, result), + #[cfg(windows)] + OP_PIPE_DRAIN => dispatch_pipe_drain(link, op_id, terminal, result), + _ => dispatch_stream(route, op, link, handle, terminal, result), + } + } +} + +unsafe fn dispatch_timer(link: OwnerLink, handle: Option, result: OpResult) { + census::note_timer_result(&result); + match result { + OpResult::Timer => { + let Some(c) = core_of(link) else { + return; + }; + let (Some(timer), true) = (handle, c.deadline == handle) else { + // Replaced or cancelled before this expiry was dispatched; + // whoever did so closed the handle. + return; + }; + c.deadline = None; + let route = c.route; + let name = Name::Link { + route, + cell: link.0, + }; + // A fired one-shot keeps its handle until closed; the `Closed` + // releases the deadline's ref. + let closed = with_driver(|driver| driver.close(timer, name.token(OP_TIMER)).is_ok()); + if closed != Some(true) { + release(link); + } + sink::emit(route, NetCompletion::timer(link.0 as i64).linked()); + } + OpResult::Closed => release(link), + _ => {} + } +} + +unsafe fn dispatch_resolve(link: OwnerLink, op_id: Option, terminal: bool, result: OpResult) { + let current = core_of(link).filter(|c| { + c.plan + .as_ref() + .is_some_and(|p| p.op.is_some() && p.op == op_id) + || (c.plan.is_some() && sabotage("plan_check")) + }); + match result { + OpResult::Resolved(addresses) => { + if let Some(c) = current { + if let Some(plan) = c.plan.as_mut() { + plan.op = None; + plan.remaining = addresses.into_iter().collect(); + } + attempt_next_address(link); + } + } + OpResult::Err(err) => { + if let Some(c) = current { + c.plan = None; + c.backlog = Backlog::default(); + let route = c.route; + let mut mapped = map_error(err, "getaddrinfo"); + // libuv (and so node) reports any failed lookup as ENOTFOUND. + mapped.code = "ENOTFOUND"; + sink::emit( + route, + NetCompletion::error(link.0 as i64, 0, 0, mapped, true).linked(), + ); + } + } + OpResult::Cancelled | OpResult::Stopped => { + if let Some(c) = current { + c.plan = None; + c.backlog = Backlog::default(); + } + } + _ => {} + } + if terminal { + release(link); + } +} + +/// Start the plan's next address, or report its final failure. +unsafe fn attempt_next_address(link: OwnerLink) { + loop { + let Some(c) = core_of(link) else { + return; + }; + let route = c.route; + let Some(plan) = c.plan.as_mut() else { + return; + }; + let (nodelay, next, last_error) = + (plan.nodelay, plan.remaining.pop_front(), plan.last_error); + let Some(addr) = next else { + c.plan = None; + c.backlog = Backlog::default(); + let err = last_error.unwrap_or(NodeError { + code: "ECONNREFUSED", + errno: 0, + syscall: "connect", + }); + sink::emit( + route, + NetCompletion::error(link.0 as i64, 0, 0, err, true).linked(), + ); + return; + }; + let submitted = with_driver(|driver| connect_addr(driver, c, link, addr, nodelay)) + .unwrap_or_else(|| Err(no_loop())); + match submitted { + Ok(()) => return, + Err(err) => { + // Submission itself failed: record it and try the next one. + if let Some(plan) = core_of(link).and_then(|c| c.plan.as_mut()) { + plan.last_error = Some(err); + } + } + } + } +} + +/// A connect attempt failed: true when absorbed into a retry. The attempt's +/// handle is closed here and its `Closed` starts the next address. +unsafe fn connect_failed(c: &mut TransportCore, link: OwnerLink, err: NodeError) -> bool { + let Some(plan) = c.plan.as_mut() else { + return false; + }; + plan.last_error = Some(err); + if plan.remaining.is_empty() { + c.plan = None; + return false; + } + plan.retrying = true; + let name = Name::Link { + route: c.route, + cell: link.0, + }; + if let Some(entry) = c.entry.as_mut() { + if !entry.closing { + entry.closing = true; + census::note_submit(OP_CLOSE); + let handle = entry.handle; + let _ = with_driver(|driver| driver.close(handle, name.token(OP_CLOSE))); + } + } + true +} + +fn clear_op(c: &mut TransportCore, op_class: u64) { + if let Some(entry) = c.entry.as_mut() { + match op_class { + OP_ACCEPT => entry.accept_op = None, + OP_READ => entry.read_op = None, + _ => {} + } + } +} + +fn close_null(conn: Handle) { + let _ = with_driver(|driver| driver.close(conn, LINK_NULL)); +} + +unsafe fn dispatch_stream( + route: u8, + op: u64, + link: OwnerLink, + handle: Option, + terminal: bool, + result: OpResult, +) { + let id = link.0 as i64; + match result { + OpResult::Closed => { + let mut route = route; + let mut retry = false; + if let Some(c) = current(link, handle) { + route = c.route; + c.entry = None; + match c.plan.as_mut() { + Some(plan) if plan.retrying => { + plan.retrying = false; + retry = true; + } + _ => { + c.plan = None; + c.backlog = Backlog::default(); + } + } + } else if let Some(c) = core_of(link) { + route = c.route; + } + if retry { + // A failed attempt's handle, not the socket the caller sees. + attempt_next_address(link); + } else { + sink::emit(route, NetCompletion::closed(id).linked()); + } + // The handle's ref, taken at install. Last: the sink ran with the + // cell pinned. + release(link); + } + OpResult::Accepted { conn, peer } => accepted(link, handle, conn, Some(peer)), + OpResult::PipeAccepted { conn } => accepted(link, handle, conn, None), + OpResult::Connected => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + let local = + handle.and_then(|h| with_driver(|driver| driver.local_addr(h).ok()).flatten()); + if let Some(entry) = c.entry.as_mut() { + entry.local = local; + entry.connecting = false; + } + c.plan = None; + sink::emit(route, NetCompletion::connect(id).linked()); + // Writes (and an `end()`) issued while connecting go out now. + flush_after(link, true); + } + OpResult::Read { n, lease } => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + #[cfg(windows)] + { + let name = Name::Link { + route, + cell: link.0, + }; + if let Some(entry) = c.entry.as_mut() { + if entry.pipe_eof_timer.is_some() { + let created = + with_driver(|driver| windows_pipe::arm_eof(driver, name, entry)); + if matches!(created, Some(Ok(true))) { + hold(link); + } + } + } + } + let bytes = lease.as_ref().map(|l| l.as_slice()).unwrap_or(&[]); + debug_assert!(bytes.len() == n || lease.is_none()); + sink::emit(route, NetCompletion::data(id, bytes).linked()); + drop(lease); + } + OpResult::Eof => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + if let Some(entry) = c.entry.as_mut() { + #[cfg(windows)] + { + let name = Name::Link { + route, + cell: link.0, + }; + let _ = with_driver(|driver| windows_pipe::cancel_eof(driver, name, entry)); + } + entry.read_op = None; + } + sink::emit(route, NetCompletion::eof(id).linked()); + } + OpResult::Wrote(n) => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + let retired = retire(c, OP_WRITE); + debug_assert!( + retired.is_empty() || retired.iter().map(|r| r.1).sum::() == n, + "a write completes its whole buffer" + ); + flush_after(link, false); + for (user, len, queued) in retired { + sink::emit(route, NetCompletion::wrote(id, user, len, queued).linked()); + } + } + OpResult::Shutdown => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + let user = retire(c, OP_SHUTDOWN).first().map_or(0, |r| r.0); + sink::emit(route, NetCompletion::shutdown(id, user).linked()); + } + OpResult::Err(err) => { + let Some(c) = current(link, handle) else { + return; + }; + let route = c.route; + let users: Vec = if op == OP_WRITE || op == OP_SHUTDOWN { + retire(c, op).into_iter().map(|r| r.0).collect() + } else { + Vec::new() + }; + let queued = total_queued(c); + if terminal { + clear_op(c, op); + } + let mapped = map_error(err, syscall_for(op)); + if op == OP_CONNECT && connect_failed(c, link, mapped) { + return; + } + report_error(route, link, &users, queued, mapped, terminal); + } + OpResult::Cancelled | OpResult::Stopped => { + if let Some(c) = current(link, handle) { + clear_op(c, op); + if op == OP_WRITE || op == OP_SHUTDOWN { + retire(c, op); + } + } + } + _ => {} + } +} + +unsafe fn accepted( + link: OwnerLink, + handle: Option, + conn: Handle, + peer: Option, +) { + let Some(c) = current(link, handle) else { + // A stale listener's connection: nobody can own it. + close_null(conn); + return; + }; + let route = c.route; + let mut slot = AcceptSlot { + conn: Some(conn), + peer, + }; + let completion = + NetCompletion::accept(link.0 as i64, &mut slot as *mut AcceptSlot as i64, peer); + sink::emit(route, completion.linked()); + if let Some(conn) = slot.conn.take() { + // The sink refused (or could not allocate) the connection. + close_null(conn); + } +} + +#[cfg(windows)] +unsafe fn dispatch_pipe_eof(link: OwnerLink, handle: Option, result: OpResult) { + match result { + OpResult::Timer => { + let Some(c) = core_of(link) else { + return; + }; + let live = c + .entry + .as_ref() + .is_some_and(|e| !e.closing && e.pipe_eof_timer == handle); + if !live { + return; + } + let route = c.route; + let name = Name::Link { + route, + cell: link.0, + }; + sink::emit(route, NetCompletion::eof(link.0 as i64).linked()); + close_stream(link, name); + } + // The EOF timer handle's own `Closed`: its ref. + OpResult::Closed => release(link), + _ => {} + } +} + +#[cfg(windows)] +unsafe fn dispatch_pipe_drain( + link: OwnerLink, + op_id: Option, + terminal: bool, + result: OpResult, +) { + let live = core_of(link).and_then(|c| { + let route = c.route; + c.entry + .as_ref() + .is_some_and(|e| !e.closing && e.pipe_drain == op_id) + .then_some(route) + }); + if let Some(route) = live { + let name = Name::Link { + route, + cell: link.0, + }; + match result { + OpResult::Blocking(_) => { + let Some(c) = core_of(link) else { + return; + }; + let user = retire(c, OP_SHUTDOWN).first().map_or(0, |r| r.0); + sink::emit(route, NetCompletion::shutdown(link.0 as i64, user).linked()); + let armed = core_of(link) + .and_then(|c| c.entry.as_mut()) + .and_then(|entry| { + with_driver(|driver| windows_pipe::arm_eof(driver, name, entry)) + }); + match armed { + Some(Ok(true)) => hold(link), + Some(Ok(false)) => {} + _ => { + sink::emit(route, NetCompletion::eof(link.0 as i64).linked()); + close_stream(link, name); + } + } + } + OpResult::Err(error) => { + let users: Vec = core_of(link) + .map(|c| retire(c, OP_SHUTDOWN).into_iter().map(|r| r.0).collect()) + .unwrap_or_default(); + report_error(route, link, &users, 0, map_error(error, "shutdown"), true); + close_stream(link, name); + } + _ => {} + } + } + if terminal { + release(link); + } +} + +/// The runtime's own close of a stream (a Windows pipe's EOF): the handle +/// stays in the core, closing, and its `Closed` is delivered as the close. +#[cfg(windows)] +unsafe fn close_stream(link: OwnerLink, name: Name) { + let Some(entry) = core_of(link).and_then(|c| c.entry.as_mut()) else { + return; + }; + if entry.closing { + return; + } + entry.closing = true; + let _ = with_driver(|driver| windows_pipe::cancel_eof(driver, name, entry)); + if let Some(op) = entry.pipe_drain.take() { + let _ = with_driver(|driver| driver.cancel(op)); + } + census::note_submit(OP_CLOSE); + let handle = entry.handle; + let _ = with_driver(|driver| driver.close(handle, name.token(OP_CLOSE))); +} + +// ── The core as an ABI value ──────────────────────────────────────────────── + +/// Construct a core in place (perry-ffi's `TransportCore::new`). +/// +/// # Safety +/// `core` is writable for `TRANSPORT_CORE_WORDS` words, 8-aligned. +pub(super) unsafe fn core_init(core: *mut TransportCore, route: u8) { + // GC_STORE_AUDIT(POINTER_FREE): a binding's payload memory on the Rust + // heap (or its stack); a TransportCore holds no GC reference. + std::ptr::write(core, TransportCore::new(route)); +} + +/// Drop a core in place (perry-ffi's `Drop for TransportCore`). +/// +/// # Safety +/// `core` was initialized by [`core_init`] and is not used afterwards. +pub(super) unsafe fn core_drop(core: *mut TransportCore) { + std::ptr::drop_in_place(core); +} + +/// Frees memory only: a core is dropped inside a collection, at release, or +/// at thread teardown, so it never reaches the loop or any other thread-local +/// (rule 2 guarantees a swept core holds no live handle). The body exists for +/// the N7 sabotage alone and is empty in a production build. +impl Drop for TransportCore { + fn drop(&mut self) { + if sabotage("drop_closes") { + // The defect the rule forbids: a Drop that releases its handle. + let handle = self.handle(); + let _ = with_driver(|driver| { + if let Some(handle) = handle { + let _ = driver.close(handle, LINK_NULL); + } + }); + } + } +} diff --git a/crates/perry-runtime/src/turnloop_net/windows_pipe.rs b/crates/perry-runtime/src/turnloop_net/windows_pipe.rs index d44cfaab78..7fbc4161cd 100644 --- a/crates/perry-runtime/src/turnloop_net/windows_pipe.rs +++ b/crates/perry-runtime/src/turnloop_net/windows_pipe.rs @@ -15,31 +15,37 @@ pub(super) fn is_pipe(driver: &turnloop::Loop, entry: &Entry) -> bool { /// Windows duplex pipes cannot half-close. Match libuv's 50 ms read grace /// after draining writes, refreshing it whenever incoming data arrives. +/// Returns whether a new timer handle was created (a link route holds one ref +/// per timer handle until its `Closed`). pub(super) fn arm_eof( driver: &mut turnloop::Loop, - id: i64, + name: Name, entry: &mut Entry, -) -> Result<(), Error> { +) -> Result { let at = driver.now() + std::time::Duration::from_millis(50); if let Some(handle) = entry.pipe_eof_timer { if driver.timer_reset(handle, at) { - return Ok(()); + return Ok(false); } - let _ = driver.close(handle, token(OP_PIPE_EOF, id)); + let _ = driver.close(handle, name.token(OP_PIPE_EOF)); } - let handle = driver.timer(at, None, token(OP_PIPE_EOF, id))?; + let handle = driver.timer(at, None, name.token(OP_PIPE_EOF))?; let _ = driver.set_ref(handle, false); entry.pipe_eof_timer = Some(handle); - Ok(()) + Ok(true) } -pub(super) fn cancel_eof(driver: &mut turnloop::Loop, id: i64, entry: &mut Entry) { +pub(super) fn cancel_eof(driver: &mut turnloop::Loop, name: Name, entry: &mut Entry) { if let Some(handle) = entry.pipe_eof_timer.take() { - let _ = driver.close(handle, token(OP_PIPE_EOF, id)); + let _ = driver.close(handle, name.token(OP_PIPE_EOF)); } } -pub(super) fn submit(driver: &mut turnloop::Loop, id: i64, entry: &mut Entry) -> Result<(), Error> { +pub(super) fn submit( + driver: &mut turnloop::Loop, + name: Name, + entry: &mut Entry, +) -> Result<(), Error> { let turnloop::RawTransport::Handle(raw) = driver.raw_transport(entry.handle)? else { return Err(Error::new(ErrorKind::InvalidInput)); }; @@ -96,7 +102,7 @@ pub(super) fn submit(driver: &mut turnloop::Loop, id: i64, entry: &mut Entry) -> .unwrap_or_else(|_| Err(Error::new(ErrorKind::Other))) }, turnloop::Occupancy::Long, - token(OP_PIPE_DRAIN, id), + name.token(OP_PIPE_DRAIN), )?; entry.pipe_drain = Some(op); census::note_submit(OP_PIPE_DRAIN); diff --git a/crates/perry-runtime/src/turnloop_net/write_queue.rs b/crates/perry-runtime/src/turnloop_net/write_queue.rs index 30344c2fb7..ff93c527da 100644 --- a/crates/perry-runtime/src/turnloop_net/write_queue.rs +++ b/crates/perry-runtime/src/turnloop_net/write_queue.rs @@ -32,34 +32,42 @@ fn invalid(syscall: &'static str) -> NodeError { /// Submit `bytes` as one driver write. The caller records the /// [`PendingWrite`]s it covers. -fn submit_bytes( +pub(super) fn submit_bytes( driver: &mut turnloop::Loop, - id: i64, + name: Name, entry: &mut Entry, bytes: Vec, ) -> Result<(), Error> { let len = bytes.len(); - driver.write(entry.handle, WriteBuf::Owned(bytes), token(OP_WRITE, id))?; + driver.write(entry.handle, WriteBuf::Owned(bytes), name.token(OP_WRITE))?; census::note_submit(OP_WRITE); entry.queued += len; entry.inflight += 1; Ok(()) } -fn submit_shutdown( +/// Submit the write-side shutdown. Returns whether it went to a Windows pipe +/// drain job instead (a job, not a handle operation: a link route must hold a +/// ref for it until its terminal completion). +pub(super) fn submit_shutdown( driver: &mut turnloop::Loop, - id: i64, + name: Name, entry: &mut Entry, user: u64, -) -> Result<(), Error> { +) -> Result { #[cfg(windows)] - if windows_pipe::is_pipe(driver, entry) { - windows_pipe::submit(driver, id, entry)?; + let drain = if windows_pipe::is_pipe(driver, entry) { + windows_pipe::submit(driver, name, entry)?; + true } else { - driver.shutdown(entry.handle, token(OP_SHUTDOWN, id))?; - } + driver.shutdown(entry.handle, name.token(OP_SHUTDOWN))?; + false + }; #[cfg(not(windows))] - driver.shutdown(entry.handle, token(OP_SHUTDOWN, id))?; + let drain = { + driver.shutdown(entry.handle, name.token(OP_SHUTDOWN))?; + false + }; census::note_submit(OP_SHUTDOWN); // The user token rides the pending-write queue's tail slot so the // `Shutdown` completion can echo it back; a zero-length entry never @@ -69,7 +77,7 @@ fn submit_shutdown( len: 0, last: true, }); - Ok(()) + Ok(drain) } /// Accept one caller write: straight to the driver when nothing is ahead of @@ -101,7 +109,8 @@ pub(super) fn accept_write( let waiting = backlogs.get(&id).is_some_and(|b| !b.is_idle()); if !entry.connecting && !waiting && entry.inflight < MAX_INFLIGHT_WRITES { let len = bytes.len(); - submit_bytes(driver, id, entry, bytes).map_err(|e| map_error(e, "write"))?; + submit_bytes(driver, Name::Id(id), entry, bytes) + .map_err(|e| map_error(e, "write"))?; entry.writes.push_back(PendingWrite { user, len, @@ -189,7 +198,7 @@ pub(super) fn flush( } let bytes = std::mem::take(&mut backlog.bytes); let mut writes = std::mem::take(&mut backlog.writes); - match submit_bytes(driver, id, entry, bytes) { + match submit_bytes(driver, Name::Id(id), entry, bytes) { Ok(()) => { if let Some(tail) = writes.last_mut() { tail.last = true; @@ -216,7 +225,7 @@ pub(super) fn flush( } if failure.is_none() { if let Some(user) = backlog.shutdown.take() { - if let Err(e) = submit_shutdown(driver, id, entry, user) { + if let Err(e) = submit_shutdown(driver, Name::Id(id), entry, user) { failure = Some(FlushFailure { error: map_error(e, "shutdown"), users: vec![user], diff --git a/docs/native-payload-pattern.md b/docs/native-payload-pattern.md index d30d6633c4..48b99fd188 100644 --- a/docs/native-payload-pattern.md +++ b/docs/native-payload-pattern.md @@ -215,3 +215,51 @@ keyed by id. For sqlite this includes NODE_SQLITE_CUSTOM_FUNCTIONS, NODE_SQLITE_CUSTOM_AGGREGATES, NODE_SQLITE_ACTIVE_AGGREGATES, scan_node_sqlite_roots_mut and release_node_sqlite_authorizer_in_freed_ranges. These deletions belong to the family lanes; the runtime API adds none of them. + +## Families in a binding crate (the C ABI) + +A binding crate links only `perry-ffi`, so it cannot instantiate the generic +`native_payload::alloc::`. It declares its family as a `static` +`perry_ffi::native_payload::PayloadFamily::new::(class_id, name, +links_owner, constructor_length, install)` and calls the same operations +through `perry_ffi::native_payload`: `alloc`, `alloc_closed`, `payload_mut` +(checked by class id and by `T`'s size and alignment), `close`, `attach`, +`lifecycle`, `owner_link`, `link_ref`/`link_unref`, `link_event_owner`, +`js_state`, `set_external_bytes`, and `PayloadPrototype` for the installer. +They forward to `native_payload_abi.rs`, which runs the in-tree code: one +cell, one lifecycle, one owner link. The descriptor's first word is perry-ffi's +layout digest; the runtime refuses a descriptor with another one. Every rule +above holds unchanged. + +## Families that own a transport (NET-TRANSPORT-DESIGN) + +A socket, server or other turnloop transport is an A family whose payload is +`TransportPayload`: the runtime's `TransportCore` first (an opaque fixed +block in perry-ffi, its size folded into `js_perry_net_abi_layout`), then the +family's fields `E`. One allocation per socket; no id, no map. + +1. Register the sink with `register_link_sink(route, sink)`. Every completion + carries `NET_FLAG_LINK` and its `id` is the payload's owner link + (`NetCompletion::link`); dispatch reaches the owner through + `link_event_owner`, with no lookup. +2. Submit with `(&mut payload.core, owner_link)` (`link_tcp_listen`, + `link_tcp_connect`, `link_write`, ...). Hold no `&mut T` across a sink + call; fetch the payload again after any JS runs. +3. The runtime holds one `link_ref` per terminal completion the driver owes + (a handle's `Closed`, a resolve, a deadline) from submission to dispatch. + A listening server or open socket with no JS reference therefore lives + until it is closed, as in Node, and the sweep never drops a payload that + names a live descriptor. The family takes no refs of its own for I/O. +4. `close`/`destroy` is a JS method: `link_close` (the handle moves into the + driver's close; `Ok(false)` means there was no handle, so schedule the + `close` event yourself), then `native_payload::close`. A pipe server + unlinks its path synchronously here. Never touch the driver from `Drop`. +5. Reopen (`connect()` / `listen()` after close) is `attach` of a fresh + payload in the same cell. Staleness is the driver's generational handle: + an old `Closed` still delivers its `close` once and touches nothing else. +6. An accepted connection arrives as `NET_ACCEPT`: allocate the child payload + inside the sink and call `link_install_accepted`; one not installed is + closed when the sink returns. +7. Worker teardown discards link completions (no JS, no token dereferenced); + `Loop::drop` releases the descriptors and the heap teardown drops the + payloads.