diff --git a/nexus/inventory/tests/output/collector_basic.txt b/nexus/inventory/tests/output/collector_basic.txt index ebef70bf529..ae2c25a2b63 100644 --- a/nexus/inventory/tests/output/collector_basic.txt +++ b/nexus/inventory/tests/output/collector_basic.txt @@ -83,7 +83,7 @@ rot pages found: CfpaScratch baseboard part "i86pc" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" sled agents found: - sled 03265caf-da7d-46c7-b1c2-39fa90ce5c65 (Scrimlet) + sled 03265caf-da7d-46c7-b1c2-39fa90ce5c65 (Gimlet) baseboard Some(BaseboardId { part_number: "i86pc", serial_number: "sim-03265caf-da7d-46c7-b1c2-39fa90ce5c65" }) ledgered sled config: generation: 3 @@ -105,7 +105,7 @@ sled agents found: (empty) result for zone 8b88a56f-3eb6-4d80-ba42-75d867bc427d: Ok reconciler task idle - sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Scrimlet) + sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet) baseboard Some(BaseboardId { part_number: "i86pc", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" }) ledgered sled config: generation: 3 diff --git a/nexus/inventory/tests/output/collector_sled_agent_errors.txt b/nexus/inventory/tests/output/collector_sled_agent_errors.txt index 3db5381a687..677ed9237e4 100644 --- a/nexus/inventory/tests/output/collector_sled_agent_errors.txt +++ b/nexus/inventory/tests/output/collector_sled_agent_errors.txt @@ -82,7 +82,7 @@ rot pages found: CfpaScratch baseboard part "i86pc" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" sled agents found: - sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Scrimlet) + sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet) baseboard Some(BaseboardId { part_number: "i86pc", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" }) ledgered sled config: generation: 3 diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 1f5f8f66aa8..b2a16a05e75 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -46,7 +46,7 @@ omicron-common.workspace = true omicron-debug-dropbox.workspace = true omicron-generation-kinds.workspace = true omicron-passwords.workspace = true -omicron-sled-agent.workspace = true +omicron-sled-agent = { workspace = true, features = ["testing"] } omicron-test-utils.workspace = true omicron-uuid-kinds.workspace = true omicron-workspace-hack.workspace = true diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 3dd8716f568..d6bc1a3173e 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -257,6 +257,7 @@ impl ControlPlaneTestContext { sled_index, sim_mode, cpu_family, + false, &self.first_sled_agent().simulated_upstairs, ) .await diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index a8d4f3d4d46..e75db969869 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -162,6 +162,10 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub dendrite: RwLock>, pub mgd: HashMap, pub ddm: HashMap, + /// Maps scrimlet sled IDs to their switch slot. Populated by + /// `record_switch_dns()` and used by `start_sled()` to configure + /// the sled-agent with `is_scrimlet = true` and start reconcilers. + scrimlets: BTreeMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -225,6 +229,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { dendrite: RwLock::new(HashMap::new()), mgd: HashMap::new(), ddm: HashMap::new(), + scrimlets: BTreeMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -509,6 +514,10 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { "switch_slot" => ?switch_slot, ); + // Record that this sled is a scrimlet so that `start_sled()` can + // configure it with `is_scrimlet = true` and start reconcilers. + self.scrimlets.insert(sled_id, switch_slot); + self.rack_init_builder .internal_dns_config .host_zone_switch( @@ -944,6 +953,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { let nexus_address = self.nexus_internal_addr.expect("Must launch Nexus first"); + let switch_slot = self.scrimlets.get(&sled_id).copied(); + let is_scrimlet = switch_slot.is_some(); + let sled_agent = start_sled_agent( self.logctx.log.new(o!( "component" => "omicron_sled_agent::sim::Server", @@ -954,11 +966,39 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.sled_index_allocator.next(), sim_mode, SledCpuFamily::AmdMilan, + is_scrimlet, &self.simulated_upstairs, ) .await .expect("Failed to start sled agent"); + // If this is a scrimlet, start the scrimlet reconcilers so they can + // react to bootstore network config updates. + if let Some(slot) = switch_slot { + let mgs_addr: SocketAddr = + self.gateway.get(&slot).unwrap().address().into(); + let dpd_addr: SocketAddr = self + .dendrite + .read() + .unwrap() + .get(&slot) + .unwrap() + .address() + .into(); + let mgd_addr: SocketAddr = + self.mgd.get(&slot).unwrap().address().into(); + // Our test mgd uses --no-bgp-dispatcher, so pass the mgd admin + // address as a dummy bgp_dispatcher_addr. As long as tests don't + // configure BGP, the reconciler won't use this address. + let bgp_dispatcher_addr = mgd_addr; + sled_agent.sled_agent.start_scrimlet_reconcilers( + mgs_addr, + dpd_addr, + mgd_addr, + bgp_dispatcher_addr, + ); + } + // Add a DNS entry for the TUF Repo Depot on this simulated sled agent. let SocketAddr::V6(server_addr_v6) = sled_agent.repo_depot_address else { @@ -1076,6 +1116,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.sled_index_allocator.next(), sim_mode, SledCpuFamily::AmdMilan, + false, &self.simulated_upstairs, ) .await @@ -1929,6 +1970,7 @@ impl SledIndexAllocator { /// Note: you should probably use the `extra_sled_agents` macro parameter on /// `nexus_test` instead! To start a sled agent partway through a test, use /// [`ControlPlaneTestContext::add_sled`]. +#[allow(clippy::too_many_arguments)] pub(crate) async fn start_sled_agent( log: Logger, nexus_address: SocketAddr, @@ -1936,6 +1978,7 @@ pub(crate) async fn start_sled_agent( sled_index: u16, sim_mode: sim::SimMode, cpu_family: SledCpuFamily, + is_scrimlet: bool, simulated_upstairs: &Arc, ) -> Result { // Generate a baseboard serial number that matches the SP configuration @@ -1950,6 +1993,7 @@ pub(crate) async fn start_sled_agent( sim::ZpoolConfig::None, cpu_family, Some(baseboard_serial), + is_scrimlet, ); start_sled_agent_with_config(log, &config, sled_index, simulated_upstairs) .await diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 56af57038a7..4d57e26d52c 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -888,3 +888,277 @@ async fn test_bgp_config_update(ctx: &ControlPlaneTestContext) { assert_eq!(updated.asn, 47); assert_eq!(updated.max_paths, MaxPathConfig::new(3).unwrap()); } + +/// Verifies that scrimlet reconcilers are triggered when Nexus's +/// `sync_switch_configuration` background task updates the sled-agent +/// bootstore. +/// +/// Tests the full pipeline: +/// Nexus (sync_switch_configuration) +/// → sled-agent bootstore updated +/// → `notify_network_config_changed()` called +/// → reconcilers' watch channel updated +/// → reconcilers run with `SystemNetworkingConfigChanged` reason +#[nexus_test(extra_sled_agents = 1)] +async fn test_scrimlet_reconcilers_update_on_bootstore_change( + ctx: &ControlPlaneTestContext, +) { + use bootstrap_agent_lockstep_types::scrimlet_reconcilers::{ + ReconcilerActivationReason, ScrimletReconcilersStatus, + }; + + let client = &ctx.external_client; + + // Create an address lot. + let lot_name = + Name::from_str("subspace").expect("subspace should be a valid name"); + let lot_params = AddressLotCreate { + identity: IdentityMetadataCreateParams { + name: lot_name.clone(), + description: "where the comms happen".into(), + }, + kind: AddressLotKind::Infra, + blocks: vec![AddressLotBlockCreate { + first_address: "1701::a".parse().unwrap(), + last_address: "1701::e".parse().unwrap(), + }], + }; + NexusRequest::objects_post( + client, + "/v1/system/networking/address-lot", + &lot_params, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Create port settings. + let settings_name = + Name::from_str("nacelle2").expect("should be a valid name"); + let mut settings = + SwitchPortSettingsCreate::new(IdentityMetadataCreateParams { + name: settings_name.clone(), + description: "just a port".into(), + }); + let link_name = + Name::from_str("phy0").expect("phy0 should be a valid name"); + settings.links.push(LinkConfigCreate { + link_name: link_name.clone(), + mtu: 1500, + lldp: LldpLinkConfigCreate { + enabled: false, + link_name: None, + link_description: None, + chassis_id: None, + system_name: None, + system_description: None, + management_ip: None, + }, + fec: None, + speed: LinkSpeed::Speed100G, + autoneg: false, + tx_eq: None, + }); + settings.interfaces.push(SwitchInterfaceConfigCreate { + link_name: link_name.clone(), + v6_enabled: true, + kind: SwitchInterfaceKind::Primary, + }); + settings.addresses.push(AddressConfig { + link_name: link_name.clone(), + addresses: vec![Address { + address: "1701::d/64".parse().unwrap(), + vlan_id: None, + address_lot: NameOrId::Name(lot_name.clone()), + }], + }); + settings.routes.push(RouteConfig { + link_name: link_name.clone(), + routes: vec![Route { + dst: "2000::/64".parse().unwrap(), + gw: "2000::1".parse().unwrap(), + vid: None, + rib_priority: None, + }], + }); + NexusRequest::objects_post( + client, + "/v1/system/networking/switch-port-settings", + &settings, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Wait for the initial bootstore config (generation 3) to be written to + // all sled agents. This happens as part of test setup. + for (i, sled_agent) in ctx.sled_agents.iter().enumerate() { + let sled_agent = sled_agent.sled_agent().clone(); + wait_for_condition( + || async { + let generation = sled_agent + .bootstore_network_config + .lock() + .unwrap() + .generation; + if generation == 3 { + Ok(()) + } else { + Err(CondCheckError::<()>::NotYet { status: None }) + } + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .unwrap_or_else(|_| { + panic!("sled-agent {i}'s bootstore should be 3 prior to update") + }); + } + + // Apply port settings to switch0. + let apply_settings = SwitchPortApplySettings { + port_settings: NameOrId::Name(settings_name.clone()), + }; + let racks: Vec = NexusRequest::iter_collection_authn( + client, + "/v1/system/hardware/racks", + "", + None, + ) + .await + .expect("failed to list racks") + .all_items; + let rack_id = racks[0].identity.id; + + NexusRequest::new( + RequestBuilder::new( + client, + Method::POST, + &format!( + "/v1/system/hardware/switch-port/qsfp0/settings?\ + rack_id={rack_id}&switch_slot=switch0" + ), + ) + .body(Some(&apply_settings)) + .expect_status(Some(StatusCode::NO_CONTENT)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Activate `sync_switch_configuration`, which writes the updated config + // to sled-agent bootstores. + let task = nexus_test_utils::background::activate_background_task( + &ctx.lockstep_client, + "switch_port_config_manager", + ) + .await; + let nexus_lockstep_client::types::LastResult::Completed(result) = task.last + else { + panic!( + "switch_port_config_manager task did not complete: {:?}", + task.last + ); + }; + let status = serde_json::from_value::< + nexus_types::internal_api::background::SwitchPortSettingsManagerStatus, + >(result.details) + .expect( + "task details should deserialize as SwitchPortSettingsManagerStatus", + ); + assert!( + status.incomplete_bootstore_configs.is_empty(), + "sync_switch_configuration should have successfully built a bootstore \ + config for all racks: {status:?}", + ); + + // Verify the sled-agent bootstores were updated to generation 4. + for (i, sled_agent) in ctx.sled_agents.iter().enumerate() { + let sled_agent = sled_agent.sled_agent().clone(); + wait_for_condition( + || async { + let generation = sled_agent + .bootstore_network_config + .lock() + .unwrap() + .generation; + if generation == 4 { + Ok(()) + } else { + Err(CondCheckError::<()>::NotYet { status: None }) + } + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .unwrap_or_else(|_| { + panic!( + "sync_switch_configuration should have written to sled-agent \ + {i}'s bootstore (generation was still 3, indicating it was \ + never contacted)", + ) + }); + } + + // After the bootstore update, `notify_network_config_changed()` was called, + // which sends the new config to the scrimlet reconcilers via a watch + // channel. Verify that the reconcilers ran in response to this config + // change, confirming the full pipeline works end-to-end. + for (i, sled_agent) in ctx.sled_agents.iter().enumerate() { + let sled_agent = sled_agent.sled_agent().clone(); + wait_for_condition( + || async { + let Some(status) = sled_agent.scrimlet_reconcilers_status() + else { + // Reconcilers not yet started (start_scrimlet_reconcilers + // hasn't been called yet). + return Err(CondCheckError::<()>::NotYet { status: None }); + }; + match status { + ScrimletReconcilersStatus::Running { + mgd_reconciler, .. + } => { + let Some(last) = + mgd_reconciler.last_completion.as_ref() + else { + // Reconciler hasn't completed a run yet. + return Err(CondCheckError::<()>::NotYet { + status: None, + }); + }; + if matches!( + last.activation_reason, + ReconcilerActivationReason::SystemNetworkingConfigChanged + ) { + Ok(()) + } else { + // Still showing startup activation; waiting for + // the config-change-triggered run. + Err(CondCheckError::<()>::NotYet { status: None }) + } + } + _ => { + // Reconcilers still initializing (determining switch + // slot or waiting for networking info). + Err(CondCheckError::<()>::NotYet { status: None }) + } + } + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .unwrap_or_else(|_| { + panic!( + "sled-agent {i}'s scrimlet reconcilers should have run in \ + response to the bootstore update (last status: {:?})", + sled_agent.scrimlet_reconcilers_status(), + ) + }); + } +} diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 47577484c32..c3e3e588236 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -182,3 +182,4 @@ image-trampoline = [] switch-asic = [] switch-stub = [] switch-softnpu = [] +testing = ["sled-agent-scrimlet-reconcilers/testing"] diff --git a/sled-agent/scrimlet-reconcilers/src/handle.rs b/sled-agent/scrimlet-reconcilers/src/handle.rs index 417ab28fff6..993e6bb8649 100644 --- a/sled-agent/scrimlet-reconcilers/src/handle.rs +++ b/sled-agent/scrimlet-reconcilers/src/handle.rs @@ -47,6 +47,10 @@ pub enum ScrimletReconcilersMode { mgs_addr: SocketAddr, dpd_addr: SocketAddr, mgd_addr: SocketAddr, + /// The address the mgd bgp-dispatcher is listening on. Used as the + /// router's listen address and to derive the remote BGP port for + /// numbered peers (replacing the standard port 179). + bgp_dispatcher_addr: SocketAddr, }, } diff --git a/sled-agent/scrimlet-reconcilers/src/handle/tests.rs b/sled-agent/scrimlet-reconcilers/src/handle/tests.rs index a0a2e3954ca..3d3a819549a 100644 --- a/sled-agent/scrimlet-reconcilers/src/handle/tests.rs +++ b/sled-agent/scrimlet-reconcilers/src/handle/tests.rs @@ -107,6 +107,7 @@ impl Harness { mgs_addr: self.mgs.address(), dpd_addr: dummy_addr, mgd_addr: dummy_addr, + bgp_dispatcher_addr: dummy_addr, }, } } diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs index 297eea46b1a..4112e5533fd 100644 --- a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs @@ -12,6 +12,7 @@ use bootstrap_agent_lockstep_types::scrimlet_reconcilers::mgd::MgdReconcilerStat use mg_admin_client::Client; use sled_agent_types::system_networking::SystemNetworkingConfig; use slog::Logger; +use std::net::SocketAddr; use std::time::Duration; mod bfd_reconciler; @@ -22,6 +23,10 @@ mod static_route_reconciler; pub(crate) struct MgdReconciler { client: Client, switch_slot: ThisSledSwitchSlot, + /// The address the mgd bgp-dispatcher is listening on, used to set the + /// correct BGP port when creating routers and numbered neighbors. `None` + /// means use the standard BGP port 179. + bgp_dispatcher_addr: Option, } impl Reconciler for MgdReconciler { @@ -35,7 +40,18 @@ impl Reconciler for MgdReconciler { switch_slot: ThisSledSwitchSlot, parent_log: &Logger, ) -> Self { - Self { client: mode.mgd_client(parent_log), switch_slot } + let bgp_dispatcher_addr = match mode { + ScrimletReconcilersMode::SwitchZone(_) => None, + #[cfg(any(test, feature = "testing"))] + ScrimletReconcilersMode::Test { bgp_dispatcher_addr, .. } => { + Some(bgp_dispatcher_addr) + } + }; + Self { + client: mode.mgd_client(parent_log), + switch_slot, + bgp_dispatcher_addr, + } } async fn do_reconciliation( @@ -55,6 +71,7 @@ impl Reconciler for MgdReconciler { &self.client, &system_networking_config.rack_network_config, self.switch_slot, + self.bgp_dispatcher_addr, log, ) .await; diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler.rs index cc1ab9e98a2..b88bfb4bd92 100644 --- a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler.rs +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler.rs @@ -60,6 +60,7 @@ pub(super) async fn reconcile( client: &Client, desired_config: &RackNetworkConfig, our_switch_slot: ThisSledSwitchSlot, + bgp_dispatcher_addr: Option, log: &Logger, ) -> MgdBgpReconcilerStatus { let current_config = match DiffableBgpConfig::fetch_current(client).await { @@ -74,6 +75,7 @@ pub(super) async fn reconcile( let desired_config = match DiffableBgpConfig::from_desired_config( &desired_config, our_switch_slot, + bgp_dispatcher_addr, log, ) { Ok(config) => config, @@ -1237,6 +1239,7 @@ impl DiffableBgpConfig { fn from_desired_config( config: &RackNetworkConfig, our_switch_slot: ThisSledSwitchSlot, + bgp_dispatcher_addr: Option, log: &Logger, ) -> anyhow::Result { // Filter down to just the peers of the ports matching our switch slot. @@ -1335,13 +1338,16 @@ impl DiffableBgpConfig { entry.insert(DiffableBgpRouterConfig { id: *asn, graceful_shutdown: false, - listen: SocketAddrV6::new( - Ipv6Addr::UNSPECIFIED, - BGP_PORT, - 0, - 0, - ) - .to_string(), + listen: match bgp_dispatcher_addr { + Some(addr) => addr.to_string(), + None => SocketAddrV6::new( + Ipv6Addr::UNSPECIFIED, + BGP_PORT, + 0, + 0, + ) + .to_string(), + }, }); originate4.insert( @@ -1446,9 +1452,12 @@ impl DiffableBgpConfig { } } RouterPeerType::Numbered(numbered_router) => { + let bgp_port = bgp_dispatcher_addr + .map(|a| a.port()) + .unwrap_or(BGP_PORT); let addr = SocketAddr::new( (numbered_router.target_addr()).into(), - BGP_PORT, + bgp_port, ); if let Some(_prev) = numbered_peers.insert(addr, common) { bail!( diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler/tests.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler/tests.rs index ffaeec1b464..c68b992cb09 100644 --- a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler/tests.rs +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/bgp_reconciler/tests.rs @@ -195,6 +195,7 @@ fn proptest_valid_input(input: TestInput) { match DiffableBgpConfig::from_desired_config( &input.rack_network_config(), ThisSledSwitchSlot::TEST_FAKE, + None, &logctx.log, ) { Ok(_) => (), @@ -485,6 +486,7 @@ async fn run_one_proptest_input( &client, &rack_network_config, ThisSledSwitchSlot::TEST_FAKE, + None, &log, ) .await @@ -578,6 +580,7 @@ async fn run_one_proptest_input( let desired_config = DiffableBgpConfig::from_desired_config( &rack_network_config, ThisSledSwitchSlot::TEST_FAKE, + None, log, ) .unwrap(); diff --git a/sled-agent/scrimlet-reconcilers/src/reconciler_task/tests.rs b/sled-agent/scrimlet-reconcilers/src/reconciler_task/tests.rs index ea50b33fca4..9c0655c9e94 100644 --- a/sled-agent/scrimlet-reconcilers/src/reconciler_task/tests.rs +++ b/sled-agent/scrimlet-reconcilers/src/reconciler_task/tests.rs @@ -143,6 +143,7 @@ impl Harness { mgs_addr: dummy_addr, dpd_addr: dummy_addr, mgd_addr: dummy_addr, + bgp_dispatcher_addr: dummy_addr, }, ThisSledSwitchSlot::TEST_FAKE, log, diff --git a/sled-agent/src/sim/config.rs b/sled-agent/src/sim/config.rs index 7483dab3c4e..004849efa75 100644 --- a/sled-agent/src/sim/config.rs +++ b/sled-agent/src/sim/config.rs @@ -82,6 +82,8 @@ pub struct Config { pub storage: ConfigStorage, /// configuration to emulate the sled agent's hardware pub hardware: ConfigHardware, + /// whether this sled is a scrimlet (connected to a switch) + pub is_scrimlet: bool, } pub enum ZpoolConfig { @@ -107,6 +109,7 @@ impl Config { zpool_config, cpu_family, None, + false, ) } @@ -117,6 +120,7 @@ impl Config { zpool_config: ZpoolConfig, cpu_family: SledCpuFamily, baseboard_serial: Option, + is_scrimlet: bool, ) -> Config { // This IP range is guaranteed by RFC 6666 to discard traffic. // For tests that don't use a Nexus, we use this address to simulate a @@ -164,6 +168,7 @@ impl Config { revision: 3, }, }, + is_scrimlet, } } } diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 1d513c37e4a..9e4a72dfa37 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -451,12 +451,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -464,12 +466,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -477,12 +481,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -490,12 +496,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -503,12 +511,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -516,12 +526,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -529,12 +541,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } @@ -542,12 +556,14 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, body: TypedBody, ) -> Result { - let mut config = - rqctx.context().bootstore_network_config.lock().unwrap(); + let sa = rqctx.context(); let body = body.into_inner(); - - *config = EarlyNetworkConfigEnvelope::from(&body.body) - .serialize_to_bootstore_with_generation(body.generation); + { + let mut config = sa.bootstore_network_config.lock().unwrap(); + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + } + sa.notify_network_config_changed(); Ok(HttpResponseUpdatedNoContent()) } diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 79f7a7c7c09..f820b19d84d 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -186,7 +186,11 @@ impl Server { &NexusTypes::SledAgentInfo { sa_address: sa_address.to_string(), repo_depot_port, - role: NexusTypes::SledRole::Scrimlet, + role: if config.is_scrimlet { + NexusTypes::SledRole::Scrimlet + } else { + NexusTypes::SledRole::Gimlet + }, baseboard: NexusTypes::Baseboard { serial: config .hardware diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index f2a62aba68d..88334c06e7a 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -118,6 +118,20 @@ pub struct SledAgent { pub repo_depot: dropshot::HttpServer>, pub log: Logger, health_monitor: HealthMonitorHandle, + /// Watch channel that sends the deserialized [`SystemNetworkingConfig`] + /// whenever Nexus writes a new bootstore config. Only populated for + /// scrimlet sleds; used to drive the scrimlet reconcilers. + #[cfg(feature = "testing")] + network_config_tx: Option< + tokio::sync::watch::Sender< + sled_agent_types::system_networking::SystemNetworkingConfig, + >, + >, + /// Keeps the scrimlet reconcilers alive once started. + #[cfg(feature = "testing")] + scrimlet_reconcilers: std::sync::OnceLock< + std::sync::Arc, + >, } impl SledAgent { @@ -178,6 +192,28 @@ impl SledAgent { let health_monitor = HealthMonitorHandle::stub(); + #[cfg(feature = "testing")] + let network_config_tx = if config.is_scrimlet { + let (tx, _) = tokio::sync::watch::channel(SystemNetworkingConfig { + rack_network_config: RackNetworkConfig { + rack_subnet: Ipv6Net::new(Ipv6Addr::UNSPECIFIED, 56) + .unwrap(), + infra_ip_first: IpAddr::V4(Ipv4Addr::UNSPECIFIED), + infra_ip_last: IpAddr::V4(Ipv4Addr::UNSPECIFIED), + ports: UplinkPorts::new(vec![PortConfig::empty_for_tests( + "qsfp0", + )]) + .expect("placeholder port list is non-empty"), + bgp: Vec::new(), + bfd: Vec::new(), + }, + blueprint_external_networking_config: None, + }); + Some(tx) + } else { + None + }; + Arc::new(SledAgent { id, ip: config.dropshot.bind_address.ip(), @@ -202,9 +238,93 @@ impl SledAgent { log, bootstore_network_config, health_monitor, + #[cfg(feature = "testing")] + network_config_tx, + #[cfg(feature = "testing")] + scrimlet_reconcilers: std::sync::OnceLock::new(), }) } + /// Called after every `write_network_bootstore_config_vXX` handler updates + /// [`Self::bootstore_network_config`]. No-op when the `testing` feature is + /// not enabled or this sled is not a scrimlet. + pub(crate) fn notify_network_config_changed(&self) { + #[cfg(feature = "testing")] + { + let Some(tx) = &self.network_config_tx else { return }; + let config = self.bootstore_network_config.lock().unwrap().clone(); + match sled_agent_types::early_networking::EarlyNetworkConfigEnvelope::deserialize_from_bootstore(&config) + .and_then(|e| e.deserialize_body()) + { + Ok(system_config) => { + tx.send_modify(|c| *c = system_config); + } + Err(e) => { + slog::warn!( + self.log, + "failed to deserialize bootstore config for \ + scrimlet reconcilers (reconcilers may lag)"; + "error" => %e, + ); + } + } + } + } + + /// Start the scrimlet reconcilers pointing at the given switch zone service + /// addresses. Must only be called once and only on scrimlet sleds. + /// + /// Only available under `cfg(feature = "testing")` because it uses + /// [`sled_agent_scrimlet_reconcilers::ScrimletReconcilersMode::Test`]. + #[cfg(feature = "testing")] + pub fn start_scrimlet_reconcilers( + &self, + mgs_addr: std::net::SocketAddr, + dpd_addr: std::net::SocketAddr, + mgd_addr: std::net::SocketAddr, + bgp_dispatcher_addr: std::net::SocketAddr, + ) { + use sled_agent_scrimlet_reconcilers::{ + ScrimletReconcilers, ScrimletReconcilersMode, ScrimletStatus, + SledAgentNetworkingInfo, + }; + + let tx = self + .network_config_tx + .as_ref() + .expect("network_config_tx must be Some for scrimlet sleds"); + + let reconcilers = ScrimletReconcilers::new(&self.log); + reconcilers.set_sled_agent_networking_info_once( + SledAgentNetworkingInfo { + system_networking_config_rx: tx.subscribe(), + mode: ScrimletReconcilersMode::Test { + mgs_addr, + dpd_addr, + mgd_addr, + bgp_dispatcher_addr, + }, + }, + ); + reconcilers.set_scrimlet_status(ScrimletStatus::Scrimlet); + + // Store to keep the reconcilers alive. Ignore the error: if called + // twice it is a programmer error and we just silently drop the second + // set (the first set is already running). + let _ = self.scrimlet_reconcilers.set(std::sync::Arc::new(reconcilers)); + } + + /// Returns the current status of the scrimlet reconcilers, or `None` if + /// `start_scrimlet_reconcilers()` has not yet been called. + #[cfg(feature = "testing")] + pub fn scrimlet_reconcilers_status( + &self, + ) -> Option< + bootstrap_agent_lockstep_types::scrimlet_reconcilers::ScrimletReconcilersStatus, + >{ + self.scrimlet_reconcilers.get().map(|r| r.status()) + } + pub async fn instance_register( self: &Arc, propolis_id: PropolisUuid, @@ -927,7 +1047,11 @@ impl SledAgent { Ok(Inventory { sled_id: self.id, sled_agent_address, - sled_role: SledRole::Scrimlet, + sled_role: if self.config.is_scrimlet { + SledRole::Scrimlet + } else { + SledRole::Gimlet + }, baseboard_id: self.config.hardware.baseboard.clone().into(), usable_hardware_threads: self.config.hardware.hardware_threads, usable_physical_ram: ByteCount::try_from(