diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 4500580ad50..e90899b585d 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -58,8 +58,9 @@ use nexus_types::internal_api::background::AttachedSubnetManagerStatus; use nexus_types::internal_api::background::AuditLogCleanupStatus; use nexus_types::internal_api::background::AuditLogTimeoutIncompleteStatus; use nexus_types::internal_api::background::BlueprintPlannerStatus; -use nexus_types::internal_api::background::BlueprintRendezvousStats; use nexus_types::internal_api::background::BlueprintRendezvousStatus; +use nexus_types::internal_api::background::DatasetRendezvousOutcome; +use nexus_types::internal_api::background::DatasetRendezvousStats; use nexus_types::internal_api::background::DatasetsRendezvousStats; use nexus_types::internal_api::background::EreporterStatus; use nexus_types::internal_api::background::FmAnalysisStatus; @@ -83,6 +84,8 @@ use nexus_types::internal_api::background::ServiceFirewallRuleStatus; use nexus_types::internal_api::background::SessionCleanupStatus; use nexus_types::internal_api::background::SitrepGcStatus; use nexus_types::internal_api::background::SitrepLoadStatus; +use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousOutcome; +use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousStats; use nexus_types::internal_api::background::SupportBundleActivationReport; use nexus_types::internal_api::background::SupportBundleCleanupReport; use nexus_types::internal_api::background::SupportBundleCollectionStepStatus; @@ -1696,82 +1699,131 @@ fn print_task_blueprint_rendezvous(details: &serde_json::Value) { error, details ), Ok(status) => { - println!(" target blueprint: {}", status.blueprint_id); - println!( - " inventory collection: {}", - status.inventory_collection_id - ); - - let BlueprintRendezvousStats { - debug_dataset, - crucible_dataset, - local_storage_dataset, - local_storage_unencrypted_dataset, + let BlueprintRendezvousStatus { + blueprint_id, sled_blueprint_availability, - } = status.stats; + datasets, + } = status; + println!(" target blueprint: {blueprint_id}"); - print_datasets_rendezvous_stats(&debug_dataset, "debug_dataset"); + match datasets { + DatasetRendezvousOutcome::NoInventoryCollection => { + println!( + " inventory collection: none loaded yet; dataset \ + reconciliation skipped" + ); + } + DatasetRendezvousOutcome::Error { + inventory_collection_id, + error, + } => { + println!( + " inventory collection: {inventory_collection_id}" + ); + println!(" dataset reconciliation failed: {error}"); + } + DatasetRendezvousOutcome::Reconciled { + inventory_collection_id, + stats, + } => { + println!( + " inventory collection: {inventory_collection_id}" + ); + print_dataset_rendezvous_stats(&stats); + } + } - // crucible datasets have a different number of rendezvous stats - println!(" crucible_dataset rendezvous counts:"); - println!( - " num_inserted: {}", - crucible_dataset.num_inserted - ); - println!( - " num_already_exist: {}", - crucible_dataset.num_already_exist - ); - println!( - " num_not_in_inventory: {}", - crucible_dataset.num_not_in_inventory - ); + match sled_blueprint_availability { + SledBlueprintAvailabilityRendezvousOutcome::Error(error) => { + println!( + " sled_blueprint_availability reconciliation \ + failed: {error}" + ); + } + SledBlueprintAvailabilityRendezvousOutcome::Reconciled( + stats, + ) => { + print_sled_blueprint_availability_rendezvous_stats(&stats); + } + } + } + } +} - print_datasets_rendezvous_stats( - &local_storage_dataset, - "local_storage_dataset", - ); +fn print_dataset_rendezvous_stats(stats: &DatasetRendezvousStats) { + let DatasetRendezvousStats { + debug_dataset, + crucible_dataset, + local_storage_dataset, + local_storage_unencrypted_dataset, + } = stats; - print_datasets_rendezvous_stats( - &local_storage_unencrypted_dataset, - "local_storage_unencrypted_dataset", - ); + print_datasets_rendezvous_stats(debug_dataset, "debug_dataset"); - println!(" sled_blueprint_availability rendezvous counts:"); - println!( - " num_marked_available: {}", - sled_blueprint_availability.num_marked_available - ); - println!( - " num_marked_unavailable: {}", - sled_blueprint_availability.num_marked_unavailable - ); - println!( - " num_unchanged: {}", - sled_blueprint_availability.num_unchanged - ); - println!( - " num_invariant_violations: {}", - sled_blueprint_availability.num_invariant_violations - ); - println!( - " num_decommissioned: {}", - sled_blueprint_availability.num_decommissioned - ); - println!( - " num_already_decommissioned: {}", - sled_blueprint_availability.num_already_decommissioned - ); - println!( - " num_not_in_blueprint: {}", - sled_blueprint_availability.num_not_in_blueprint - ); - println!( - " num_decommissioned_not_in_blueprint: {}", - sled_blueprint_availability.num_decommissioned_not_in_blueprint - ); - } - } + // crucible datasets have a different number of rendezvous stats + println!(" crucible_dataset rendezvous counts:"); + println!(" num_inserted: {}", crucible_dataset.num_inserted); + println!( + " num_already_exist: {}", + crucible_dataset.num_already_exist + ); + println!( + " num_not_in_inventory: {}", + crucible_dataset.num_not_in_inventory + ); + + print_datasets_rendezvous_stats( + local_storage_dataset, + "local_storage_dataset", + ); + + print_datasets_rendezvous_stats( + local_storage_unencrypted_dataset, + "local_storage_unencrypted_dataset", + ); +} + +fn print_sled_blueprint_availability_rendezvous_stats( + stats: &SledBlueprintAvailabilityRendezvousStats, +) { + let SledBlueprintAvailabilityRendezvousStats { + num_marked_available, + num_marked_unavailable, + num_unchanged, + num_invariant_violations, + num_decommissioned, + num_already_decommissioned, + num_not_in_blueprint, + num_decommissioned_not_in_blueprint, + } = stats; + + println!(" sled_blueprint_availability rendezvous counts:"); + println!( + " num_marked_available: {num_marked_available}" + ); + println!( + " num_marked_unavailable: \ + {num_marked_unavailable}" + ); + println!(" num_unchanged: {num_unchanged}"); + println!( + " num_invariant_violations: \ + {num_invariant_violations}" + ); + println!( + " num_decommissioned: {num_decommissioned}" + ); + println!( + " num_already_decommissioned: \ + {num_already_decommissioned}" + ); + println!( + " num_not_in_blueprint: {num_not_in_blueprint}" + ); + println!( + " num_decommissioned_not_in_blueprint: \ + {num_decommissioned_not_in_blueprint}" + ); } fn print_task_dns_config(details: &serde_json::Value) { diff --git a/nexus/reconfigurator/rendezvous/src/lib.rs b/nexus/reconfigurator/rendezvous/src/lib.rs index 07eca42bf4d..14173257209 100644 --- a/nexus/reconfigurator/rendezvous/src/lib.rs +++ b/nexus/reconfigurator/rendezvous/src/lib.rs @@ -13,7 +13,8 @@ use nexus_db_queries::db::DataStore; use nexus_db_queries::db::model::SledBlueprintAvailabilityInput; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintDatasetDisposition; -use nexus_types::internal_api::background::BlueprintRendezvousStats; +use nexus_types::internal_api::background::DatasetRendezvousStats; +use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousStats; use nexus_types::inventory::Collection; mod crucible_dataset; @@ -22,12 +23,37 @@ mod local_storage_dataset; mod local_storage_unencrypted_dataset; mod sled_blueprint_availability; -pub async fn reconcile_blueprint_rendezvous_tables( +/// Reconcile the `rendezvous_sled_bp_availability` table for the given +/// blueprint. +/// +/// This is a pure projection of the target blueprint and does not depend on +/// inventory. +pub async fn reconcile_sled_blueprint_availability( + opctx: &OpContext, + datastore: &DataStore, + blueprint: &Blueprint, +) -> anyhow::Result { + let sled_inputs = + SledBlueprintAvailabilityInput::all_from_blueprint(blueprint); + sled_blueprint_availability::reconcile( + opctx, + datastore, + blueprint.id, + sled_inputs, + ) + .await +} + +/// Reconcile the dataset rendezvous tables for the given blueprint. +/// +/// These need inventory to confirm that a dataset exists before other +/// subsystems may use it. +pub async fn reconcile_dataset_rendezvous_tables( opctx: &OpContext, datastore: &DataStore, blueprint: &Blueprint, inventory: &Collection, -) -> anyhow::Result { +) -> anyhow::Result { let inventory_dataset_ids = inventory .sled_agents .iter() @@ -79,23 +105,11 @@ pub async fn reconcile_blueprint_rendezvous_tables( ) .await?; - let sled_inputs = - SledBlueprintAvailabilityInput::all_from_blueprint(blueprint); - let sled_blueprint_availability = - sled_blueprint_availability::reconcile_sled_blueprint_availability( - opctx, - datastore, - blueprint.id, - sled_inputs, - ) - .await?; - - Ok(BlueprintRendezvousStats { + Ok(DatasetRendezvousStats { debug_dataset, crucible_dataset, local_storage_dataset, local_storage_unencrypted_dataset, - sled_blueprint_availability, }) } diff --git a/nexus/reconfigurator/rendezvous/src/sled_blueprint_availability.rs b/nexus/reconfigurator/rendezvous/src/sled_blueprint_availability.rs index abc86b644f6..13fe86cf1ba 100644 --- a/nexus/reconfigurator/rendezvous/src/sled_blueprint_availability.rs +++ b/nexus/reconfigurator/rendezvous/src/sled_blueprint_availability.rs @@ -27,7 +27,7 @@ use slog::info; /// /// `blueprint_sleds` should contain a [`SledBlueprintAvailabilityInput`] for /// every sled in the blueprint, including decommissioned sleds. -pub(crate) async fn reconcile_sled_blueprint_availability( +pub(crate) async fn reconcile( opctx: &OpContext, datastore: &DataStore, blueprint_id: BlueprintUuid, @@ -552,7 +552,7 @@ mod tests { &prep, ).await; - let result_stats = reconcile_sled_blueprint_availability( + let result_stats = reconcile( opctx, datastore, blueprint_id, @@ -623,7 +623,7 @@ mod tests { .await .expect("seeded the stored row"); - let stats = reconcile_sled_blueprint_availability( + let stats = reconcile( opctx, datastore, bp_reconciled, diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d5659dac257..453c496cb94 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -676,7 +676,13 @@ impl BackgroundTasksInitializer { ), ), opctx: opctx.child(BTreeMap::new()), - watchers: vec![Box::new(inventory_load_watcher.clone())], + // A new target blueprint must reach the sled availability table + // promptly, even if inventory is stalled for whatever reason, so + // watch both channels. + watchers: vec![ + Box::new(rx_blueprint.clone()), + Box::new(inventory_load_watcher.clone()), + ], activator: task_blueprint_rendezvous, }); diff --git a/nexus/src/app/background/tasks/blueprint_rendezvous.rs b/nexus/src/app/background/tasks/blueprint_rendezvous.rs index d0a2142c536..4c10da5b549 100644 --- a/nexus/src/app/background/tasks/blueprint_rendezvous.rs +++ b/nexus/src/app/background/tasks/blueprint_rendezvous.rs @@ -12,10 +12,12 @@ use futures::FutureExt; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; -use nexus_reconfigurator_rendezvous::reconcile_blueprint_rendezvous_tables; -use nexus_types::{ - internal_api::background::BlueprintRendezvousStatus, inventory::Collection, -}; +use nexus_reconfigurator_rendezvous::reconcile_dataset_rendezvous_tables; +use nexus_reconfigurator_rendezvous::reconcile_sled_blueprint_availability; +use nexus_types::internal_api::background::BlueprintRendezvousStatus; +use nexus_types::internal_api::background::DatasetRendezvousOutcome; +use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousOutcome; +use nexus_types::inventory::Collection; use serde_json::json; use std::sync::Arc; use tokio::sync::watch; @@ -57,42 +59,85 @@ impl BlueprintRendezvous { return json!({"error": "no blueprint" }); }; + // Reconcile sled availability -- this can be done without an inventory + // collection available. + let sled_blueprint_availability = + match reconcile_sled_blueprint_availability( + opctx, + &self.datastore, + &blueprint, + ) + .await + { + Ok(stats) => { + SledBlueprintAvailabilityRendezvousOutcome::Reconciled( + stats, + ) + } + Err(err) => { + error!( + &opctx.log, + "Blueprint rendezvous: sled availability reconciliation \ + failed"; + "blueprint_id" => %blueprint.id, + "error" => format!("{err:#}"), + ); + SledBlueprintAvailabilityRendezvousOutcome::Error(format!( + "{err:#}" + )) + } + }; + // Get the inventory most recently seen by the inventory loader // background task. We clone the Arc to avoid keeping the channel locked // for the rest of our execution. - let Some(collection) = - self.rx_inventory.borrow_and_update().as_ref().map(Arc::clone) - else { - warn!( - &opctx.log, "Blueprint rendezvous: skipped"; - "reason" => "no inventory collection", - ); - return json!({"error": "no inventory collection" }); + let inventory = + self.rx_inventory.borrow_and_update().as_ref().map(Arc::clone); + let datasets = match inventory { + None => { + warn!( + &opctx.log, + "Blueprint rendezvous: skipped dataset reconciliation"; + "reason" => "no inventory collection", + ); + DatasetRendezvousOutcome::NoInventoryCollection + } + Some(collection) => { + match reconcile_dataset_rendezvous_tables( + opctx, + &self.datastore, + &blueprint, + &collection, + ) + .await + { + Ok(stats) => DatasetRendezvousOutcome::Reconciled { + inventory_collection_id: collection.id, + stats, + }, + Err(err) => { + error!( + &opctx.log, + "Blueprint rendezvous: dataset reconciliation \ + failed"; + "blueprint_id" => %blueprint.id, + "inventory_collection_id" => %collection.id, + "error" => format!("{err:#}"), + ); + DatasetRendezvousOutcome::Error { + inventory_collection_id: collection.id, + error: format!("{err:#}"), + } + } + } + } }; - // Actually perform rendezvous table reconciliation - let result = reconcile_blueprint_rendezvous_tables( - opctx, - &self.datastore, - &blueprint, - &collection, - ) - .await; - - // Return the result as a `serde_json::Value` - match result { - Ok(stats) => { - let status = BlueprintRendezvousStatus { - blueprint_id: blueprint.id, - inventory_collection_id: collection.id, - stats, - }; - json!(status) - } - Err(err) => json!({ "error": - format!("rendezvous reconciliation failed: {err:#}"), - }), - } + json!(BlueprintRendezvousStatus { + blueprint_id: blueprint.id, + sled_blueprint_availability, + datasets, + }) } } diff --git a/nexus/test-utils/src/background.rs b/nexus/test-utils/src/background.rs index b9083f36940..433bc82b878 100644 --- a/nexus/test-utils/src/background.rs +++ b/nexus/test-utils/src/background.rs @@ -639,7 +639,9 @@ pub async fn run_blueprint_executor(lockstep_client: &ClientTestContext) { } /// Run the blueprint_rendezvous background task -pub async fn run_blueprint_rendezvous(lockstep_client: &ClientTestContext) { +pub async fn run_blueprint_rendezvous( + lockstep_client: &ClientTestContext, +) -> BlueprintRendezvousStatus { let last_background_task = activate_background_task(&lockstep_client, "blueprint_rendezvous") .await; @@ -653,8 +655,27 @@ pub async fn run_blueprint_rendezvous(lockstep_client: &ClientTestContext) { ); }; - let _status = serde_json::from_value::( + let status = serde_json::from_value::( last_result_completed.details, ) - .unwrap(); + .expect("parsed blueprint_rendezvous task status"); + + match &status.sled_blueprint_availability { + SledBlueprintAvailabilityRendezvousOutcome::Reconciled(_) => (), + SledBlueprintAvailabilityRendezvousOutcome::Error(error) => { + panic!("blueprint_rendezvous sled availability failed: {error}") + } + } + match &status.datasets { + DatasetRendezvousOutcome::NoInventoryCollection + | DatasetRendezvousOutcome::Reconciled { .. } => (), + DatasetRendezvousOutcome::Error { inventory_collection_id, error } => { + panic!( + "blueprint_rendezvous dataset reconciliation against \ + inventory collection {inventory_collection_id} failed: {error}" + ) + } + } + + status } diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 104bd5f2226..4dc2c3023f2 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -613,24 +613,51 @@ impl IdOrdItem for TufRepoInfo { id_upcast!(); } -/// The status of an `blueprint_rendezvous` background task activation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +/// The status of a `blueprint_rendezvous` background task activation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct BlueprintRendezvousStatus { /// ID of the target blueprint during this activation. pub blueprint_id: BlueprintUuid, - /// ID of the inventory collection used by this activation. - pub inventory_collection_id: CollectionUuid, - /// Counts of operations performed. - pub stats: BlueprintRendezvousStats, + /// Outcome of reconciling sled availability from the blueprint. + pub sled_blueprint_availability: SledBlueprintAvailabilityRendezvousOutcome, + /// Outcome of reconciling the dataset rendezvous tables from the blueprint + /// and inventory. + pub datasets: DatasetRendezvousOutcome, +} + +/// Outcome of reconciling `rendezvous_bp_sled_availability`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum SledBlueprintAvailabilityRendezvousOutcome { + /// Reconciliation ran to completion. + Reconciled(SledBlueprintAvailabilityRendezvousStats), + /// Reconciliation failed partway through; rows written before the + /// failure stay written. + Error(String), +} + +/// Outcome of reconciling the rendezvous dataset tables. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DatasetRendezvousOutcome { + /// No inventory collection has been loaded yet, so dataset reconciliation + /// was skipped. (It needs inventory to confirm a dataset exists before + /// making it available to other subsystems.) + NoInventoryCollection, + /// Reconciliation ran to completion against `inventory_collection_id`. + Reconciled { + inventory_collection_id: CollectionUuid, + stats: DatasetRendezvousStats, + }, + /// Reconciliation against `inventory_collection_id` failed partway through. + /// The rows written before the failure stay written. + Error { inventory_collection_id: CollectionUuid, error: String }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct BlueprintRendezvousStats { +pub struct DatasetRendezvousStats { pub debug_dataset: DatasetsRendezvousStats, pub crucible_dataset: CrucibleDatasetsRendezvousStats, pub local_storage_dataset: DatasetsRendezvousStats, pub local_storage_unencrypted_dataset: DatasetsRendezvousStats, - pub sled_blueprint_availability: SledBlueprintAvailabilityRendezvousStats, } /// Stats for a sled availability rendezvous run.