Skip to content

[10/n] [reconfigurator] reconcile sled availability without waiting on inventory #11219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sunshowers wants to merge 2 commits into
base: sunshowers/spr/main.10n-reconfigurator-reconcile-sled-availability-without-waiting-on-inventory
Choose a base branch
from
Open

[10/n] [reconfigurator] reconcile sled availability without waiting on inventory #11219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5274aaf
[spr] initial version
sunshowers Sep 1, 2026
f97197e
fix build
sunshowers Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 123 additions & 71 deletions dev-tools/omdb/src/bin/omdb/nexus.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ use nexus_types::internal_api::background::AttachedSubnetManagerStatus;
use nexus_types::internal_api::background::AuditLogCleanupStatus;
use nexus_types::internal_api::background::AuditLogTimeoutIncompleteStatus;
use nexus_types::internal_api::background::BlueprintPlannerStatus;
use nexus_types::internal_api::background::BlueprintRendezvousStats;
use nexus_types::internal_api::background::BlueprintRendezvousStatus;
use nexus_types::internal_api::background::DatasetRendezvousOutcome;
use nexus_types::internal_api::background::DatasetRendezvousStats;
use nexus_types::internal_api::background::DatasetsRendezvousStats;
use nexus_types::internal_api::background::EreporterStatus;
use nexus_types::internal_api::background::FmAnalysisStatus;
Expand All @@ -83,6 +84,8 @@ use nexus_types::internal_api::background::ServiceFirewallRuleStatus;
use nexus_types::internal_api::background::SessionCleanupStatus;
use nexus_types::internal_api::background::SitrepGcStatus;
use nexus_types::internal_api::background::SitrepLoadStatus;
use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousOutcome;
use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousStats;
use nexus_types::internal_api::background::SupportBundleActivationReport;
use nexus_types::internal_api::background::SupportBundleCleanupReport;
use nexus_types::internal_api::background::SupportBundleCollectionStepStatus;
Expand Down Expand Up @@ -1696,82 +1699,131 @@ fn print_task_blueprint_rendezvous(details: &serde_json::Value) {
error, details
),
Ok(status) => {
println!(" target blueprint: {}", status.blueprint_id);
println!(
" inventory collection: {}",
status.inventory_collection_id
);

let BlueprintRendezvousStats {
debug_dataset,
crucible_dataset,
local_storage_dataset,
local_storage_unencrypted_dataset,
let BlueprintRendezvousStatus {
blueprint_id,
sled_blueprint_availability,
} = status.stats;
datasets,
} = status;
println!(" target blueprint: {blueprint_id}");

print_datasets_rendezvous_stats(&debug_dataset, "debug_dataset");
match datasets {
DatasetRendezvousOutcome::NoInventoryCollection => {
println!(
" inventory collection: none loaded yet; dataset \
reconciliation skipped"
);
}
DatasetRendezvousOutcome::Error {
inventory_collection_id,
error,
} => {
println!(
" inventory collection: {inventory_collection_id}"
);
println!(" dataset reconciliation failed: {error}");
}
DatasetRendezvousOutcome::Reconciled {
inventory_collection_id,
stats,
} => {
println!(
" inventory collection: {inventory_collection_id}"
);
print_dataset_rendezvous_stats(&stats);
}
}

// crucible datasets have a different number of rendezvous stats
println!(" crucible_dataset rendezvous counts:");
println!(
" num_inserted: {}",
crucible_dataset.num_inserted
);
println!(
" num_already_exist: {}",
crucible_dataset.num_already_exist
);
println!(
" num_not_in_inventory: {}",
crucible_dataset.num_not_in_inventory
);
match sled_blueprint_availability {
SledBlueprintAvailabilityRendezvousOutcome::Error(error) => {
println!(
" sled_blueprint_availability reconciliation \
failed: {error}"
);
}
SledBlueprintAvailabilityRendezvousOutcome::Reconciled(
stats,
) => {
print_sled_blueprint_availability_rendezvous_stats(&stats);
}
}
}
}
}

print_datasets_rendezvous_stats(
&local_storage_dataset,
"local_storage_dataset",
);
fn print_dataset_rendezvous_stats(stats: &DatasetRendezvousStats) {
let DatasetRendezvousStats {
debug_dataset,
crucible_dataset,
local_storage_dataset,
local_storage_unencrypted_dataset,
} = stats;

print_datasets_rendezvous_stats(
&local_storage_unencrypted_dataset,
"local_storage_unencrypted_dataset",
);
print_datasets_rendezvous_stats(debug_dataset, "debug_dataset");

println!(" sled_blueprint_availability rendezvous counts:");
println!(
" num_marked_available: {}",
sled_blueprint_availability.num_marked_available
);
println!(
" num_marked_unavailable: {}",
sled_blueprint_availability.num_marked_unavailable
);
println!(
" num_unchanged: {}",
sled_blueprint_availability.num_unchanged
);
println!(
" num_invariant_violations: {}",
sled_blueprint_availability.num_invariant_violations
);
println!(
" num_decommissioned: {}",
sled_blueprint_availability.num_decommissioned
);
println!(
" num_already_decommissioned: {}",
sled_blueprint_availability.num_already_decommissioned
);
println!(
" num_not_in_blueprint: {}",
sled_blueprint_availability.num_not_in_blueprint
);
println!(
" num_decommissioned_not_in_blueprint: {}",
sled_blueprint_availability.num_decommissioned_not_in_blueprint
);
}
}
// crucible datasets have a different number of rendezvous stats
println!(" crucible_dataset rendezvous counts:");
println!(" num_inserted: {}", crucible_dataset.num_inserted);
println!(
" num_already_exist: {}",
crucible_dataset.num_already_exist
);
println!(
" num_not_in_inventory: {}",
crucible_dataset.num_not_in_inventory
);

print_datasets_rendezvous_stats(
local_storage_dataset,
"local_storage_dataset",
);

print_datasets_rendezvous_stats(
local_storage_unencrypted_dataset,
"local_storage_unencrypted_dataset",
);
}

fn print_sled_blueprint_availability_rendezvous_stats(
stats: &SledBlueprintAvailabilityRendezvousStats,
) {
let SledBlueprintAvailabilityRendezvousStats {
num_marked_available,
num_marked_unavailable,
num_unchanged,
num_invariant_violations,
num_decommissioned,
num_already_decommissioned,
num_not_in_blueprint,
num_decommissioned_not_in_blueprint,
} = stats;

println!(" sled_blueprint_availability rendezvous counts:");
println!(
" num_marked_available: {num_marked_available}"
);
println!(
" num_marked_unavailable: \
{num_marked_unavailable}"
);
println!(" num_unchanged: {num_unchanged}");
println!(
" num_invariant_violations: \
{num_invariant_violations}"
);
println!(
" num_decommissioned: {num_decommissioned}"
);
println!(
" num_already_decommissioned: \
{num_already_decommissioned}"
);
println!(
" num_not_in_blueprint: {num_not_in_blueprint}"
);
println!(
" num_decommissioned_not_in_blueprint: \
{num_decommissioned_not_in_blueprint}"
);
}

fn print_task_dns_config(details: &serde_json::Value) {
Expand Down
46 changes: 30 additions & 16 deletions nexus/reconfigurator/rendezvous/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ use nexus_db_queries::db::DataStore;
use nexus_db_queries::db::model::SledBlueprintAvailabilityInput;
use nexus_types::deployment::Blueprint;
use nexus_types::deployment::BlueprintDatasetDisposition;
use nexus_types::internal_api::background::BlueprintRendezvousStats;
use nexus_types::internal_api::background::DatasetRendezvousStats;
use nexus_types::internal_api::background::SledBlueprintAvailabilityRendezvousStats;
use nexus_types::inventory::Collection;

mod crucible_dataset;
Expand All @@ -22,12 +23,37 @@ mod local_storage_dataset;
mod local_storage_unencrypted_dataset;
mod sled_blueprint_availability;

pub async fn reconcile_blueprint_rendezvous_tables(
/// Reconcile the `rendezvous_sled_bp_availability` table for the given
/// blueprint.
///
/// This is a pure projection of the target blueprint and does not depend on
/// inventory.
pub async fn reconcile_sled_blueprint_availability(
opctx: &OpContext,
datastore: &DataStore,
blueprint: &Blueprint,
) -> anyhow::Result<SledBlueprintAvailabilityRendezvousStats> {
let sled_inputs =
SledBlueprintAvailabilityInput::all_from_blueprint(blueprint);
sled_blueprint_availability::reconcile(
opctx,
datastore,
blueprint.id,
sled_inputs,
)
.await
}

/// Reconcile the dataset rendezvous tables for the given blueprint.
///
/// These need inventory to confirm that a dataset exists before other
/// subsystems may use it.
pub async fn reconcile_dataset_rendezvous_tables(
opctx: &OpContext,
datastore: &DataStore,
blueprint: &Blueprint,
inventory: &Collection,
) -> anyhow::Result<BlueprintRendezvousStats> {
) -> anyhow::Result<DatasetRendezvousStats> {
let inventory_dataset_ids = inventory
.sled_agents
.iter()
Expand Down Expand Up @@ -79,23 +105,11 @@ pub async fn reconcile_blueprint_rendezvous_tables(
)
.await?;

let sled_inputs =
SledBlueprintAvailabilityInput::all_from_blueprint(blueprint);
let sled_blueprint_availability =
sled_blueprint_availability::reconcile_sled_blueprint_availability(
opctx,
datastore,
blueprint.id,
sled_inputs,
)
.await?;

Ok(BlueprintRendezvousStats {
Ok(DatasetRendezvousStats {
debug_dataset,
crucible_dataset,
local_storage_dataset,
local_storage_unencrypted_dataset,
sled_blueprint_availability,
})
}

Expand Down
6 changes: 3 additions & 3 deletions nexus/reconfigurator/rendezvous/src/sled_blueprint_availability.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ use slog::info;
///
/// `blueprint_sleds` should contain a [`SledBlueprintAvailabilityInput`] for
/// every sled in the blueprint, including decommissioned sleds.
pub(crate) async fn reconcile_sled_blueprint_availability(
pub(crate) async fn reconcile(
opctx: &OpContext,
datastore: &DataStore,
blueprint_id: BlueprintUuid,
Expand Down Expand Up @@ -552,7 +552,7 @@ mod tests {
&prep,
).await;

let result_stats = reconcile_sled_blueprint_availability(
let result_stats = reconcile(
opctx,
datastore,
blueprint_id,
Expand Down Expand Up @@ -623,7 +623,7 @@ mod tests {
.await
.expect("seeded the stored row");

let stats = reconcile_sled_blueprint_availability(
let stats = reconcile(
opctx,
datastore,
bp_reconciled,
Expand Down
8 changes: 7 additions & 1 deletion nexus/src/app/background/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -676,7 +676,13 @@ impl BackgroundTasksInitializer {
),
),
opctx: opctx.child(BTreeMap::new()),
watchers: vec![Box::new(inventory_load_watcher.clone())],
// A new target blueprint must reach the sled availability table
// promptly, even if inventory is stalled for whatever reason, so
// watch both channels.
watchers: vec![
Box::new(rx_blueprint.clone()),
Box::new(inventory_load_watcher.clone()),
],
activator: task_blueprint_rendezvous,
});

Expand Down
Loading
Loading