Skip to content

Commit cbdd6f6

Browse files
committed
fix(runtime): close capability lifecycle durably
Persist canonical ToolResult through DurableEventCommitter::commit_step for production commit_result and interrupt, retain ExecutionLease from prepare until commit, recover open tokens on stop/shutdown/drop, reject same-run retry of unresolved calls, and add authorize() for future cap::* effects.
1 parent e7b970f commit cbdd6f6

8 files changed

Lines changed: 834 additions & 18 deletions

File tree

src/capabilities/host.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ fn error_message(error: &LifecycleError) -> String {
4545
"registry identity does not match frozen snapshot".to_string()
4646
}
4747
LifecycleError::InvalidMetadata(message) => message.clone(),
48+
LifecycleError::UnresolvedCall => {
49+
"an unresolved execution token already exists for this call".to_string()
50+
}
4851
}
4952
}
5053

src/capabilities/lifecycle.rs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,14 @@ impl CapabilityLifecycle {
310310
)? {
311311
return Ok(PrepareOutcome::Replay { result });
312312
}
313+
{
314+
let unresolved = self.inner.token_states.lock().values().any(|state| {
315+
matches!(state, TokenState::Open(claims) if claims.call_id == metadata.call_id)
316+
});
317+
if unresolved {
318+
return Err(LifecycleError::UnresolvedCall);
319+
}
320+
}
313321
if self.inner.clock.now_ms() >= self.inner.deadline_ms {
314322
return Err(LifecycleError::DeadlineElapsed);
315323
}
@@ -439,6 +447,51 @@ impl CapabilityLifecycle {
439447
}
440448
}
441449

450+
/// Lookup and authorize an open execution token before a future `cap::*` effect.
451+
pub fn authorize(
452+
&self,
453+
owner: &CapabilityOwner,
454+
token: &str,
455+
requested: CapabilityRisk,
456+
) -> Result<TokenClaims, LifecycleError> {
457+
if owner != &self.inner.owner {
458+
return Err(LifecycleError::OwnerMismatch {
459+
expected: self.inner.owner.key(),
460+
actual: owner.key(),
461+
});
462+
}
463+
if self.inner.cancellation.is_cancelled() {
464+
return Err(LifecycleError::Cancelled);
465+
}
466+
let states = self.inner.token_states.lock();
467+
let claims = match states.get(token) {
468+
Some(TokenState::Open(claims)) => claims.as_ref().clone(),
469+
Some(TokenState::Committed) => return Err(LifecycleError::DuplicateClose),
470+
Some(TokenState::Interrupted) => return Err(LifecycleError::Interrupted),
471+
None => return Err(LifecycleError::TokenUnknown),
472+
};
473+
drop(states);
474+
if &claims.owner != owner {
475+
return Err(LifecycleError::OwnerMismatch {
476+
expected: claims.owner.key(),
477+
actual: owner.key(),
478+
});
479+
}
480+
if self.inner.clock.now_ms() >= claims.deadline_ms {
481+
return Err(LifecycleError::DeadlineElapsed);
482+
}
483+
if claims.generation != self.inner.generation.load(Ordering::SeqCst) {
484+
return Err(LifecycleError::Interrupted);
485+
}
486+
if requested > claims.risk_ceiling {
487+
return Err(LifecycleError::ApprovalCeiling {
488+
requested,
489+
ceiling: claims.risk_ceiling,
490+
});
491+
}
492+
Ok(claims)
493+
}
494+
442495
pub fn recover_open_tokens(&self) -> Result<Vec<String>, LifecycleError> {
443496
let mut states = self.inner.token_states.lock();
444497
let open: Vec<(String, String)> = states
@@ -485,6 +538,11 @@ impl ExecutionLease {
485538
pub fn token(&self) -> &str {
486539
&self.token
487540
}
541+
542+
/// Disarm the lease after a successful commit so Drop does not interrupt.
543+
pub fn disarm(&mut self) {
544+
self.closed = true;
545+
}
488546
}
489547

490548
impl Drop for ExecutionLease {

src/capabilities/types.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ pub enum LifecycleError {
168168
Interrupted,
169169
RegistryMismatch,
170170
InvalidMetadata(String),
171+
UnresolvedCall,
171172
}
172173

173174
impl LifecycleError {
@@ -189,6 +190,7 @@ impl LifecycleError {
189190
Self::Interrupted => "interrupted",
190191
Self::RegistryMismatch => "registry_mismatch",
191192
Self::InvalidMetadata(_) => "invalid_metadata",
193+
Self::UnresolvedCall => "unresolved_call",
192194
}
193195
}
194196
}

src/runtime/agent_host.rs

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
//! through these host functions. Provider adapters stay in RSS; this module
55
//! does not add an OpenAI-compatible inference path.
66
7-
use std::collections::VecDeque;
7+
use std::collections::{HashMap, VecDeque};
88
use std::sync::{Arc, Mutex};
99
use std::thread;
1010
use std::time::{Duration, Instant};
@@ -18,8 +18,8 @@ use serde_json::{Value as JsonValue, json};
1818

1919
use super::rss_runner::RunCancellation;
2020
use crate::capabilities::{
21-
CapabilityLifecycle, CapabilityOwner, LifecycleError, parse_prepare_metadata, tool_commit,
22-
tool_prepare,
21+
CapabilityLifecycle, CapabilityOwner, ExecutionLease, LifecycleError, parse_prepare_metadata,
22+
tool_commit, tool_prepare,
2323
};
2424
use crate::domain::{ToolCall, json_to_vm_value, vm_value_to_json};
2525
use crate::metrics::Metrics;
@@ -141,6 +141,7 @@ pub struct AgentHostState {
141141
pub metrics: Option<Arc<Metrics>>,
142142
pub lifecycle: Option<Arc<CapabilityLifecycle>>,
143143
pub capability_owner: Option<CapabilityOwner>,
144+
pub(crate) leases: Arc<Mutex<HashMap<String, ExecutionLease>>>,
144145
}
145146

146147
impl AgentHostState {
@@ -172,10 +173,21 @@ impl AgentHostState {
172173
"capability owner is not installed".to_string(),
173174
));
174175
};
175-
match parse_prepare_metadata(metadata) {
176+
let envelope = match parse_prepare_metadata(metadata) {
176177
Ok(metadata) => tool_prepare(lifecycle, owner, metadata),
177-
Err(error) => crate::capabilities::host::error_envelope(&error),
178+
Err(error) => return crate::capabilities::host::error_envelope(&error),
179+
};
180+
if envelope.get("ok") == Some(&JsonValue::Bool(true))
181+
&& envelope.get("kind") == Some(&JsonValue::String("execute".to_string()))
182+
&& let Some(token) = envelope.get("execution_token").and_then(JsonValue::as_str)
183+
&& let Ok(lease) = lifecycle.lease(token)
184+
{
185+
self.leases
186+
.lock()
187+
.unwrap_or_else(|poisoned| poisoned.into_inner())
188+
.insert(token.to_string(), lease);
178189
}
190+
envelope
179191
}
180192

181193
fn capability_commit(&self, token: &str, result: &JsonValue) -> JsonValue {
@@ -189,7 +201,17 @@ impl AgentHostState {
189201
"capability owner is not installed".to_string(),
190202
));
191203
};
192-
tool_commit(lifecycle, owner, token, result.clone())
204+
let envelope = tool_commit(lifecycle, owner, token, result.clone());
205+
if envelope.get("ok") == Some(&JsonValue::Bool(true))
206+
&& let Some(mut lease) = self
207+
.leases
208+
.lock()
209+
.unwrap_or_else(|poisoned| poisoned.into_inner())
210+
.remove(token)
211+
{
212+
lease.disarm();
213+
}
214+
envelope
193215
}
194216

195217
fn tool_dispatch(&self, call: &JsonValue) -> JsonValue {

src/runtime/rss_runner.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,7 @@ impl AgentRunner {
624624
metrics: self.host.metrics.clone(),
625625
lifecycle: self.host.lifecycle.clone(),
626626
capability_owner: self.host.capability_owner.clone(),
627+
leases: Arc::new(Mutex::new(HashMap::new())),
627628
});
628629
if let Some(cancellation) = cancellation {
629630
vm.set_epoch_check_interval(RUN_EPOCH_CHECK_INTERVAL)

src/service.rs

Lines changed: 95 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ impl NativeDispatchState {
303303
if let Some(observer) = &self.shutdown_entered {
304304
observer();
305305
}
306+
let _ = self.lifecycle.recover_open_tokens();
306307
self.dispatcher.close();
307308
let quiesced = self.dispatcher.try_quiesce(grace);
308309
let owner = self.owner();
@@ -357,6 +358,19 @@ impl RunHandle {
357358

358359
fn cancel_native_tools(&self) {
359360
self.tool_cancel.cancel();
361+
let lifecycle = {
362+
let phase = self
363+
.native_dispatch
364+
.lock()
365+
.unwrap_or_else(|poisoned| poisoned.into_inner());
366+
match &*phase {
367+
NativeDispatchPhase::Ready(state) => Some(Arc::clone(&state.lifecycle)),
368+
_ => None,
369+
}
370+
};
371+
if let Some(lifecycle) = lifecycle {
372+
let _ = lifecycle.recover_open_tokens();
373+
}
360374
}
361375

362376
fn native_dispatch_closed(&self) -> bool {
@@ -987,6 +1001,26 @@ impl AgentService {
9871001
.commit_step(event_type, data, result)
9881002
}
9891003

1004+
/// Run-scoped capability engine used by `agent_runtime::tool_prepare`
1005+
/// and `agent_runtime::tool_commit`. Initializes native dispatch if needed.
1006+
pub fn capability_lifecycle(
1007+
&self,
1008+
run_id: &str,
1009+
) -> Result<(Arc<CapabilityLifecycle>, CapabilityOwner), RunContextError> {
1010+
let handle = self
1011+
.handle(run_id)
1012+
.ok_or_else(|| RunContextError::Missing {
1013+
run_id: run_id.to_string(),
1014+
})?;
1015+
match self.native_dispatch_state(run_id, &handle)? {
1016+
Some(state) => Ok((Arc::clone(&state.lifecycle), state.capability_owner.clone())),
1017+
None => Err(RunContextError::InvalidMetadata {
1018+
run_id: run_id.to_string(),
1019+
reason: "native dispatch is closed".to_string(),
1020+
}),
1021+
}
1022+
}
1023+
9901024
/// Serial, validated native dispatch against the admitted registry snapshot.
9911025
///
9921026
/// The live registry is not consulted. Durable event append uses the same
@@ -2952,6 +2986,7 @@ impl AgentService {
29522986
// observing the cancellation commits exactly this reason.
29532987
*handle.cancel_reason.lock().expect("cancel reason lock") = Some("requested");
29542988
handle.cancel.request(CancellationReason::Requested);
2989+
drop(store);
29552990
handle.cancel_native_tools();
29562991
tracing::debug!(
29572992
run_id,
@@ -4222,14 +4257,21 @@ impl DurableToolLifecycle for ServiceDurableLifecycle {
42224257
call_id: &str,
42234258
result: &serde_json::Value,
42244259
) -> Result<serde_json::Value, LifecycleError> {
4260+
let tool_result = canonical_tool_result(result)?;
4261+
let event_type = if tool_result.ok {
4262+
"tool.completed"
4263+
} else {
4264+
"tool.failed"
4265+
};
4266+
let mut data = json!({
4267+
"tool_call_id": call_id,
4268+
"ok": tool_result.ok,
4269+
});
4270+
if let Some(error) = &tool_result.error {
4271+
data["error_code"] = json!(error.code);
4272+
}
42254273
self.events
4226-
.commit(
4227-
"tool.completed",
4228-
json!({
4229-
"tool_call_id": call_id,
4230-
"result": result,
4231-
}),
4232-
)
4274+
.commit_step(event_type, data, Some(&tool_result))
42334275
.map_err(|error| match error {
42344276
EventCommitError::PersistFailed(message) => {
42354277
LifecycleError::ResultCommitFailed(message)
@@ -4240,16 +4282,17 @@ impl DurableToolLifecycle for ServiceDurableLifecycle {
42404282
}
42414283

42424284
fn interrupt(&self, call_id: &str) -> Result<(), LifecycleError> {
4285+
let tool_result =
4286+
ToolResult::failure("interrupted_effect", "effect interrupted by restart");
42434287
self.events
4244-
.commit(
4288+
.commit_step(
42454289
"tool.failed",
42464290
json!({
42474291
"tool_call_id": call_id,
4248-
"error": {
4249-
"code": "interrupted",
4250-
"message": "execution was interrupted",
4251-
}
4292+
"error_code": "interrupted_effect",
4293+
"ok": false,
42524294
}),
4295+
Some(&tool_result),
42534296
)
42544297
.map_err(map_event_commit_error)
42554298
}
@@ -4265,6 +4308,46 @@ fn map_event_commit_error(error: EventCommitError) -> LifecycleError {
42654308
}
42664309
}
42674310

4311+
fn canonical_tool_result(result: &JsonValue) -> Result<ToolResult, LifecycleError> {
4312+
let ok = result
4313+
.get("ok")
4314+
.and_then(JsonValue::as_bool)
4315+
.unwrap_or(false);
4316+
if ok {
4317+
let mut tool_result = ToolResult::success(
4318+
result
4319+
.get("content")
4320+
.and_then(JsonValue::as_str)
4321+
.unwrap_or("")
4322+
.to_string(),
4323+
result.get("data").cloned().unwrap_or_else(|| json!({})),
4324+
);
4325+
tool_result.truncated = result
4326+
.get("truncated")
4327+
.and_then(JsonValue::as_bool)
4328+
.unwrap_or(false);
4329+
if let Some(artifacts) = result.get("artifacts").and_then(JsonValue::as_array) {
4330+
tool_result.artifacts = artifacts
4331+
.iter()
4332+
.filter_map(JsonValue::as_str)
4333+
.map(str::to_string)
4334+
.collect();
4335+
}
4336+
Ok(tool_result)
4337+
} else {
4338+
let error = result.get("error");
4339+
let code = error
4340+
.and_then(|value| value.get("code"))
4341+
.and_then(JsonValue::as_str)
4342+
.unwrap_or("tool_failed");
4343+
let message = error
4344+
.and_then(|value| value.get("message"))
4345+
.and_then(JsonValue::as_str)
4346+
.unwrap_or("tool failed");
4347+
Ok(ToolResult::failure(code, message))
4348+
}
4349+
}
4350+
42684351
struct ServiceEventCommitter {
42694352
store: Arc<RwLock<GatewayStore>>,
42704353
persistence: Option<Arc<GatewayPersistence>>,

0 commit comments

Comments
 (0)