Repository navigation
feat(session): revert crashed runtimes through RevertSession - #3395
Conversation
bc2b0f4 to
dbd3b00
Compare
dbd3b00 to
aac74f8
Compare
aac74f8 to
dfc3bcf
Compare
| func (w *ActorWorkflow) recoverCrashedSession(ctx context.Context, operation *database.SessionOperation) (_ *apiv1.Session, err error) { | ||
| session := operation.Instance | ||
| if session.Operation != apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME { | ||
| return nil, database.ErrConflict | ||
| } | ||
| executorID := uuid.New() | ||
| claimed, err := w.store.ClaimSessionOperation(ctx, session.Id, operation.ID, executorID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if !claimed { | ||
| current, err := w.store.GetSessionOperation(ctx, session.Id, operation.ID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return operationOutcome(current) | ||
| } | ||
| defer func() { | ||
| if err != nil { | ||
| releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) | ||
| defer cancel() | ||
| err = errors.Join(err, w.store.ReleaseRuntimeOperation(releaseCtx, session.Id, operation.ID, executorID)) | ||
| } | ||
| }() | ||
| revision, err := w.store.GetRuntimeRevision(ctx, session.PreparedRevision) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| atespace, name := revision.ActorTemplateAtespace, substrate.ActorName(session.Id) | ||
| actor, err := w.actors.GetActor(ctx, atespace, name) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if err := w.verifyActor(ctx, session, revision, actor); err != nil { | ||
| return nil, err | ||
| } | ||
| before := actor | ||
| beforeSnapshot := proto.CloneOf(substrate.LatestDurableActorSnapshot(actor)) | ||
| if beforeSnapshot != nil { | ||
| beforeSnapshot.Locality = "" | ||
| } | ||
| switch actor.GetStatus().GetState() { | ||
| case ateapipb.ActorState_ACTOR_STATE_CRASHED, ateapipb.ActorState_ACTOR_STATE_REVERTING: | ||
| actor, err = w.actors.RevertActor(ctx, atespace, name) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("revert crashed Actor %s/%s: %w", atespace, name, err) | ||
| } | ||
| case ateapipb.ActorState_ACTOR_STATE_SUSPENDED: | ||
| default: | ||
| return nil, fmt.Errorf("crashed Actor %s/%s has unexpected state %s", atespace, name, actor.GetStatus().GetState()) | ||
| } | ||
| if !validActorIdentity(actor, revision, name) || actor.GetMetadata().GetUid() != before.GetMetadata().GetUid() || actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_SUSPENDED || !proto.Equal(beforeSnapshot, substrate.LatestActorSnapshot(actor)) { | ||
| return nil, fmt.Errorf("revert Actor %s/%s returned unexpected identity or state", atespace, name) | ||
| } | ||
| finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) | ||
| defer cancel() | ||
| return w.store.FinishSessionOperation(finishCtx, session.Id, operation.ID, executorID, "", actor.GetMetadata().GetUid(), "") | ||
| } |
There was a problem hiding this comment.
Could recovery just reuse execute?
This looks like a second copy of execute: claim, deferred release, GetActor, verifyActor, mutate, FinishSessionOperation. It also pulls the Substrate state handling (CRASHED/REVERTING → RevertActor, etc.) up into the service layer. Lifecycle should own lifecycle: suspend/resume/delete already keep those decisions in substrate/lifecycle.go, and the service just sequences the steps. Crash recovery feels like it belongs there too. Having one execute path also means fixes land everywhere at once (see the claim comment below).
I tried folding it in: a PrepareActorRevert + revert branch in ApplyActorTransition, and execute picks it when the operation is a confirmed crash recovery:
crashed := operation.Instance.State == FAILED && operation.Instance.Failure.GetReason() == database.SessionRuntimeCrashed
...
if crashed {
// Admission already settled the interrupted turn; keep recovery intent.
if transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding); err != nil {
return nil, err
}
} else if transition, err = prepare(ctx, w.actors, binding, kind, creation); err != nil {
return w.failPreparation(ctx, operation, err)
}RevertActor moves onto LifecycleClient and this function goes away (+47/−69). All of your recovery tests still pass, plus the existing quiescence/sandbox/substrate ones. Was there a reason to keep it separate that I'm missing?
There was a problem hiding this comment.
No good reason to keep it separate, so I took your shape in 05cfb0c. substrate.PrepareActorRevert does the read-only checks, ApplyActorTransition issues the revert, and RevertActor moves onto LifecycleClient. The checks are the same as before: same UID, ends SUSPENDED, restores the snapshot it observed. Execute keeps the recovery intent when preparation fails, as in your snippet, but I put that in failPreparation, so the UID check after preparation keeps the intent too. It also fixes something we both missed: the old branch at the top of execute sent every operation on a crashed Session into recovery, so even a superseding Delete would have come back as a conflict. TestSessionRevertCrashedRuntime/kagent passes against a controller built with this change (log, RevertSession spans).
| claimed, err := w.store.ClaimSessionOperation(ctx, session.Id, operation.ID, executorID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if !claimed { | ||
| current, err := w.store.GetSessionOperation(ctx, session.Id, operation.ID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return operationOutcome(current) | ||
| } | ||
| defer func() { | ||
| if err != nil { | ||
| releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) | ||
| defer cancel() | ||
| err = errors.Join(err, w.store.ReleaseRuntimeOperation(releaseCtx, session.Id, operation.ID, executorID)) | ||
| } | ||
| }() | ||
| revision, err := w.store.GetRuntimeRevision(ctx, session.PreparedRevision) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| atespace, name := revision.ActorTemplateAtespace, substrate.ActorName(session.Id) | ||
| actor, err := w.actors.GetActor(ctx, atespace, name) |
There was a problem hiding this comment.
Sessions can get stuck undeletable here.
This claims before doing its reads (GetRuntimeRevision/GetActor). Release keeps executor_id, so DELETE gets refused (runtime_instances.go:84) even though we never actually called RevertActor. One flaky GetActor and the Session is stuck until recovery succeeds. If the Actor is gone, it's stuck for good (and the scan logs an error every interval). Repro'd against Postgres:
after failed attempt: state=RUNTIME_STATE_FAILED op=RUNTIME_OPERATION_RESUME
ERROR recover crashed session ... error="rpc error: code = NotFound desc = missing"
delete err=begin Session operation: resource conflict
execute doesn't have this problem since it reads before it claims, so reusing it (comment above) fixes this too. Either way, I think we should either read before claiming or let DELETE win over a recovery that hasn't been issued yet. Thoughts?
There was a problem hiding this comment.
Thanks for the repro. Fixed by reading before the claim (05cfb0c). A failed or missing Actor read now leaves executor_id unset, so Delete can supersede the recovery. TestDeleteSupersedesUnissuedCrashRecovery covers both the unreadable and the missing Actor, and on the previous code it fails with your error (before). On kind I admitted a recovery, scaled ate-api to 0 so the scan's read failed, then called DeleteSession. The new controller accepts the Delete and finishes it once Substrate is back. The old one refuses it (new, old, spans):
new: DeleteSession while Substrate is down: code=Unavailable
new: DeleteSession after Substrate returns: attempts=3 code=OK; final GetSession: code=NotFound
old: DeleteSession while Substrate is down: code=Aborted desc = begin Session operation: resource conflict
old: DeleteSession after Substrate returns: attempts=61 code=Aborted (2 minutes)
If a revert was actually issued, Delete still waits for it to resolve. That case is uncertain work, not a stuck read.
| var recoveryAfter string | ||
| nextRecovery := time.Now().Add(w.recoveryInterval) | ||
| for ctx.Err() == nil { | ||
| if worker == 0 && !time.Now().Before(nextRecovery) { |
There was a problem hiding this comment.
+1, I couldn't find one either. The tests call recoverSessionPage directly, and the Start-level stub returns no candidates.
Also, the scan runs inline on worker 0 between quiescence claims, so a slow idle suspension delays crash retries and vice versa. Would its own goroutine + ticker be simpler to reason about and test?
2e4e461 to
7ee6d63
Compare
72d68a3 to
9f76a9a
Compare
When Substrate reports a Session's Actor CRASHED, the workflow can now revert it to its last durable snapshot. `BeginCrashedSessionRecovery` admits the recovery under the Session lock. It requires the recorded Actor UID, joins an already admitted recovery, accepts only an idle READY Session whose lifecycle generation still matches the one read before the crash was observed, and refuses while a checkpoint is being created. In the same transaction it fails the interrupted turn with a timeline-positioned crash note, keeps waits the snapshot captured, and starts a RESUME operation. `SettleSessionTask` is split into `settleSessionTask(tx, ...)` so admission can reuse it under the lock. `observeCrash` is the one check that confirms a crash. A missing, replaced, or retemplated Actor is unrecoverable rather than crashed. `execute` reverts in place for an admitted crash, and `failPreparation` keeps that intent instead of clearing it, because the turn has already failed. Admitted recoveries are retried on their own ticker through `scanRecovery`, one ID-ordered page per interval, so a slow idle suspension cannot delay them. Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Expose crash recovery as `SessionService.RevertSession`. The request validates the Session ID as a UUID, and the method needs update access, so a read-only share cannot invoke it. `Service.Revert` refuses with FAILED_PRECONDITION when there is nothing to revert, and an ErrorInfo reason tells the two cases apart. `KAGENT_SESSION_RUNTIME_NOT_CRASHED` means the runtime is healthy or at rest. `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` means the recorded Actor is gone or was replaced. Contention stays ABORTED and transient failures stay UNAVAILABLE. Adds API E2E coverage for the revert and documents manual recovery in the lifecycle architecture notes. Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
A manual revert ran 27 SQL statements before `FinishSessionOperation`. It now runs 19, with the same three Substrate calls and the same two transactions. - `SessionRecoveryFence` reads the lifecycle generation, the recorded Actor UID, and the Actor template in one statement, before the Actor is observed. It replaces `SessionLifecycleGeneration`, the observation's `GetRuntimeRevision`, and `verifyActor`'s post-observation `GetSessionForRuntime`. `observeCrash` checks the observed Actor against an `ActorBinding` built from the fence and compares UIDs. The recorded UID changes only through a lifecycle operation, which also replaces the generation, so admission's locked generation check still rejects a stale observation. - Admission reads the Actor-UID guard and the checkpoint-CREATING guard in one statement under the Session lock. It still evaluates them in their original order. - One CTE clears the history's pending quiescence marker and selects each interrupted task with its latest version and pending-boundary flag. That replaces a separate clear, a task-ID read, and two reads per task. A task is re-read only after a pending boundary was just settled, and a task with no events is still rejected. Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
…admitted check Review follow-ups on #3395: - SessionRecoveryFence becomes SessionGeneration, read by GetSessionGeneration, matching the lifecycle code's name for operation_id. BeginCrashedSessionRecovery takes the whole read back, so the store compares both the generation and the recorded Actor UID, and the caller only holds onto it. - database.CrashRecoveryAdmitted replaces three copies of the admitted-recovery check, and every caller now requires the recovery's RESUME to be in flight. FinishSessionOperation replaces the failure on every finish, so the copies only differed while a Delete supersedes the recovery, where both return a conflict. - Revert adds its timeout only on the admission path. run already bounds a joined attempt. No query changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
Review follow-ups on #3395: - RUNTIME_OPERATION_REVERT replaces the revert flag on ActorTransition and the FAILED + RESUME + RuntimeCrashed encoding. CrashRecoveryAdmitted is now Operation == REVERT, revert is a case in ApplyActorTransition, and BeginSessionOperation can join a REVERT but never start one. The runtime_instance operation CHECK accepts the new value. - Recovery no longer runs on a background ticker. An admitted recovery is a durable REVERT operation that a retried RevertSession joins, the way Create and Delete retries work. ListSessionRecoveryCandidates, the scan functions, and the recovery goroutine in Start are removed. - Admission pins the durable snapshot URI it settled waits against, keyed by the REVERT operation ID. PrepareActorRevert requires the Actor to still report that snapshot, so a snapshot uploaded after admission cannot bring back a wait admission already failed. - A recovery whose Actor is missing, replaced, or holds another snapshot is abandoned even after an earlier attempt claimed it: no revert of the recorded runtime can still be in progress. The Session stays FAILED with reason RuntimeUnrecoverable and no operation, so Delete proceeds. A live claim still wins. - RevertSession returns the Session when it is READY and its recorded Actor is SUSPENDED, so a retry after a lost success response succeeds instead of reporting KAGENT_SESSION_RUNTIME_NOT_CRASHED. Docs describe what a client sees while a recovery is pending. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
The previous commit edited 000001_initial.sql in place, which the required migration-immutability check rejects: merged migrations are immutable. Restore 000001 and add 000002_session_revert_operation.sql, which widens the runtime_instance operation CHECK to accept RUNTIME_OPERATION_REVERT and adds session.recovery_operation_id and recovery_snapshot_uri. Down refuses while a recovery is in flight rather than rewriting its operation. The database README still told contributors to fold changes into migration 1, which contradicts the required check; it now says to add a new migration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
PrepareActorRevert checked the Actor's state before its identity, and the recorded UID only after preparation. A same-name replacement in any state other than CRASHED, REVERTING, or SUSPENDED therefore failed with a transient error. When an earlier attempt had claimed the revert, the operation was never abandoned and Delete stayed refused, which is the stuck Session review point 2 describes. The recovery pin now carries the recorded Actor UID with the snapshot URI, and PrepareActorRevert checks identity first, so any replacement is unrecoverable. Execute selects the revert through CrashRecoveryAdmitted, the one shared check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
Only BeginCrashedSessionRecovery starts a REVERT, and it writes the snapshot URI in the same transaction, so the stored URI always belongs to the current REVERT. recovery_operation_id, its paired CHECK, and the operation-ID join guarded a stale read that cannot happen. PrepareActorRevert checks the recorded Actor UID itself, so the REVERT-specific remapping of execute's later UID check was unreachable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
Delete supersedes an admitted REVERT even after an attempt issued it. Issued work blocks Delete because it may still create something Delete would miss; a revert restores the recorded Actor in place and creates nothing, and Substrate serializes the two calls. That removes AbandonSessionRecovery, the RuntimeUnrecoverable failure reason, and the early check in Revert. A retry against a gone or replaced Actor still returns KAGENT_SESSION_RUNTIME_UNRECOVERABLE without storing anything, so the client deletes the Session. RevertSession now returns the Session whenever the runtime is not crashed, at rest or running, as Create and Delete retries return current state. KAGENT_SESSION_RUNTIME_NOT_CRASHED and its mapping are gone; UNRECOVERABLE is the only refusal. The chat reading for a pending revert moves to the UI PR. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
Remove comments added for crash recovery that narrate the next lines, restate helper names, or repeat test assertions. Keep the ordering, concurrency, and safety notes, shortened. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
… ErrorInfo Crash settlement now reuses deletion's two common steps: publishSavedSessionResults publishes results whose cleanup acknowledgement was lost, and retireSessionQuiescence drops idle-work markers. finishDeletedSessionTasks calls the same helpers. The crash path keeps only what differs: it fails the open work the revert discards and keeps waits the snapshot captured. This replaces the CTE that cleared markers while reading tasks, and the re-read after settling. RevertSession's unrecoverable refusal is a plain FAILED_PRECONDITION. Nothing reads the KAGENT_SESSION_RUNTIME_UNRECOVERABLE reason, and a revert of a healthy runtime now succeeds, so the status alone says the runtime cannot be recovered. Revert maps errors exactly like Delete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
Substrate's RevertActor restores the Actor's last completed durable snapshot, and kagent suspends with DURABILITY_MODE_SYNCHRONOUS, so an upload completes before SuspendActor returns. A crashed Actor cannot be suspended, and a revert takes no snapshot. The snapshot admission settles against is therefore the one the revert restores, and the pin guarded a case that cannot happen. applyRevert still requires the reverted Actor to report the snapshot it held before. Removed: session.recovery_snapshot_uri from migration 000002, SessionRecoveryPin and GetSessionRecoveryPin, and the "durable snapshot changed after recovery was admitted" refusal. That refusal was the only remaining path to a FAILED/REVERT Session that could only be deleted. PrepareActorRevert now only reads the Actor and checks its name and template. execute's existing recorded-UID check reports a replacement as unrecoverable, and applyRevert handles the Actor's state. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
applyRevert compared the Actor's durable snapshot before and after RevertActor. That comparison cannot fail. Substrate's revert keeps the latest completed durable snapshot and discards pending uploads (workflow_revert.go, ensureInProgressSnapshotDiscarded). Every suspend of a kagent Actor is synchronous: kagent's SuspendActor passes DURABILITY_MODE_SYNCHRONOUS, and so does Substrate's template reconciler. Atelet's RequestActorSuspend, the only asynchronous path, has no caller in kagent, its guest library, or Substrate at v1.0.0-rc4. So no durable upload can complete between admission and the revert. applyRevert now requires only the same Actor back, SUSPENDED. That also answers the review question on #3395. Settling at admission is deliberate: the snapshot it settles against is the one the revert restores, and the interrupted turn fails as soon as the crash is confirmed. An asynchronous suspend would break this, and settlement would then move to when the revert finishes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
The actor_uid comparison in BeginCrashedSessionRecovery was redundant. The recorded UID only changes through a lifecycle operation, and every operation replaces operation_id, so the generation check already rejects a stale UID. It also had an edge case: with actor_uid NULL, `actor_uid = $2` scanned NULL into a bool, so the caller got Unavailable instead of a retryable Conflict. The query now checks only for a checkpoint being created. Same-name replacements are still caught before admission, in observeCrash. TestSuspendActorWaitsForLatestDurableSnapshot already asserts DURABILITY_MODE_SYNCHRONOUS. It now says crash recovery depends on that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Ada Lundhe <adalundhe@lundhe.audio>
52c709a to
b802704
Compare
Summary
When Substrate reports a Session's Actor
CRASHED, a caller can revert it to its last durable snapshot throughSessionService.RevertSession. Recovery runs only when a client asks: an admitted recovery is a durableRUNTIME_OPERATION_REVERToperation, and a retriedRevertSessionjoins it, the same way Create and Delete retries work.How a revert works
observeCrashreads the Session's generation (lifecycle operation ID, recorded Actor UID, Actor template) in one statement, then reads the Actor from Substrate. A missing, replaced, or retemplated Actor is unrecoverable, never a crash.BeginCrashedSessionRecoveryruns one transaction under the Session lock. It:executepath reverts the Actor in place. Substrate restores the last completed durable snapshot. Every suspend of a kagent Actor is synchronous, so that is the snapshot admission settled against.applyRevertrequires the same Actor back, SUSPENDED. If preparation fails, the REVERT is kept for a retry.API
RevertSessionvalidatessession_idas a UUID and needsAccessUpdate, so a read-only share cannot invoke it.FAILED_PRECONDITION. Contention returnsABORTED, and transient failures returnUNAVAILABLE.Schema
000002_session_revert_operation.sqladdsRUNTIME_OPERATION_REVERTto theruntime_instanceoperation CHECK.Testing
make -C go lint,make proto-generatego testforservice/session,substrate,grpcserver(revert), andpkg/migrations(up/down round trip)TestInlineSQLPreparesgo/core/test/e2e/session_revert_test.goStack: this PR → #3399 (GetSession recovery report and chat).
🤖 Generated with Claude Code