From c5a592bbfe5d30875ec816898e6d922fe12e33a6 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Wed, 4 Nov 2026 13:25:53 -0800 Subject: [PATCH 01/15] feat(substrate): revert a crashed Actor to its last durable snapshot Add the Substrate primitives crash recovery needs: - `RevertActor` calls ate-api's revert. - `PrepareActorRevert` does the read-only checks before the store's durable claim. It requires the bound Actor, in CRASHED, REVERTING (an interrupted revert), or SUSPENDED (a revert whose response was lost). - `applyRevert` reverts only a CRASHED or REVERTING Actor. It then requires the same UID, SUSPENDED, and the durable snapshot observed before the claim. - `LatestDurableActorSnapshot` picks the newest snapshot a revert can restore. Signed-off-by: Ada Lundhe Co-Authored-By: Claude Opus 5.5 --- go/core/internal/substrate/client.go | 10 +++ go/core/internal/substrate/lifecycle.go | 51 +++++++++++++ go/core/internal/substrate/lifecycle_test.go | 77 ++++++++++++++++++++ go/core/internal/substrate/snapshot.go | 11 +++ go/core/internal/substrate/snapshot_test.go | 20 +++++ 5 files changed, 169 insertions(+) diff --git a/go/core/internal/substrate/client.go b/go/core/internal/substrate/client.go index 1db776de1a..002916413d 100644 --- a/go/core/internal/substrate/client.go +++ b/go/core/internal/substrate/client.go @@ -203,6 +203,16 @@ func (c *Client) ResumeActor(ctx context.Context, atespace, actorID string) (*at return resp.GetActor(), nil } +func (c *Client) RevertActor(ctx context.Context, atespace, actorID string) (*ateapipb.Actor, error) { + ctx, cancel := c.callCtx(ctx) + defer cancel() + resp, err := c.ControlClient.RevertActor(ctx, &ateapipb.RevertActorRequest{Actor: actorRef(atespace, actorID)}) + if err != nil { + return nil, fmt.Errorf("failed to revert Actor %s/%s: %w", atespace, actorID, err) + } + return resp.GetActor(), nil +} + // SuspendActor waits for durable storage, including an upload left by an earlier // attempt. Substrate's synchronous mode returns already-suspended Actors as-is. func (c *Client) SuspendActor(ctx context.Context, atespace, actorID string) (*ateapipb.Actor, error) { diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index b5f2e6b436..ff243a77e3 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -19,6 +19,7 @@ type LifecycleClient interface { CheckActorEgressPolicy(context.Context, string, string) error EnsureActorEgressPolicy(context.Context, string, string, string) error ResumeActor(context.Context, string, string) (*ateapipb.Actor, error) + RevertActor(context.Context, string, string) (*ateapipb.Actor, error) SuspendActor(context.Context, string, string) (*ateapipb.Actor, error) DeleteActor(context.Context, string, string, string) error } @@ -51,6 +52,9 @@ type ActorTransition struct { operation apiv1.RuntimeOperation actor *ateapipb.Actor creation *ActorCreation + // revert restores a crashed Actor to its last durable snapshot. The + // operation stays RESUME because that is what the Session records. + revert bool } // ActorUID returns the observed runtime identity, including after creation. @@ -118,6 +122,26 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return &ActorTransition{binding: binding, operation: operation, actor: actor, creation: creation}, nil } +// PrepareActorRevert performs the read-only checks for crash recovery before +// the store's durable claim. Reading first keeps a failed or missing Actor from +// leaving a claim behind, so Delete can still supersede the unissued recovery. +// A SUSPENDED Actor means an earlier revert completed and its response was lost. +func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding) (*ActorTransition, error) { + actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) + if err != nil { + return nil, fmt.Errorf("get Actor %s/%s: %w", binding.Atespace, binding.Name, err) + } + if !binding.Matches(actor) { + return nil, fmt.Errorf("actor %s/%s identity or template changed", binding.Atespace, binding.Name) + } + switch actor.GetStatus().GetState() { + case ateapipb.ActorState_ACTOR_STATE_CRASHED, ateapipb.ActorState_ACTOR_STATE_REVERTING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED: + default: + return nil, fmt.Errorf("crashed Actor %s/%s has unexpected state %s", binding.Atespace, binding.Name, actor.GetStatus().GetState()) + } + return &ActorTransition{binding: binding, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, actor: actor, revert: true}, nil +} + // ApplyActorTransition requires durable execution authority. Errors retain the // claim's durable intent and pins. Another bounded attempt can retry that intent. func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transition *ActorTransition) error { @@ -125,6 +149,9 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return err } binding, actor := transition.binding, transition.actor + if transition.revert { + return transition.applyRevert(ctx, actors) + } var err error switch transition.operation { case apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE: @@ -194,6 +221,30 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } +// applyRevert must not replace the Actor or roll it past the snapshot observed +// before the claim. A changed UID or snapshot means something other than this +// recovery acted on the runtime, so the Session must not adopt the result. +func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClient) error { + binding, actor := t.binding, t.actor + before := proto.CloneOf(LatestDurableActorSnapshot(actor)) + if before != nil { + // Revert restores onto any node, so the locality of the source copy is + // expected to change. + before.Locality = "" + } + if state := actor.GetStatus().GetState(); state == ateapipb.ActorState_ACTOR_STATE_CRASHED || state == ateapipb.ActorState_ACTOR_STATE_REVERTING { + reverted, err := actors.RevertActor(ctx, binding.Atespace, binding.Name) + if err != nil { + return fmt.Errorf("revert crashed Actor %s/%s: %w", binding.Atespace, binding.Name, err) + } + actor = reverted + } + if t.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) != nil || !proto.Equal(before, LatestActorSnapshot(actor)) { + return fmt.Errorf("revert Actor %s/%s returned unexpected identity or state", binding.Atespace, binding.Name) + } + return nil +} + // Matches checks that the actor belongs to the expected runtime and template. func (b ActorBinding) Matches(actor *ateapipb.Actor) bool { return actor.GetMetadata().GetName() == b.Name && actor.GetMetadata().GetAtespace() == b.Atespace && actor.GetMetadata().GetUid() != "" && diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 7bd221c199..6d700e51f8 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -23,6 +23,8 @@ type transitionActors struct { checkErr error resumeUID string suspendSnapshot *ateapipb.Snapshot + revertUID string + revertSnapshot *ateapipb.Snapshot } func (a *transitionActors) GetActor(context.Context, string, string) (*ateapipb.Actor, error) { @@ -70,6 +72,26 @@ func (a *transitionActors) SuspendActor(context.Context, string, string) (*ateap return proto.CloneOf(a.actor), nil } +// RevertActor restores the latest durable snapshot unless the test overrides +// the identity or snapshot it returns. +func (a *transitionActors) RevertActor(context.Context, string, string) (*ateapipb.Actor, error) { + a.calls = append(a.calls, "revert") + a.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + snapshot := proto.CloneOf(LatestDurableActorSnapshot(a.actor)) + if a.revertSnapshot != nil { + snapshot = proto.CloneOf(a.revertSnapshot) + } + a.actor.Status.Snapshots = nil + if snapshot != nil { + snapshot.Locality = "" + a.actor.Status.Snapshots = []*ateapipb.Snapshot{snapshot} + } + if a.revertUID != "" { + a.actor.Metadata.Uid = a.revertUID + } + return proto.CloneOf(a.actor), nil +} + func (a *transitionActors) DeleteActor(context.Context, string, string, string) error { a.calls = append(a.calls, "delete") a.actor = nil @@ -242,3 +264,58 @@ func TestActorSuspendRequiresDurableVolumesSnapshot(t *testing.T) { }) } } + +func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { + durable := func(generation int32, uri string) *ateapipb.Snapshot { + return &ateapipb.Snapshot{Generation: generation, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Locality: "lost-node", + DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}}} + } + for _, test := range []struct { + name string + missing bool + state ateapipb.ActorState + revertUID string + revertSnapshot *ateapipb.Snapshot + wantPrepareErr bool + wantApplyErr bool + wantCalls []string + }{ + {name: "crashed", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantCalls: []string{"revert"}}, + {name: "revert in progress", state: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantCalls: []string{"revert"}}, + {name: "earlier revert response lost", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, + {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantPrepareErr: true}, + {name: "actor gone", missing: true, wantPrepareErr: true}, + {name: "replaced actor", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertUID: "another-actor", wantApplyErr: true, wantCalls: []string{"revert"}}, + {name: "different snapshot", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertSnapshot: durable(1, "s3://snapshots/older"), wantApplyErr: true, wantCalls: []string{"revert"}}, + } { + t.Run(test.name, func(t *testing.T) { + actors := &transitionActors{revertUID: test.revertUID, revertSnapshot: test.revertSnapshot} + if !test.missing { + actors.actor = &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team", Name: "instance", Uid: "actor-uid"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: "team", Name: "revision"}, + Status: &ateapipb.ActorStatus{State: test.state, Snapshots: []*ateapipb.Snapshot{durable(2, "s3://snapshots/latest")}}, + } + if test.state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED { + actors.actor.Status.Snapshots[0].Locality = "" + } + } + binding := ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"} + transition, err := PrepareActorRevert(t.Context(), actors, binding) + if test.wantPrepareErr { + require.Error(t, err) + require.Empty(t, actors.calls, "a failed read issues nothing") + return + } + require.NoError(t, err) + require.Equal(t, "actor-uid", transition.ActorUID()) + err = ApplyActorTransition(t.Context(), actors, transition) + if test.wantApplyErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, test.wantCalls, actors.calls) + }) + } +} diff --git a/go/core/internal/substrate/snapshot.go b/go/core/internal/substrate/snapshot.go index 3d1163c28e..302c07e975 100644 --- a/go/core/internal/substrate/snapshot.go +++ b/go/core/internal/substrate/snapshot.go @@ -22,3 +22,14 @@ func SnapshotURI(snapshot *ateapipb.Snapshot) string { } return storage.GetObject().GetSnapshotUri() } + +// Checkpoints require the newest generation; revert retains the newest durable copy. +func LatestDurableActorSnapshot(actor *ateapipb.Actor) *ateapipb.Snapshot { + var latest *ateapipb.Snapshot + for _, snapshot := range actor.GetStatus().GetSnapshots() { + if SnapshotURI(snapshot) != "" && snapshot.GetGeneration() > latest.GetGeneration() { + latest = snapshot + } + } + return latest +} diff --git a/go/core/internal/substrate/snapshot_test.go b/go/core/internal/substrate/snapshot_test.go index faee8a936c..4e923131a8 100644 --- a/go/core/internal/substrate/snapshot_test.go +++ b/go/core/internal/substrate/snapshot_test.go @@ -40,3 +40,23 @@ func TestActorSnapshotGenerationAndDurability(t *testing.T) { require.Nil(t, LatestActorSnapshot(nil)) require.Empty(t, SnapshotURI(nil)) } + +func TestRevertSnapshotSkipsIncompleteGenerations(t *testing.T) { + durable := &ateapipb.Snapshot{Generation: 2, DurableSnapshot: &ateapipb.SnapshotStorage{ + State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, + Object: &ateapipb.ObjectSnapshot{SnapshotUri: "s3://snapshots/durable"}, + }} + for _, latest := range []*ateapipb.Snapshot{ + {Generation: 3, Locality: "node-1"}, + {Generation: 3, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING}}, + {Generation: 3, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED}}, + } { + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{Snapshots: []*ateapipb.Snapshot{latest, durable}}} + require.Same(t, durable, LatestDurableActorSnapshot(actor)) + require.Same(t, latest, LatestActorSnapshot(actor)) + } + require.Nil(t, LatestDurableActorSnapshot(nil)) + require.Nil(t, LatestDurableActorSnapshot(&ateapipb.Actor{Status: &ateapipb.ActorStatus{ + Snapshots: []*ateapipb.Snapshot{{Generation: 1, Locality: "node-1"}}, + }})) +} From cf889737364a59547cf9be4ca126199ca3deeb35 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Wed, 4 Nov 2026 13:25:57 -0800 Subject: [PATCH 02/15] feat(session): admit crash recovery and retry it durably When Substrate reports a Session's Actor CRASHED, the workflow can now revert it to its last durable snapshot. `BeginCrashedSessionRecovery` admits the recovery under the Session lock. It requires the recorded Actor UID, joins an already admitted recovery, accepts only an idle READY Session whose lifecycle generation still matches the one read before the crash was observed, and refuses while a checkpoint is being created. In the same transaction it fails the interrupted turn with a timeline-positioned crash note, keeps waits the snapshot captured, and starts a RESUME operation. `SettleSessionTask` is split into `settleSessionTask(tx, ...)` so admission can reuse it under the lock. `observeCrash` is the one check that confirms a crash. A missing, replaced, or retemplated Actor is unrecoverable rather than crashed. `execute` reverts in place for an admitted crash, and `failPreparation` keeps that intent instead of clearing it, because the turn has already failed. Admitted recoveries are retried on their own ticker through `scanRecovery`, one ID-ordered page per interval, so a slow idle suspension cannot delay them. Signed-off-by: Ada Lundhe Co-Authored-By: Claude Opus 5.5 --- go/core/internal/database/session_recovery.go | 199 ++++++ .../database/session_recovery_test.go | 278 ++++++++ .../internal/database/task_finalization.go | 3 +- .../internal/service/session/quiescence.go | 28 +- .../service/session/quiescence_test.go | 4 + go/core/internal/service/session/recovery.go | 162 +++++ .../internal/service/session/recovery_test.go | 675 ++++++++++++++++++ go/core/internal/service/session/workflow.go | 30 +- 8 files changed, 1371 insertions(+), 8 deletions(-) create mode 100644 go/core/internal/database/session_recovery.go create mode 100644 go/core/internal/database/session_recovery_test.go create mode 100644 go/core/internal/service/session/recovery.go create mode 100644 go/core/internal/service/session/recovery_test.go diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go new file mode 100644 index 0000000000..dbf78f3e52 --- /dev/null +++ b/go/core/internal/database/session_recovery.go @@ -0,0 +1,199 @@ +package database + +import ( + "context" + "fmt" + "time" + + "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "google.golang.org/protobuf/types/known/timestamppb" + + apia2a "github.com/kagent-dev/kagent/go/api/a2a" + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" +) + +// SessionRuntimeCrashed identifies a recovery admitted after Substrate confirmed +// that the recorded Actor crashed. Transport failures cannot admit this work. +const SessionRuntimeCrashed = "RuntimeCrashed" + +// ListSessionRecoveryCandidates finds recoveries admitted by an explicit request. +// Retrying durable work cannot admit recovery for a READY Session. +func (c *Client) ListSessionRecoveryCandidates(ctx context.Context, afterID string, limit int) ([]*apiv1.Session, error) { + rows, err := queryMany(ctx, c.db, ` + SELECT id, user_id, prepared_revision, state, data, operation, context_id, + source_checkpoint_id, history_id, operation_id, executor_id + FROM session_record i + WHERE ($1::text = '' OR i.id > $1::uuid) + AND i.state = 'RUNTIME_STATE_FAILED' AND i.operation = 'RUNTIME_OPERATION_RESUME' + ORDER BY i.id LIMIT $2 + `, pgx.RowToStructByName[sessionRow], afterID, limit) + if err != nil { + return nil, err + } + result := make([]*apiv1.Session, 0, len(rows)) + for _, row := range rows { + session, err := toSession(row) + if err != nil { + return nil, err + } + result = append(result, session) + } + return result, nil +} + +// SessionLifecycleGeneration returns the ID of the Session's latest lifecycle +// operation, or uuid.Nil when none is recorded. Every operation replaces it, and +// completion keeps it. Recovery reads it before observing Substrate. +func (c *Client) SessionLifecycleGeneration(ctx context.Context, id string) (uuid.UUID, error) { + generation, err := queryOne(ctx, c.db, `SELECT operation_id FROM session_record WHERE id = $1`, pgx.RowTo[*uuid.UUID], id) + if err != nil { + return uuid.Nil, notFoundOr(err) + } + if generation == nil { + return uuid.Nil, nil + } + return *generation, nil +} + +// BeginCrashedSessionRecovery fences confirmed-crash recovery under the Session lock. +// It preserves durable results; obsolete Actors and conflicting operations fail. +// generation is the caller's SessionLifecycleGeneration read from before it +// observed the crash. A lifecycle operation begun since then makes the observation +// stale, because a concurrent revert keeps the Actor UID and returns to READY. +func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, generation uuid.UUID, actorUID string, snapshot SessionTaskSnapshot) (*SessionOperation, error) { + var operation *SessionOperation + err := c.withTx(ctx, func(tx pgx.Tx) error { + row, err := lockSession(ctx, tx, id) + if err != nil { + return notFoundOr(err) + } + matches, err := queryOne(ctx, tx, `SELECT actor_uid = $2 FROM session WHERE id = $1`, pgx.RowTo[bool], row.ID, actorUID) + if err != nil { + return err + } + if !matches || actorUID == "" { + return ErrConflict + } + session, err := toSession(row) + if err != nil { + return err + } + if session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && session.Failure.GetReason() == SessionRuntimeCrashed { + operation, err = toSessionOperation(row) + return err + } + if session.State != apiv1.RuntimeState_RUNTIME_STATE_READY || session.Operation != apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { + return ErrConflict + } + current := uuid.Nil + if row.OperationID != nil { + current = *row.OperationID + } + if current != generation { + return ErrConflict + } + // An issued idle suspension does not exclude recovery here. It cannot finish + // against the crashed Actor, and its worker keeps the claim when suspension + // fails, so waiting for it would block recovery permanently. Settlement below + // retires the claim in this transaction; a late finish for it changes nothing. + // The generation check keeps the crash observation current: without another + // lifecycle operation, nothing returns the crashed Actor to service. + busy, err := queryOne(ctx, tx, ` + SELECT EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') + `, pgx.RowTo[bool], row.ID) + if err != nil { + return err + } + if busy { + return ErrConflict + } + row.State = apiv1.RuntimeState_RUNTIME_STATE_FAILED.String() + row.runtimeInstanceRow, err = beginRuntimeOperation(ctx, tx, row.runtimeInstanceRow, runtimeKindAgent, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME) + if err != nil { + return err + } + session.State, session.Operation = apiv1.RuntimeState_RUNTIME_STATE_FAILED, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME + session.Failure = &apiv1.Failure{Reason: SessionRuntimeCrashed, Message: "Runtime crashed; recovery to its last snapshot is pending"} + session.UpdatedAt = timestamppb.Now() + row.Data, err = marshalSession(session) + if err != nil { + return err + } + if err := execSQL(ctx, tx, `UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL WHERE id = $1`, row.ID, row.Data); err != nil { + return err + } + if err := settleCrashedSessionTask(ctx, tx, row, snapshot); err != nil { + return err + } + operation, err = toSessionOperation(row) + return err + }) + return operation, err +} + +// settleCrashedSessionTask retains captured waits and fails discarded nonterminal work. +// Callers hold the Session lock; recovery cannot checkpoint discarded state. +func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { + if err := execSQL(ctx, tx, `UPDATE session_task_event SET quiescence_pending = FALSE WHERE history_id = $1 AND quiescence_pending`, session.HistoryID); err != nil { + return err + } + taskIDs, err := queryMany(ctx, tx, ` + SELECT t.id FROM session_task t WHERE t.history_id = $1 + AND (t.state IN ('TASK_STATE_SUBMITTED', 'TASK_STATE_WORKING') + OR (t.state IN ('TASK_STATE_INPUT_REQUIRED', 'TASK_STATE_AUTH_REQUIRED') + AND NOT EXISTS (SELECT 1 FROM session_task_event captured + WHERE captured.history_id = t.history_id AND $2::text <> '' + AND captured.snapshot_uri = $2 AND captured.snapshot_fidelity = $3 + AND captured.snapshot_atespace = $4 + AND captured.sequence >= (SELECT MAX(sequence) FROM session_task_event latest + WHERE latest.history_id = t.history_id AND latest.task_id = t.id)))) + `, pgx.RowTo[string], session.HistoryID, snapshot.URI, snapshot.Fidelity, snapshot.Atespace) + if err != nil { + return fmt.Errorf("read interrupted Session tasks: %w", err) + } + for _, taskID := range taskIDs { + version, err := taskVersion(ctx, tx, session.HistoryID, taskID) + if err != nil { + return err + } + pending, err := queryOne(ctx, tx, ` + SELECT EXISTS (SELECT 1 FROM session_task_event + WHERE sequence = $1 AND NOT published AND quiescence_pending IS NOT NULL) + `, pgx.RowTo[bool], version) + if err != nil { + return err + } + if pending { + if err := settleSessionTask(ctx, tx, session, taskID, version); err != nil { + return err + } + } + stored, err := readSessionTask(ctx, tx, session.HistoryID, taskID) + if err != nil { + return err + } + task, err := unmarshalSessionTask(stored.Data) + if err != nil { + return err + } + if !task.Status.State.Terminal() { + task.Status = a2a.TaskStatus{State: a2a.TaskStateFailed, Timestamp: new(time.Now().UTC()), + Message: a2a.NewMessage(a2a.MessageRoleAgent, a2a.NewTextPart("The runtime crashed before this turn completed. Send a new message to continue."))} + apia2a.SetTimelinePosition(task.Status.Message, *task.Status.Timestamp) + event := &a2a.TaskStatusUpdateEvent{TaskID: task.ID, ContextID: task.ContextID, Status: task.Status} + if err := storeSessionTaskEvent(ctx, tx, session, task, event, false); err != nil { + return err + } + version, err = taskVersion(ctx, tx, session.HistoryID, taskID) + if err != nil { + return err + } + if err := publishPushBoundary(ctx, tx, session.HistoryID, taskID, version, task); err != nil { + return err + } + } + } + return nil +} diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go new file mode 100644 index 0000000000..7cdb9ceade --- /dev/null +++ b/go/core/internal/database/session_recovery_test.go @@ -0,0 +1,278 @@ +package database + +import ( + "testing" + + "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + apia2a "github.com/kagent-dev/kagent/go/api/a2a" + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" +) + +func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { + for _, test := range []struct { + name string + state, want a2a.TaskState + published, captured bool + }{ + {name: "submitted", state: a2a.TaskStateSubmitted, want: a2a.TaskStateFailed}, + {name: "working", state: a2a.TaskStateWorking, want: a2a.TaskStateFailed}, + {name: "completed before publication", state: a2a.TaskStateCompleted, want: a2a.TaskStateCompleted}, + {name: "unpublished input wait", state: a2a.TaskStateInputRequired, want: a2a.TaskStateFailed}, + {name: "unpublished auth wait", state: a2a.TaskStateAuthRequired, want: a2a.TaskStateFailed}, + {name: "published input wait", state: a2a.TaskStateInputRequired, want: a2a.TaskStateFailed, published: true}, + {name: "published auth wait", state: a2a.TaskStateAuthRequired, want: a2a.TaskStateFailed, published: true}, + {name: "captured input wait", state: a2a.TaskStateInputRequired, want: a2a.TaskStateInputRequired, published: true, captured: true}, + {name: "captured auth wait", state: a2a.TaskStateAuthRequired, want: a2a.TaskStateAuthRequired, published: true, captured: true}, + } { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, _, err := client.CreateSession(t.Context(), newSessionRequest(uuid.NewString(), "assistant", "kagent", ""), uuid.NewString()) + require.NoError(t, err) + session, err = markSessionReady(t.Context(), client, session.Id, "runtime.example") + require.NoError(t, err) + task := newSessionTask(uuid.NewString(), "original input") + task.ContextID, task.Status.State = session.ContextId, test.state + version, err := client.CreateRuntimeTask(t.Context(), session.Id, taskMutationHash("create"), task, "") + require.NoError(t, err) + if test.published { + require.NoError(t, client.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + } + snapshot := SessionTaskSnapshot{} + if test.captured { + work, err := client.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + snapshot = SessionTaskSnapshot{Atespace: "team-a", URI: "s3://snapshots/captured", Fidelity: "SNAPSHOT_FIDELITY_VOLUMES"} + require.NoError(t, client.FinishSessionQuiescence(t.Context(), work, &snapshot)) + } + before, _, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + page, err := client.ListSessionRecoveryCandidates(t.Context(), "", 100) + require.NoError(t, err) + require.Empty(t, page) + + operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, snapshot) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, operation.Instance.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) + require.Equal(t, SessionRuntimeCrashed, operation.Instance.Failure.Reason) + stored, err := client.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, test.want, stored.Status.State) + require.Equal(t, before.History, stored.History) + require.Equal(t, before.Artifacts, stored.Artifacts) + if test.want == test.state { + require.Equal(t, before.Status, stored.Status) + } + _, settledVersion, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + replayed, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, snapshot) + require.NoError(t, err) + require.Equal(t, operation.ID, replayed.ID) + _, replayedVersion, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + require.Equal(t, settledVersion, replayedVersion) + _, err = client.UpdateSessionTask(t.Context(), session.Id, version, taskMutationHash("late"), task, task, "") + if test.state.Terminal() { + require.ErrorIs(t, err, ErrFailedPrecondition) + } else { + require.ErrorIs(t, err, ErrConflict) + } + require.ErrorIs(t, client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "next"), ErrConflict) + _, _, err = client.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID)}, session.Creator, uuid.NewString()) + require.ErrorIs(t, err, ErrConflict) + page, err = client.ListSessionRecoveryCandidates(t.Context(), "", 100) + require.NoError(t, err) + require.Len(t, page, 1) + require.Equal(t, session.Id, page[0].Id) + }) + } +} + +func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) { + for _, test := range []struct { + name string + actorUID string + checkpoint, superseded, lifecycle bool + }{ + {name: "replacement actor", actorUID: "replacement"}, + {name: "checkpoint", checkpoint: true}, + {name: "revert after the crash was observed", superseded: true}, + {name: "lifecycle", lifecycle: true}, + } { + t.Run(test.name, func(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, task := waitingTaskFixture(t, client) + uid := "actor-" + session.Id + // Callers read the generation before they observe the crash. + generation := lifecycleGeneration(t, client, session.Id) + switch { + case test.actorUID != "": + uid = test.actorUID + case test.checkpoint: + task.Status.State = a2a.TaskStateCompleted + require.NoError(t, saveRuntimeTask(t, client, session.Id, task, task, &SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/completed", Fidelity: "SNAPSHOT_FIDELITY_VOLUMES", + })) + _, _, err := client.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID)}, session.Creator, uuid.NewString()) + require.NoError(t, err) + case test.superseded: + // Another revert finishes on the same Actor UID, and a new turn + // starts once ingress wakes the restored Actor. + completeCrashedRecovery(t, client, session.Id, uid) + task = newSessionTask(uuid.NewString(), "next input") + task.ContextID, task.Status.State = session.ContextId, a2a.TaskStateWorking + _, err := client.CreateRuntimeTask(t.Context(), session.Id, taskMutationHash("next"), task, "") + require.NoError(t, err) + case test.lifecycle: + _, err := client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, false) + require.NoError(t, err) + } + before, err := client.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + _, err = client.BeginCrashedSessionRecovery(t.Context(), session.Id, generation, uid, SessionTaskSnapshot{}) + require.ErrorIs(t, err, ErrConflict) + after, err := client.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.True(t, proto.Equal(before, after)) + if test.superseded { + running, _, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateWorking, running.Status.State, "a stale observation must not fail the new turn") + } + }) + } +} + +// A crash during post-turn suspension leaves the worker's claim issued, because +// the worker cannot tell whether its suspension took effect. The confirmed crash +// settles that: admission retires the claim instead of waiting for it. +func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, task := waitingTaskFixture(t, client) + _, version, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + for _, state := range []a2a.TaskState{a2a.TaskStateWorking, a2a.TaskStateInputRequired} { + task.Status.State = state + version, err = client.UpdateSessionTask(t.Context(), session.Id, version, taskMutationHash(string(state)), task, task, "") + require.NoError(t, err) + } + require.NoError(t, client.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + work, err := client.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + uid := "actor-" + session.Id + + operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), uid, SessionTaskSnapshot{}) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) + pending, err := queryOne(t.Context(), client.db, `SELECT quiescence_pending FROM session_task_event WHERE sequence = $1`, pgx.RowTo[bool], work.Version) + require.NoError(t, err) + require.False(t, pending, "admission retires the claim in its own transaction") + finishCrashedRecovery(t, client, session.Id, operation, uid) + + late := SessionTaskSnapshot{Atespace: "team-a", URI: "s3://snapshots/late", Fidelity: "SNAPSHOT_FIDELITY_VOLUMES"} + require.NoError(t, client.FinishSessionQuiescence(t.Context(), work, &late)) + row, err := readSession(t.Context(), client.db, session.Id) + require.NoError(t, err) + recorded, err := queryOne(t.Context(), client.db, ` + SELECT EXISTS (SELECT 1 FROM session_task WHERE history_id = $1 AND snapshot_uri = $2) + OR EXISTS (SELECT 1 FROM session_task_event WHERE history_id = $1 AND snapshot_uri = $2) + `, pgx.RowTo[bool], row.HistoryID, late.URI) + require.NoError(t, err) + require.False(t, recorded, "a late finish of the retired claim records nothing") + _, err = client.ClaimSessionQuiescence(t.Context()) + require.ErrorIs(t, err, ErrNotFound) + next := newSessionTask(uuid.NewString(), "next input") + next.ContextID, next.Status.State = session.ContextId, a2a.TaskStateWorking + _, err = client.CreateRuntimeTask(t.Context(), session.Id, taskMutationHash("next"), next, "") + require.NoError(t, err, "the retired claim no longer blocks a new turn") +} + +func lifecycleGeneration(t *testing.T, client *Client, sessionID string) uuid.UUID { + t.Helper() + generation, err := client.SessionLifecycleGeneration(t.Context(), sessionID) + require.NoError(t, err) + return generation +} + +// completeCrashedRecovery admits and finishes a revert as a successful concurrent +// caller would, leaving the Session READY on the same Actor UID. +func completeCrashedRecovery(t *testing.T, client *Client, sessionID, actorUID string) { + t.Helper() + operation, err := client.BeginCrashedSessionRecovery(t.Context(), sessionID, lifecycleGeneration(t, client, sessionID), actorUID, SessionTaskSnapshot{}) + require.NoError(t, err) + finishCrashedRecovery(t, client, sessionID, operation, actorUID) +} + +func finishCrashedRecovery(t *testing.T, client *Client, sessionID string, operation *SessionOperation, actorUID string) { + t.Helper() + executor := uuid.New() + claimed, err := client.ClaimSessionOperation(t.Context(), sessionID, operation.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + recovered, err := client.FinishSessionOperation(t.Context(), sessionID, operation.ID, executor, "", actorUID, "") + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, recovered.State) +} + +func TestCrashedRecoveryAdmissionRollsBackOnMalformedTask(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, task := waitingTaskFixture(t, client) + _, version, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + for _, state := range []a2a.TaskState{a2a.TaskStateWorking, a2a.TaskStateInputRequired} { + task.Status.State = state + version, err = client.UpdateSessionTask(t.Context(), session.Id, version, taskMutationHash(string(state)), task, task, "") + require.NoError(t, err) + } + require.NoError(t, client.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + row, err := readSession(t.Context(), client.db, session.Id) + require.NoError(t, err) + before, err := readSessionTask(t.Context(), client.db, row.HistoryID, string(task.ID)) + require.NoError(t, err) + // A corrupt projection makes settlement fail after the lifecycle write. + _, err = client.db.Exec(t.Context(), `UPDATE session_task SET data = $2 WHERE history_id = $1 AND id = $3`, row.HistoryID, []byte{0xff}, string(task.ID)) + require.NoError(t, err) + _, err = client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, SessionTaskSnapshot{}) + require.Error(t, err) + current, err := client.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) + require.Nil(t, current.Failure) + _, err = client.db.Exec(t.Context(), `UPDATE session_task SET data = $2 WHERE history_id = $1 AND id = $3`, row.HistoryID, before.Data, string(task.ID)) + require.NoError(t, err) + work, err := client.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.Equal(t, string(task.ID), work.TaskID) +} + +// The crash note carries a timeline position. Without one, chat orders it by +// an older key and shows it above the partial reply it ends. +func TestCrashedRecoveryNotePinsItsTimelinePosition(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, task := waitingTaskFixture(t, client) + _, version, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + task.Status = a2a.TaskStatus{State: a2a.TaskStateWorking} + _, err = client.UpdateSessionTask(t.Context(), session.Id, version, taskMutationHash("working"), task, task, "") + require.NoError(t, err) + completeCrashedRecovery(t, client, session.Id, "actor-"+session.Id) + + failed, err := client.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateFailed, failed.Status.State) + require.NotNil(t, failed.Status.Timestamp) + position, ok := apia2a.TimelinePosition(failed.Status.Message) + require.True(t, ok, "the crash note has a timeline position") + require.True(t, position.Equal(*failed.Status.Timestamp), "position %s, status timestamp %s", position, failed.Status.Timestamp) +} diff --git a/go/core/internal/database/task_finalization.go b/go/core/internal/database/task_finalization.go index 8ffbd36457..6e2f575df6 100644 --- a/go/core/internal/database/task_finalization.go +++ b/go/core/internal/database/task_finalization.go @@ -28,7 +28,8 @@ func (c *Client) SettleSessionTask(ctx context.Context, sessionID, taskID string } // settleSessionTask publishes one saved boundary under the session lock. Runtime -// acknowledgement or confirmed actor deletion must establish cleanup first. +// acknowledgement, confirmed actor deletion, or a confirmed crash must establish +// cleanup first. func settleSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, taskID string, version int64) error { type boundary struct { Data []byte diff --git a/go/core/internal/service/session/quiescence.go b/go/core/internal/service/session/quiescence.go index 283e70c7cc..97e37a5a2a 100644 --- a/go/core/internal/service/session/quiescence.go +++ b/go/core/internal/service/session/quiescence.go @@ -28,11 +28,15 @@ func (w *ActorWorkflow) wakeQuiescence() { } } -// Start suspends idle sessions to VOLUMES independently of task publication. -// Startup and periodic recovery scans discover work missed by local signals. -// Workers drain claims with bounded concurrency; task reads never wait for them. +// Start suspends idle sessions to VOLUMES independently of task publication and +// retries crash recoveries admitted by explicit requests. +// Startup and periodic scans discover idle work missed by local signals. Workers +// drain claims with bounded concurrency. Crash recovery scans on its own ticker, so +// a slow idle suspension cannot delay a recovery retry and a slow revert cannot +// hold up idle work. func (w *ActorWorkflow) Start(ctx context.Context) error { var workers sync.WaitGroup + workers.Go(func() { w.recoverCrashedSessions(ctx) }) for range 4 { workers.Go(func() { timer := time.NewTimer(w.recoveryInterval) @@ -72,6 +76,24 @@ func (w *ActorWorkflow) Start(ctx context.Context) error { return nil } +// recoverCrashedSessions runs one recovery scan per tick. +// The cursor carries across ticks, so a large backlog cannot starve the Sessions +// after its first page, and an Actor that stays unreadable is revisited when the +// scan wraps. +func (w *ActorWorkflow) recoverCrashedSessions(ctx context.Context) { + ticker := time.NewTicker(w.recoveryInterval) + defer ticker.Stop() + var after string + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + after = w.scanRecovery(ctx, after) + } +} + func (w *ActorWorkflow) quiesceIdleSession(ctx context.Context, work *database.SessionQuiescence) { runtimeCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) snapshot, err := w.Quiesce(runtimeCtx, work.Session) diff --git a/go/core/internal/service/session/quiescence_test.go b/go/core/internal/service/session/quiescence_test.go index 99e8852d72..d6b9af4991 100644 --- a/go/core/internal/service/session/quiescence_test.go +++ b/go/core/internal/service/session/quiescence_test.go @@ -32,6 +32,10 @@ type idleQuiescenceStore struct { var _ workflowStore = (*idleQuiescenceStore)(nil) +func (s *idleQuiescenceStore) ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) { + return nil, nil +} + func (s *idleQuiescenceStore) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) { s.claims.Add(1) if s.claimErr != nil { diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go new file mode 100644 index 0000000000..de296fc141 --- /dev/null +++ b/go/core/internal/service/session/recovery.go @@ -0,0 +1,162 @@ +package session + +import ( + "context" + "errors" + "fmt" + + ateapipb "go.ate.dev/substrate/api/ateapi/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/pkg/logging" +) + +// Revert refuses a Session with nothing to act on for one of two reasons. They +// ask opposite things of a client, so the API reports them with distinct reasons. +// errRuntimeNotCrashed means nothing needs recovering: the runtime is healthy or +// suspended. A retry after a revert whose response was lost lands here, and the +// next interaction proceeds. +// errRuntimeUnrecoverable means the recorded Actor is gone or was replaced. No +// retry restores it, and the client has to start another Session. Transient +// Substrate and database failures wrap neither, so they stay retryable. +var ( + errRuntimeNotCrashed = fmt.Errorf("runtime is not crashed: %w", database.ErrFailedPrecondition) + errRuntimeUnrecoverable = fmt.Errorf("runtime cannot be recovered: %w", database.ErrFailedPrecondition) +) + +// crashRecoveryAdmitted reports a revert already admitted for a confirmed crash. +// Revert continues it without observing Substrate again. +func crashRecoveryAdmitted(session *apiv1.Session) bool { + return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Failure.GetReason() == database.SessionRuntimeCrashed +} + +// lifecycleIdle reports a Session with no lifecycle operation in flight. Only an +// idle Session can admit a new revert; any other operation owns the Actor. +func lifecycleIdle(session *apiv1.Session) bool { + return session.State == apiv1.RuntimeState_RUNTIME_STATE_READY && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE +} + +// observeCrash reads the Session's recorded Actor and reports whether Substrate +// confirms it crashed. Revert admits recovery only through this check. An Actor +// that is missing, replaced, or from another template is an error, never a +// crash: reverting it would restore the wrong runtime. Those errors wrap +// errRuntimeUnrecoverable; transient read failures do not. +func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session) (*ateapipb.Actor, bool, error) { + revision, err := w.store.GetRuntimeRevision(ctx, session.PreparedRevision) + if err != nil { + return nil, false, err + } + actor, err := w.actors.GetActor(ctx, revision.ActorTemplateAtespace, substrate.ActorName(session.Id)) + if status.Code(err) == codes.NotFound { + return nil, false, fmt.Errorf("runtime Actor is missing: %w", errRuntimeUnrecoverable) + } + if err != nil { + return nil, false, err + } + if !validActorIdentity(actor, revision, substrate.ActorName(session.Id)) { + return nil, false, fmt.Errorf("runtime Actor identity or template changed: %w", errRuntimeUnrecoverable) + } + if err := w.verifyActor(ctx, session, revision, actor); errors.Is(err, database.ErrNotFound) { + return nil, false, fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) + } else if err != nil { + return nil, false, err + } + return actor, actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED, nil +} + +// scanRecovery runs one recovery interval and returns the cursor for the next. +// Admitted recoveries page through PostgreSQL in ID order. +func (w *ActorWorkflow) scanRecovery(ctx context.Context, afterID string) string { + scanCtx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) + after, err := w.recoverSessionPage(scanCtx, afterID) + cancel() + if err != nil && ctx.Err() == nil { + logging.FromContext(ctx).ErrorContext(ctx, "scan interrupted sessions", "error", err) + } + return after +} + +// recoverSessionPage retries one ID-ordered page of admitted recoveries. +// READY Sessions are excluded, so a background retry cannot admit new work. +func (w *ActorWorkflow) recoverSessionPage(ctx context.Context, afterID string) (string, error) { + const pageSize = 100 + sessions, err := w.store.ListSessionRecoveryCandidates(ctx, afterID, pageSize) + if err != nil { + return afterID, err + } + for _, session := range sessions { + w.recoverCandidate(ctx, session) + afterID = session.Id + if ctx.Err() != nil { + return afterID, ctx.Err() + } + } + if len(sessions) < pageSize { + afterID = "" + } + return afterID, nil +} + +func (w *ActorWorkflow) recoverCandidate(ctx context.Context, session *apiv1.Session) { + attemptCtx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) + err := w.recoverSessionCandidate(attemptCtx, session) + cancel() + if err != nil && !errors.Is(err, database.ErrConflict) && ctx.Err() == nil { + logging.FromContext(ctx).ErrorContext(ctx, "recover crashed session", "session_id", session.Id, "error", err) + } +} + +func (w *ActorWorkflow) recoverSessionCandidate(ctx context.Context, session *apiv1.Session) error { + _, err := w.Revert(ctx, session) + if errors.Is(err, database.ErrFailedPrecondition) { + return nil + } + return err +} + +// Revert restores a confirmed crashed runtime without replaying its interrupted turn. +// Retries join the same durable operation and preserve the Actor identity. +func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*apiv1.Session, error) { + ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) + defer cancel() + if crashRecoveryAdmitted(session) { + return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, false) + } + // Only an active lifecycle operation is contention: a retry converges once it + // finishes. A Session at rest in another state, such as one suspended by + // deadline cleanup, stays there until an interaction resumes it, so a retry + // would spin until the caller's deadline. + if session.Operation != apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { + return nil, database.ErrConflict + } + if !lifecycleIdle(session) { + return nil, fmt.Errorf("runtime state is %s: %w", session.State, errRuntimeNotCrashed) + } + // Read before observing Substrate. A concurrent revert keeps the Actor UID and + // returns to READY, so only the generation shows that this crash observation + // predates it. + generation, err := w.store.SessionLifecycleGeneration(ctx, session.Id) + if err != nil { + return nil, err + } + actor, crashed, err := w.observeCrash(ctx, session) + if err != nil { + return nil, err + } + if !crashed { + return nil, errRuntimeNotCrashed + } + snapshot := substrate.LatestDurableActorSnapshot(actor) + operation, err := w.store.BeginCrashedSessionRecovery(ctx, session.Id, generation, actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{ + Atespace: actor.GetMetadata().GetAtespace(), URI: substrate.SnapshotURI(snapshot), + Fidelity: snapshot.GetFidelity().String(), + }) + if err != nil { + return nil, err + } + return w.execute(ctx, operation) +} diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go new file mode 100644 index 0000000000..16515f7cee --- /dev/null +++ b/go/core/internal/service/session/recovery_test.go @@ -0,0 +1,675 @@ +package session + +import ( + "context" + "crypto/sha256" + "fmt" + "slices" + "sync/atomic" + "testing" + "time" + + "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + ateapipb "go.ate.dev/substrate/api/ateapi/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func (a *lifecycleTestActors) RevertActor(_ context.Context, atespace, name string) (*ateapipb.Actor, error) { + a.mu.Lock() + defer a.mu.Unlock() + actor := a.actors[actorKey(atespace, name)] + if actor.Status.State != ateapipb.ActorState_ACTOR_STATE_CRASHED && actor.Status.State != ateapipb.ActorState_ACTOR_STATE_REVERTING { + return nil, status.Error(codes.FailedPrecondition, "actor is not crashed or reverting") + } + actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + actor.Status.Crash = nil + snapshot := proto.CloneOf(substrate.LatestDurableActorSnapshot(actor)) + actor.Status.Snapshots = nil + if snapshot != nil { + snapshot.Locality = "" + actor.Status.Snapshots = []*ateapipb.Snapshot{snapshot} + } + return proto.CloneOf(actor), nil +} + +type recoveryTestActors struct { + *lifecycleTestActors + revert func(context.Context, string, string) (*ateapipb.Actor, error) + readErr error + // afterRead runs once, after the first successful read returns its copy. + afterRead func(context.Context) +} + +func (a *recoveryTestActors) RevertActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + return a.revert(ctx, atespace, name) +} + +func (a *recoveryTestActors) GetActor(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + if a.readErr != nil { + return nil, a.readErr + } + actor, err := a.lifecycleTestActors.GetActor(ctx, atespace, name) + if hook := a.afterRead; hook != nil && err == nil { + a.afterRead = nil + hook(ctx) + } + return actor, err +} + +func revertGeneration(t *testing.T, store *lifecycleTestStore, sessionID string) uuid.UUID { + t.Helper() + generation, err := store.SessionLifecycleGeneration(t.Context(), sessionID) + require.NoError(t, err) + return generation +} + +func crashedTurnFixture(t *testing.T) (*lifecycleTestStore, *lifecycleTestActors, *apiv1.Session, *a2a.Task, int64) { + t.Helper() + store, session := lifecycleFixture(t) + actors := &lifecycleTestActors{actors: map[string]*ateapipb.Actor{}} + var err error + session, err = NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second).Create(t.Context(), session) + require.NoError(t, err) + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("original input")) + message.ContextID = session.ContextId + task := a2a.NewSubmittedTask(message, message) + task.Status.State = a2a.TaskStateWorking + hash := sha256.Sum256([]byte("working")) + version, err := store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") + require.NoError(t, err) + actors.actors[actorKey("team-a", substrate.ActorName(session.Id))].Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + return store, actors, session, task, version +} + +func TestCrashedSessionRecoveryRetriesWithoutLosingState(t *testing.T) { + for _, test := range []struct { + name string + lostResponse bool + wantReverts int + }{ + {name: "revert failed", wantReverts: 2}, + {name: "revert response lost", lostResponse: true, wantReverts: 1}, + } { + t.Run(test.name, func(t *testing.T) { + store, base, session, task, _ := crashedTurnFixture(t) + calls := 0 + actors := &recoveryTestActors{lifecycleTestActors: base} + actors.revert = func(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + calls++ + if calls == 1 { + if test.lostResponse { + _, err := base.RevertActor(ctx, atespace, name) + require.NoError(t, err) + } + return nil, status.Error(codes.Unavailable, "response unavailable") + } + return base.RevertActor(ctx, atespace, name) + } + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) + _, err := workflow.Revert(t.Context(), session) + require.Error(t, err) + failed, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, failed.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, failed.Operation) + require.Equal(t, database.SessionRuntimeCrashed, failed.Failure.Reason) + interrupted, interruptedVersion, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateFailed, interrupted.Status.State) + published, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateFailed, published.Status.State) + require.Len(t, published.History, 1) + require.ErrorIs(t, store.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), uuid.NewString()), database.ErrConflict) + // A new workflow instance discovers the admitted operation from PostgreSQL. + restarted := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) + _, err = restarted.recoverSessionPage(t.Context(), "") + require.NoError(t, err) + ready, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, ready.State) + require.Nil(t, ready.Failure) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateFailed, stored.Status.State) + require.Equal(t, test.wantReverts, calls) + require.Len(t, stored.History, 1, "the original input remains archived once") + _, recoveredVersion, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) + require.NoError(t, err) + require.Equal(t, interruptedVersion, recoveredVersion, "retries do not append another terminal event") + _, err = restarted.Revert(t.Context(), ready) + require.ErrorIs(t, err, database.ErrFailedPrecondition) + require.Equal(t, test.wantReverts, calls, "completed recovery cannot discard another execution") + }) + } +} + +func TestCrashedSessionRecoveryFencesConcurrentWork(t *testing.T) { + store, base, session, task, version := crashedTurnFixture(t) + entered, release := make(chan struct{}), make(chan struct{}) + actors := &recoveryTestActors{lifecycleTestActors: base} + actors.revert = func(ctx context.Context, atespace, name string) (*ateapipb.Actor, error) { + close(entered) + select { + case <-release: + return base.RevertActor(ctx, atespace, name) + case <-ctx.Done(): + return nil, ctx.Err() + } + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) + done := make(chan error, 1) + go func() { _, err := workflow.Revert(ctx, session); done <- err }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("recovery was not admitted") + } + failed, err := store.GetSessionByID(ctx, session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, failed.State) + _, err = workflow.Revert(ctx, failed) + require.ErrorIs(t, err, database.ErrConflict) + hash := sha256.Sum256([]byte("late write")) + _, err = store.UpdateSessionTask(ctx, session.Id, version, hash[:], task, task, "") + require.ErrorIs(t, err, database.ErrConflict) + close(release) + require.NoError(t, <-done) + _, err = store.UpdateSessionTask(ctx, session.Id, version, hash[:], task, task, "") + require.ErrorIs(t, err, database.ErrConflict) +} + +func TestSessionRecoveryRequiresConfirmedRecordedActorCrash(t *testing.T) { + for _, test := range []struct { + name string + state ateapipb.ActorState + uid string + template string + readErr error + // want is the refusal a client sees; nil means the error must stay + // retryable, so it may match neither refusal. + want error + }{ + {name: "quiet running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, want: errRuntimeNotCrashed}, + {name: "transient read failure", readErr: status.Error(codes.Unavailable, "unreachable")}, + {name: "missing actor", readErr: status.Error(codes.NotFound, "actor not found"), want: errRuntimeUnrecoverable}, + {name: "same-name replacement", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, uid: "another-actor", want: errRuntimeUnrecoverable}, + {name: "template changed", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, template: "another-template", want: errRuntimeUnrecoverable}, + } { + t.Run(test.name, func(t *testing.T) { + store, base, session, task, _ := crashedTurnFixture(t) + actor := base.actors[actorKey("team-a", substrate.ActorName(session.Id))] + actor.Status.State = test.state + if test.uid != "" { + actor.Metadata.Uid = test.uid + } + if test.template != "" { + actor.ActorTemplate.Name = test.template + } + actors := &recoveryTestActors{lifecycleTestActors: base, readErr: test.readErr} + actors.revert = func(context.Context, string, string) (*ateapipb.Actor, error) { + t.Fatal("unconfirmed crash issued a revert") + return nil, nil + } + _, err := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second).Revert(t.Context(), session) + require.Error(t, err) + if test.want != nil { + require.ErrorIs(t, err, test.want) + require.ErrorIs(t, err, database.ErrFailedPrecondition) + } else { + require.NotErrorIs(t, err, database.ErrFailedPrecondition, "a transient failure must stay retryable") + } + current, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) + require.Nil(t, current.Failure) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateWorking, stored.Status.State) + }) + } +} + +// Only an active lifecycle operation is contention a retry waits out. A Session +// at rest outside READY, such as one suspended by deadline cleanup, would +// return ABORTED forever if it were reported as contention. +func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { + for _, test := range []struct { + name string + state apiv1.RuntimeState + operation apiv1.RuntimeOperation + want error + }{ + {name: "suspended by deadline cleanup", state: apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, want: errRuntimeNotCrashed}, + {name: "suspension in progress", state: apiv1.RuntimeState_RUNTIME_STATE_READY, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, want: database.ErrConflict}, + {name: "resume in progress", state: apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, want: database.ErrConflict}, + {name: "creation in progress", state: apiv1.RuntimeState_RUNTIME_STATE_CREATING, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, want: database.ErrConflict}, + } { + t.Run(test.name, func(t *testing.T) { + // Both checks run on the Session the caller read, before any store + // or Substrate access, so neither dependency is needed. + workflow := NewActorWorkflow(nil, nil, make(chan struct{}, 1), time.Second) + _, err := workflow.Revert(t.Context(), &apiv1.Session{Id: uuid.NewString(), State: test.state, Operation: test.operation}) + require.ErrorIs(t, err, test.want) + if test.want == database.ErrConflict { + require.NotErrorIs(t, err, database.ErrFailedPrecondition) + } + }) + } +} + +func TestSessionRecoveryCandidatePolicy(t *testing.T) { + store, actors, active, _, _ := crashedTurnFixture(t) + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) + for _, state := range []a2a.TaskState{"", a2a.TaskStateInputRequired, a2a.TaskStateAuthRequired, a2a.TaskStateWorking} { + session, _, err := store.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: active.Agent}, uuid.NewString()) + require.NoError(t, err) + session, err = workflow.Create(t.Context(), session) + require.NoError(t, err) + if state == "" { + continue + } + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("input")) + message.ContextID = session.ContextId + task := a2a.NewSubmittedTask(message, message) + task.Status.State = state + hash := sha256.Sum256([]byte(task.ID)) + version, err := store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") + require.NoError(t, err) + if state != a2a.TaskStateWorking { + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + } + } + page, err := store.ListSessionRecoveryCandidates(t.Context(), "", 100) + require.NoError(t, err) + require.Empty(t, page, "manual recovery does not inspect idle or active runtimes") + actor, err := actors.GetActor(t.Context(), "team-a", substrate.ActorName(active.Id)) + require.NoError(t, err) + _, err = store.BeginCrashedSessionRecovery(t.Context(), active.Id, revertGeneration(t, store, active.Id), actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + require.NoError(t, err) + page, err = store.ListSessionRecoveryCandidates(t.Context(), "", 100) + require.NoError(t, err) + require.Len(t, page, 1, "explicitly admitted recovery remains eligible for retry") + require.Equal(t, active.Id, page[0].Id) +} + +func TestCrashedSessionRecoversAnOlderResumedTask(t *testing.T) { + store, actors, session, older, version := crashedTurnFixture(t) + older.Status.State = a2a.TaskStateInputRequired + hash := sha256.Sum256([]byte("waiting")) + waitingVersion, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], older, older, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(older.ID), waitingVersion)) + work, err := store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/waiting-turn", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + })) + + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("another turn")) + message.ContextID = session.ContextId + newer := a2a.NewSubmittedTask(message, message) + newer.Status.State = a2a.TaskStateCompleted + hash = sha256.Sum256([]byte("newer completed task")) + newerVersion, err := store.CreateRuntimeTask(t.Context(), session.Id, hash[:], newer, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(newer.ID), newerVersion)) + work, err = store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/newer-completed-turn", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + })) + + older.Status.State = a2a.TaskStateWorking + hash = sha256.Sum256([]byte("resume older task")) + _, err = store.UpdateSessionTask(t.Context(), session.Id, waitingVersion, hash[:], older, older, "") + require.NoError(t, err) + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) + _, err = workflow.Revert(t.Context(), session) + require.NoError(t, err) + failed, err := store.GetSettledSessionTask(t.Context(), session.Id, string(older.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateFailed, failed.Status.State) + retained, retainedVersion, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(newer.ID)) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateCompleted, retained.Status.State) + require.Equal(t, newerVersion, retainedVersion, "recovery cannot rewrite a later completed task") + require.NoError(t, store.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), uuid.NewString())) +} + +func TestExplicitRevertRespectsCheckpointCreation(t *testing.T) { + store, actors, session, task, version := crashedTurnFixture(t) + task.Status.State = a2a.TaskStateCompleted + hash := sha256.Sum256([]byte("completed")) + version, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], task, task, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + work, err := store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/completed", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + })) + _, _, err = store.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{ + Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID), + }, session.Creator, uuid.NewString()) + require.NoError(t, err) + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + _, err = workflow.Revert(t.Context(), session) + require.ErrorIs(t, err, database.ErrConflict) + actor, err := actors.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_CRASHED, actor.Status.State) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateCompleted, stored.Status.State) +} + +// The Actor crashed during post-turn suspension, and the worker kept its claim +// because it cannot tell whether the suspension took effect. +func TestExplicitRevertRetiresIdleSuspensionOfCrashedActor(t *testing.T) { + store, actors, session, task, version := crashedTurnFixture(t) + task.Status.State = a2a.TaskStateCompleted + hash := sha256.Sum256([]byte("completed")) + version, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], task, task, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + work, err := store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + recovered, err := workflow.Revert(t.Context(), session) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, recovered.State) + actor, err := actors.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, actor.Status.State) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateCompleted, stored.Status.State) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/late", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + }), "a late finish of the retired claim is a no-op") + _, err = store.ClaimSessionQuiescence(t.Context()) + require.ErrorIs(t, err, database.ErrNotFound) +} + +// Revert A observes the crash, then pauses while Revert B restores the same Actor +// and ingress wakes it for a new turn. A then sees the same Actor UID and a READY +// Session, so only the lifecycle generation shows that its observation is stale. +func TestExplicitRevertRejectsCrashObservedBeforeAnotherRevert(t *testing.T) { + store, base, session, _, _ := crashedTurnFixture(t) + key := actorKey("team-a", substrate.ActorName(session.Id)) + var next *a2a.Task + paused := &recoveryTestActors{lifecycleTestActors: base, revert: base.RevertActor, afterRead: func(ctx context.Context) { + recovered, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Revert(ctx, session) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, recovered.State) + base.mu.Lock() + base.actors[key].Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + base.mu.Unlock() + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("next input")) + message.ContextID = session.ContextId + next = a2a.NewSubmittedTask(message, message) + next.Status.State = a2a.TaskStateWorking + hash := sha256.Sum256([]byte("next")) + _, err = store.CreateRuntimeTask(ctx, session.Id, hash[:], next, "") + require.NoError(t, err) + }} + _, err := NewActorWorkflow(store, paused, make(chan struct{}, 1), time.Minute).Revert(t.Context(), session) + require.ErrorIs(t, err, database.ErrConflict) + current, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, current.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, current.Operation) + running, _, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(next.ID)) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateWorking, running.Status.State, "the stale revert must not fail the new turn") +} + +func TestExplicitRevertCannotCheckpointDiscardedTerminalState(t *testing.T) { + store, actors, session, task, version := crashedTurnFixture(t) + task.Status.State = a2a.TaskStateCompleted + hash := sha256.Sum256([]byte("completed before snapshot")) + version, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], task, task, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + _, err = workflow.Revert(t.Context(), session) + require.NoError(t, err) + _, err = store.ClaimSessionQuiescence(t.Context()) + require.ErrorIs(t, err, database.ErrNotFound, "the reverted snapshot cannot represent discarded terminal state") + _, _, err = store.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{ + Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID), + }, session.Creator, uuid.NewString()) + require.ErrorIs(t, err, database.ErrSnapshotPending) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateCompleted, stored.Status.State) +} + +func TestExplicitRevertPreservesOnlyCapturedWaits(t *testing.T) { + for _, state := range []a2a.TaskState{a2a.TaskStateInputRequired, a2a.TaskStateAuthRequired} { + for _, captured := range []bool{false, true} { + t.Run(fmt.Sprint(state, "/captured=", captured), func(t *testing.T) { + store, actors, session, task, version := crashedTurnFixture(t) + task.Status.State = state + hash := sha256.Sum256([]byte(state)) + version, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], task, task, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + if captured { + work, err := store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/waiting", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + })) + actors.actors[actorKey("team-a", substrate.ActorName(session.Id))].Status.Snapshots = []*ateapipb.Snapshot{{Generation: 1, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "s3://snapshots/waiting"}}}} + actor := actors.actors[actorKey("team-a", substrate.ActorName(session.Id))] + actor.Status.Snapshots[0].Locality = "lost-node" + actor.Status.Snapshots = append(actor.Status.Snapshots, &ateapipb.Snapshot{ + Generation: 2, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_PENDING}, + }) + } + _, err = NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute).Revert(t.Context(), session) + require.NoError(t, err) + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + want := a2a.TaskStateFailed + if captured { + want = state + } + require.Equal(t, want, stored.Status.State) + }) + } + } +} + +func TestExplicitRevertRetainsWaitCapturedByLaterTurn(t *testing.T) { + store, actors, session, waiting, version := crashedTurnFixture(t) + waiting.Status.State = a2a.TaskStateInputRequired + hash := sha256.Sum256([]byte("waiting")) + version, err := store.UpdateSessionTask(t.Context(), session.Id, version, hash[:], waiting, waiting, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(waiting.ID), version)) + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("another turn")) + message.ContextID = session.ContextId + later := a2a.NewSubmittedTask(message, message) + later.Status.State = a2a.TaskStateCompleted + hash = sha256.Sum256([]byte("later completed turn")) + version, err = store.CreateRuntimeTask(t.Context(), session.Id, hash[:], later, "") + require.NoError(t, err) + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(later.ID), version)) + work, err := store.ClaimSessionQuiescence(t.Context()) + require.NoError(t, err) + require.NoError(t, store.FinishSessionQuiescence(t.Context(), work, &database.SessionTaskSnapshot{ + Atespace: "team-a", URI: "s3://snapshots/later-turn", Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String(), + })) + actors.actors[actorKey("team-a", substrate.ActorName(session.Id))].Status.Snapshots = []*ateapipb.Snapshot{{Generation: 1, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "s3://snapshots/later-turn"}}}} + _, err = NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute).Revert(t.Context(), session) + require.NoError(t, err) + retained, err := store.GetSettledSessionTask(t.Context(), session.Id, string(waiting.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateInputRequired, retained.Status.State) + retained, err = store.GetSettledSessionTask(t.Context(), session.Id, string(later.ID), nil) + require.NoError(t, err) + require.Equal(t, a2a.TaskStateCompleted, retained.Status.State) +} + +func TestSessionRecoveryPagesPastUnavailableActors(t *testing.T) { + store, base, first, _, _ := crashedTurnFixture(t) + actors := &recoveryTestActors{lifecycleTestActors: base, revert: base.RevertActor, readErr: status.Error(codes.Unavailable, "control plane unavailable")} + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + ids := []string{first.Id} + for range 100 { + session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ + Id: uuid.NewString(), Creator: first.Creator, Agent: first.Agent, + }, uuid.NewString()) + require.NoError(t, err) + session, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Create(t.Context(), session) + require.NoError(t, err) + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("accepted turn")) + message.ContextID = session.ContextId + task := a2a.NewSubmittedTask(message, message) + task.Status.State = a2a.TaskStateWorking + hash := sha256.Sum256([]byte(task.ID)) + _, err = store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") + require.NoError(t, err) + base.actors[actorKey("team-a", substrate.ActorName(session.Id))].Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + ids = append(ids, session.Id) + } + for _, id := range ids { + actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(id)) + require.NoError(t, err) + _, err = store.BeginCrashedSessionRecovery(t.Context(), id, revertGeneration(t, store, id), actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + require.NoError(t, err) + } + slices.Sort(ids) + after, err := workflow.recoverSessionPage(t.Context(), "") + require.NoError(t, err) + require.Equal(t, ids[99], after, "failed Actor reads still advance the bounded page") + actors.readErr = nil + after, err = workflow.recoverSessionPage(t.Context(), after) + require.NoError(t, err) + require.Empty(t, after, "the short second page resets the cursor") + last, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(ids[100])) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, last.GetStatus().GetState(), "the second page recovers the 101st candidate") + firstActor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(ids[0])) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_CRASHED, firstActor.GetStatus().GetState(), "unavailable first-page Actors still need recovery") + after, err = workflow.recoverSessionPage(t.Context(), after) + require.NoError(t, err) + require.Equal(t, ids[99], after, "the wrapped scan revisits previously unavailable Actors") + after, err = workflow.recoverSessionPage(t.Context(), after) + require.NoError(t, err) + require.Empty(t, after, "an empty page resets the cursor") + for _, id := range ids { + actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(id)) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, actor.GetStatus().GetState()) + } +} + +// blockedSuspendActors holds every idle suspension in runtime I/O until the +// test ends, the way a slow Substrate upload would. +type blockedSuspendActors struct { + *lifecycleTestActors + entered atomic.Int32 +} + +func (a *blockedSuspendActors) SuspendActor(ctx context.Context, _, _ string) (*ateapipb.Actor, error) { + a.entered.Add(1) + <-ctx.Done() + return nil, ctx.Err() +} + +// Start retries an admitted recovery on its own ticker. Every quiescence worker +// is blocked in an idle suspension for the whole test, so a scan that shared +// those workers would never run. +func TestStartRetriesAdmittedRecoveryWhileIdleWorkBlocks(t *testing.T) { + store, base, crashed, _, _ := crashedTurnFixture(t) + unavailable := &recoveryTestActors{lifecycleTestActors: base, revert: func(context.Context, string, string) (*ateapipb.Actor, error) { + return nil, status.Error(codes.Unavailable, "control plane unavailable") + }} + _, err := NewActorWorkflow(store, unavailable, make(chan struct{}, 1), time.Hour).Revert(t.Context(), crashed) + require.Error(t, err, "the first attempt leaves an admitted recovery for Start to retry") + + const idleSessions = 4 // one per quiescence worker + for range idleSessions { + idle, _, err := store.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: crashed.Creator, Agent: crashed.Agent}, uuid.NewString()) + require.NoError(t, err) + idle, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Hour).Create(t.Context(), idle) + require.NoError(t, err) + request := stageQuiescenceTask(t, store.Client, idle, a2a.TaskStateCompleted) + require.NoError(t, store.SettleSessionTask(t.Context(), idle.Id, request.TaskId, request.Version)) + } + + actors := &blockedSuspendActors{lifecycleTestActors: base} + startQuiescenceWorker(t, NewActorWorkflow(store, actors, make(chan struct{}, 1), 20*time.Millisecond)) + require.Eventually(t, func() bool { return actors.entered.Load() == idleSessions }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { + current, err := store.GetSessionByID(t.Context(), crashed.Id) + return err == nil && current.State == apiv1.RuntimeState_RUNTIME_STATE_READY + }, 5*time.Second, 10*time.Millisecond) + actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(crashed.Id)) + require.NoError(t, err) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, actor.GetStatus().GetState()) + require.EqualValues(t, idleSessions, actors.entered.Load(), "idle work stayed blocked while recovery completed") +} + +// A recovery whose Actor cannot be read has issued nothing, so it must not take +// a claim that would make Delete refuse the Session. +func TestDeleteSupersedesUnissuedCrashRecovery(t *testing.T) { + for _, test := range []struct { + name string + gone bool + readErr error + }{ + {name: "actor read fails", readErr: status.Error(codes.Unavailable, "control plane unavailable")}, + {name: "actor is gone", gone: true}, + } { + t.Run(test.name, func(t *testing.T) { + store, base, session, _, _ := crashedTurnFixture(t) + key := actorKey("team-a", substrate.ActorName(session.Id)) + _, err := store.BeginCrashedSessionRecovery(t.Context(), session.Id, revertGeneration(t, store, session.Id), base.actors[key].GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + require.NoError(t, err) + if test.gone { + delete(base.actors, key) + } + actors := &recoveryTestActors{lifecycleTestActors: base, readErr: test.readErr} + actors.revert = func(context.Context, string, string) (*ateapipb.Actor, error) { + t.Fatal("an unread Actor was reverted") + return nil, nil + } + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + admitted, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + _, err = workflow.Revert(t.Context(), admitted) + require.Error(t, err) + _, err = workflow.recoverSessionPage(t.Context(), "") + require.NoError(t, err) + + pending, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, pending.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, pending.Operation, "a failed read keeps the recovery for the next scan") + deleted, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), pending, false) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_DELETED, deleted.State) + _, err = base.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) + require.Equal(t, codes.NotFound, status.Code(err)) + page, err := store.ListSessionRecoveryCandidates(t.Context(), "", 100) + require.NoError(t, err) + require.Empty(t, page) + }) + } +} diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index ac4a66f107..20a758b435 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -15,6 +15,9 @@ import ( ) type workflowStore interface { + ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) + SessionLifecycleGeneration(context.Context, string) (uuid.UUID, error) + BeginCrashedSessionRecovery(context.Context, string, uuid.UUID, string, database.SessionTaskSnapshot) (*database.SessionOperation, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) FinishSessionQuiescence(context.Context, *database.SessionQuiescence, *database.SessionTaskSnapshot) error @@ -157,11 +160,15 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session creation.Snapshot = &substrate.ActorSnapshot{Tag: &ateapipb.ObjectRef{Atespace: snapshot.Atespace, Name: tagName}, URI: snapshot.URI, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES} } } - prepare := substrate.PrepareActorTransition - if operation.ExecutorID != uuid.Nil { - prepare = substrate.PrepareActorRetry + var transition *substrate.ActorTransition + switch { + case crashRecovery(operation): + transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding) + case operation.ExecutorID != uuid.Nil: + transition, err = substrate.PrepareActorRetry(ctx, w.actors, binding, kind, creation) + default: + transition, err = substrate.PrepareActorTransition(ctx, w.actors, binding, kind, creation) } - transition, err := prepare(ctx, w.actors, binding, kind, creation) if err != nil { return w.failPreparation(ctx, operation, err) } @@ -214,6 +221,12 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. if admitted.ExecutorID != uuid.Nil { return nil, cause // An earlier attempt may have issued work; retain its intent. } + if crashRecovery(admitted) { + // Admission already failed the interrupted turn, so clearing the operation + // would leave a FAILED Session that nothing retries. Keep the intent for the + // next scan. No claim was taken, so Delete can still supersede it. + return nil, cause + } finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() _, err := w.store.FinishSessionOperation(finishCtx, admitted.Instance.Id, admitted.ID, uuid.Nil, "", "", "lifecycle preparation failed") @@ -227,6 +240,15 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. return nil, errors.Join(cause, err) } +// crashRecovery reports a RESUME admitted after Substrate confirmed a crash. +// It reverts the recorded Actor in place instead of resuming it. +func crashRecovery(operation *database.SessionOperation) bool { + session := operation.Instance + return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && + session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && + session.Failure.GetReason() == database.SessionRuntimeCrashed +} + func operationOutcome(operation *database.SessionOperation) (*apiv1.Session, error) { if operation.Instance.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { return operation.Instance, nil From 1595fcfca9474aec3706f07d7b4fa8a15d60c10e Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Wed, 4 Nov 2026 13:26:05 -0800 Subject: [PATCH 03/15] feat(api): revert crashed Sessions through RevertSession Expose crash recovery as `SessionService.RevertSession`. The request validates the Session ID as a UUID, and the method needs update access, so a read-only share cannot invoke it. `Service.Revert` refuses with FAILED_PRECONDITION when there is nothing to revert, and an ErrorInfo reason tells the two cases apart. `KAGENT_SESSION_RUNTIME_NOT_CRASHED` means the runtime is healthy or at rest. `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` means the recorded Actor is gone or was replaced. Contention stays ABORTED and transient failures stay UNAVAILABLE. Adds API E2E coverage for the revert and documents manual recovery in the lifecycle architecture notes. Signed-off-by: Ada Lundhe Co-Authored-By: Claude Opus 5.5 --- docs/architecture/runtime-and-lifecycle.md | 40 +++ docs/lifecycle-retries.md | 30 +- go/api/gen/kagent/api/v1/sessions.pb.go | 267 ++++++++++++------ go/api/gen/kagent/api/v1/sessions_grpc.pb.go | 40 +++ go/core/internal/grpcserver/policy.go | 1 + go/core/internal/grpcserver/policy_test.go | 1 + .../internal/grpcserver/protovalidate_test.go | 1 + go/core/internal/grpcserver/session.go | 8 + .../grpcserver/session_recovery_test.go | 221 +++++++++++++++ go/core/internal/mcp/tasks_test.go | 4 + go/core/internal/service/session/service.go | 45 +++ .../internal/service/session/service_test.go | 72 ++++- go/core/test/e2e/session_revert_test.go | 171 +++++++++++ proto/kagent/api/v1/sessions.proto | 10 + ui/src/generated/kagent/api/v1/sessions_pb.ts | 64 ++++- 15 files changed, 864 insertions(+), 111 deletions(-) create mode 100644 go/core/internal/grpcserver/session_recovery_test.go create mode 100644 go/core/test/e2e/session_revert_test.go diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 7dfd1f0e8f..0a54e9d2f2 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -223,6 +223,46 @@ claims still block new work, but completed results remain readable. The recorded actor UID is checked before lifecycle calls; a same-name replacement cannot be adopted implicitly. +`SessionService.RevertSession` lets an authorized caller restore a confirmed +crashed runtime. It accepts active and idle Sessions, but verifies Substrate's +CRASHED state for the recorded Actor UID and pinned template before admitting +recovery. Quiet turns, transport failures, missing Actors and same-name +replacements cannot authorize a revert. A quiet turn returns `FAILED_PRECONDITION` +with reason `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. A missing or replaced Actor +returns `FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`, +because no retry restores it. A transport failure returns `UNAVAILABLE`. +Only an active lifecycle operation returns `ABORTED`. A Session at rest outside +READY returns `KAGENT_SESSION_RUNTIME_NOT_CRASHED`, so a client does not retry +it until its deadline. + +Admission records FAILED/RESUME with a RuntimeCrashed failure under the Session +lock. The same transaction publishes a durable terminal result saved before the +crash, or fails nonterminal work absent from the retained snapshot. Waiting tasks +captured by that snapshot remain available. Checkpoint creation excludes competing +recovery. + +The caller reads the Session's lifecycle generation before it observes Substrate. +Admission rejects the crash when another lifecycle operation began after that +read. A concurrent revert keeps the Actor UID and returns the Session to READY, +so only the generation exposes a stale observation. + +An issued idle suspension does not exclude recovery. It cannot finish against the +crashed Actor, and its worker keeps the claim when suspension fails. Admission +retires the claim in its transaction, so a late finish records nothing. + +The worker reverts the same Actor to its last durable snapshot without replaying +input. A runtime without a durable snapshot returns to its initial state. +An interrupted revert continues from REVERTING; a lost response can be reconciled +from SUSPENDED. Already admitted recoveries survive API restarts and retry on the +existing recovery interval in ID-ordered pages of at most 100. The worker does +not inspect READY Sessions or initiate a new recovery. + +A successful revert returns the Session to READY with the same Actor suspended. +The next authorized interaction resumes it through Substrate ingress. Runtime +changes after the retained snapshot are lost; the discarded turn receives no +checkpoint at that older snapshot. Another revert against the now-healthy Actor +returns `FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. + ```mermaid sequenceDiagram participant Client diff --git a/docs/lifecycle-retries.md b/docs/lifecycle-retries.md index cbb2cf71ec..3a9b129686 100644 --- a/docs/lifecycle-retries.md +++ b/docs/lifecycle-retries.md @@ -2,8 +2,8 @@ Session and Sandbox lifecycle mutations run within the API request. The server records the intended operation before calling the runtime and records completion -before returning success. Sessions expose Create, Fork, and Delete; explicit Suspend -and Resume are available only for Sandboxes. Session runtime suspension and resumption +before returning success. Sessions expose Create, Fork, Revert, and Delete; explicit +Suspend and Resume are available only for Sandboxes. Session runtime suspension and resumption are platform-managed. A failed request can have changed the runtime: a timeout does not mean that nothing happened, and canceling a request does not undo it. @@ -39,6 +39,32 @@ for subsequent invocations. Omitting it generates a new ID on each invocation, so rerunning that command is a new creation rather than a retry. MCP callers should likewise retain the `request_id` passed to `create_sandbox`. +## Recovering a crashed Session + +Call `SessionService.RevertSession` with the Session ID to restore its recorded runtime to the last durable snapshot. +The caller needs Session update access. The operation verifies Substrate's CRASHED state and the recorded Actor identity. +Recovery fails interrupted turns and waiting tasks absent from the retained snapshot. +It preserves durable terminal results and returns the Session to READY with its Actor suspended. +It discards runtime changes since that snapshot and does not replay input. +A runtime without a snapshot restarts from its initial state. + +Retry an interrupted revert against the same Session ID. Pending recovery retains its operation across API restarts. +The lifecycle worker retries already admitted recoveries; it does not initiate recovery for other crashed Sessions. +Checkpoint creation returns `ABORTED` until it finishes. +A revert that observed the crash before another lifecycle operation began also returns `ABORTED`. +So does a revert while another lifecycle operation is active. + +A revert that can never succeed against the Session as it stands returns `FAILED_PRECONDITION`. +Its `google.rpc.ErrorInfo` detail, in domain `kagent.dev`, names one of two reasons: + +| Reason | Meaning | Client action | +| --- | --- | --- | +| `KAGENT_SESSION_RUNTIME_NOT_CRASHED` | The runtime is healthy or suspended. A retry after a revert whose response was lost lands here. | Treat as recovered and continue the conversation. | +| `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` | The recorded Actor is missing, or a same-name Actor replaced it. | Stop retrying. Start another Session, or fork one from a checkpoint. | + +A `FAILED_PRECONDITION` without either reason follows the general table below. +Read the reason rather than the message text, which is not stable. + ## Errors | gRPC status | Client action | diff --git a/go/api/gen/kagent/api/v1/sessions.pb.go b/go/api/gen/kagent/api/v1/sessions.pb.go index 5657cda54f..ab556d6f9b 100644 --- a/go/api/gen/kagent/api/v1/sessions.pb.go +++ b/go/api/gen/kagent/api/v1/sessions.pb.go @@ -669,6 +669,94 @@ func (x *UpdateSessionNameResponse) GetSession() *Session { return nil } +type RevertSessionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevertSessionRequest) Reset() { + *x = RevertSessionRequest{} + mi := &file_kagent_api_v1_sessions_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevertSessionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertSessionRequest) ProtoMessage() {} + +func (x *RevertSessionRequest) ProtoReflect() protoreflect.Message { + mi := &file_kagent_api_v1_sessions_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertSessionRequest.ProtoReflect.Descriptor instead. +func (*RevertSessionRequest) Descriptor() ([]byte, []int) { + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{11} +} + +func (x *RevertSessionRequest) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +type RevertSessionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Session *Session `protobuf:"bytes,1,opt,name=session,proto3" json:"session,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevertSessionResponse) Reset() { + *x = RevertSessionResponse{} + mi := &file_kagent_api_v1_sessions_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevertSessionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertSessionResponse) ProtoMessage() {} + +func (x *RevertSessionResponse) ProtoReflect() protoreflect.Message { + mi := &file_kagent_api_v1_sessions_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertSessionResponse.ProtoReflect.Descriptor instead. +func (*RevertSessionResponse) Descriptor() ([]byte, []int) { + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{12} +} + +func (x *RevertSessionResponse) GetSession() *Session { + if x != nil { + return x.Session + } + return nil +} + type DeleteSessionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` @@ -681,7 +769,7 @@ type DeleteSessionRequest struct { func (x *DeleteSessionRequest) Reset() { *x = DeleteSessionRequest{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[11] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -693,7 +781,7 @@ func (x *DeleteSessionRequest) String() string { func (*DeleteSessionRequest) ProtoMessage() {} func (x *DeleteSessionRequest) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[11] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -706,7 +794,7 @@ func (x *DeleteSessionRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteSessionRequest.ProtoReflect.Descriptor instead. func (*DeleteSessionRequest) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{11} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{13} } func (x *DeleteSessionRequest) GetSessionId() string { @@ -732,7 +820,7 @@ type DeleteSessionResponse struct { func (x *DeleteSessionResponse) Reset() { *x = DeleteSessionResponse{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[12] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -744,7 +832,7 @@ func (x *DeleteSessionResponse) String() string { func (*DeleteSessionResponse) ProtoMessage() {} func (x *DeleteSessionResponse) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[12] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -757,7 +845,7 @@ func (x *DeleteSessionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteSessionResponse.ProtoReflect.Descriptor instead. func (*DeleteSessionResponse) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{12} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{14} } func (x *DeleteSessionResponse) GetSession() *Session { @@ -781,7 +869,7 @@ type SessionShare struct { func (x *SessionShare) Reset() { *x = SessionShare{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[13] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -793,7 +881,7 @@ func (x *SessionShare) String() string { func (*SessionShare) ProtoMessage() {} func (x *SessionShare) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[13] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -806,7 +894,7 @@ func (x *SessionShare) ProtoReflect() protoreflect.Message { // Deprecated: Use SessionShare.ProtoReflect.Descriptor instead. func (*SessionShare) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{13} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{15} } func (x *SessionShare) GetId() string { @@ -850,7 +938,7 @@ type CreateSessionShareRequest struct { func (x *CreateSessionShareRequest) Reset() { *x = CreateSessionShareRequest{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[14] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -862,7 +950,7 @@ func (x *CreateSessionShareRequest) String() string { func (*CreateSessionShareRequest) ProtoMessage() {} func (x *CreateSessionShareRequest) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[14] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -875,7 +963,7 @@ func (x *CreateSessionShareRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSessionShareRequest.ProtoReflect.Descriptor instead. func (*CreateSessionShareRequest) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{14} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{16} } func (x *CreateSessionShareRequest) GetSessionId() string { @@ -903,7 +991,7 @@ type CreateSessionShareResponse struct { func (x *CreateSessionShareResponse) Reset() { *x = CreateSessionShareResponse{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[15] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -915,7 +1003,7 @@ func (x *CreateSessionShareResponse) String() string { func (*CreateSessionShareResponse) ProtoMessage() {} func (x *CreateSessionShareResponse) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[15] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -928,7 +1016,7 @@ func (x *CreateSessionShareResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSessionShareResponse.ProtoReflect.Descriptor instead. func (*CreateSessionShareResponse) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{15} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{17} } func (x *CreateSessionShareResponse) GetShare() *SessionShare { @@ -955,7 +1043,7 @@ type ListSessionSharesRequest struct { func (x *ListSessionSharesRequest) Reset() { *x = ListSessionSharesRequest{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[16] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -967,7 +1055,7 @@ func (x *ListSessionSharesRequest) String() string { func (*ListSessionSharesRequest) ProtoMessage() {} func (x *ListSessionSharesRequest) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[16] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -980,7 +1068,7 @@ func (x *ListSessionSharesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSessionSharesRequest.ProtoReflect.Descriptor instead. func (*ListSessionSharesRequest) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{16} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{18} } func (x *ListSessionSharesRequest) GetSessionId() string { @@ -1007,7 +1095,7 @@ type ListSessionSharesResponse struct { func (x *ListSessionSharesResponse) Reset() { *x = ListSessionSharesResponse{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[17] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1019,7 +1107,7 @@ func (x *ListSessionSharesResponse) String() string { func (*ListSessionSharesResponse) ProtoMessage() {} func (x *ListSessionSharesResponse) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[17] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1032,7 +1120,7 @@ func (x *ListSessionSharesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSessionSharesResponse.ProtoReflect.Descriptor instead. func (*ListSessionSharesResponse) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{17} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{19} } func (x *ListSessionSharesResponse) GetShares() []*SessionShare { @@ -1058,7 +1146,7 @@ type RevokeSessionShareRequest struct { func (x *RevokeSessionShareRequest) Reset() { *x = RevokeSessionShareRequest{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[18] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1070,7 +1158,7 @@ func (x *RevokeSessionShareRequest) String() string { func (*RevokeSessionShareRequest) ProtoMessage() {} func (x *RevokeSessionShareRequest) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[18] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1083,7 +1171,7 @@ func (x *RevokeSessionShareRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RevokeSessionShareRequest.ProtoReflect.Descriptor instead. func (*RevokeSessionShareRequest) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{18} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{20} } func (x *RevokeSessionShareRequest) GetShareId() string { @@ -1101,7 +1189,7 @@ type RevokeSessionShareResponse struct { func (x *RevokeSessionShareResponse) Reset() { *x = RevokeSessionShareResponse{} - mi := &file_kagent_api_v1_sessions_proto_msgTypes[19] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1113,7 +1201,7 @@ func (x *RevokeSessionShareResponse) String() string { func (*RevokeSessionShareResponse) ProtoMessage() {} func (x *RevokeSessionShareResponse) ProtoReflect() protoreflect.Message { - mi := &file_kagent_api_v1_sessions_proto_msgTypes[19] + mi := &file_kagent_api_v1_sessions_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1126,7 +1214,7 @@ func (x *RevokeSessionShareResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RevokeSessionShareResponse.ProtoReflect.Descriptor instead. func (*RevokeSessionShareResponse) Descriptor() ([]byte, []int) { - return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{19} + return file_kagent_api_v1_sessions_proto_rawDescGZIP(), []int{21} } var File_kagent_api_v1_sessions_proto protoreflect.FileDescriptor @@ -1182,6 +1270,11 @@ const file_kagent_api_v1_sessions_proto_rawDesc = "" + "session_id\x18\x01 \x01(\tB\b\xbaH\x05r\x03\xb0\x01\x01R\tsessionId\x12Q\n" + "\x04name\x18\x02 \x01(\tB=\xbaH:r8\x18\xc8\x0123^(?:$|[^\\p{Z}\\p{Cc}](?:[^\\p{Cc}]*[^\\p{Z}\\p{Cc}])?)$R\x04name\"M\n" + "\x19UpdateSessionNameResponse\x120\n" + + "\asession\x18\x01 \x01(\v2\x16.kagent.api.v1.SessionR\asession\"?\n" + + "\x14RevertSessionRequest\x12'\n" + + "\n" + + "session_id\x18\x01 \x01(\tB\b\xbaH\x05r\x03\xb0\x01\x01R\tsessionId\"I\n" + + "\x15RevertSessionResponse\x120\n" + "\asession\x18\x01 \x01(\v2\x16.kagent.api.v1.SessionR\asession\"U\n" + "\x14DeleteSessionRequest\x12'\n" + "\n" + @@ -1213,13 +1306,14 @@ const file_kagent_api_v1_sessions_proto_rawDesc = "" + "\x04page\x18\x02 \x01(\v2\x1b.kagent.api.v1.PageResponseR\x04page\"@\n" + "\x19RevokeSessionShareRequest\x12#\n" + "\bshare_id\x18\x01 \x01(\tB\b\xbaH\x05r\x03\xb0\x01\x01R\ashareId\"\x1c\n" + - "\x1aRevokeSessionShareResponse2\x9a\x06\n" + + "\x1aRevokeSessionShareResponse2\xf6\x06\n" + "\x0eSessionService\x12Z\n" + "\rCreateSession\x12#.kagent.api.v1.CreateSessionRequest\x1a$.kagent.api.v1.CreateSessionResponse\x12Q\n" + "\n" + "GetSession\x12 .kagent.api.v1.GetSessionRequest\x1a!.kagent.api.v1.GetSessionResponse\x12W\n" + "\fListSessions\x12\".kagent.api.v1.ListSessionsRequest\x1a#.kagent.api.v1.ListSessionsResponse\x12f\n" + "\x11UpdateSessionName\x12'.kagent.api.v1.UpdateSessionNameRequest\x1a(.kagent.api.v1.UpdateSessionNameResponse\x12Z\n" + + "\rRevertSession\x12#.kagent.api.v1.RevertSessionRequest\x1a$.kagent.api.v1.RevertSessionResponse\x12Z\n" + "\rDeleteSession\x12#.kagent.api.v1.DeleteSessionRequest\x1a$.kagent.api.v1.DeleteSessionResponse\x12i\n" + "\x12CreateSessionShare\x12(.kagent.api.v1.CreateSessionShareRequest\x1a).kagent.api.v1.CreateSessionShareResponse\x12f\n" + "\x11ListSessionShares\x12'.kagent.api.v1.ListSessionSharesRequest\x1a(.kagent.api.v1.ListSessionSharesResponse\x12i\n" + @@ -1237,7 +1331,7 @@ func file_kagent_api_v1_sessions_proto_rawDescGZIP() []byte { return file_kagent_api_v1_sessions_proto_rawDescData } -var file_kagent_api_v1_sessions_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_kagent_api_v1_sessions_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_kagent_api_v1_sessions_proto_goTypes = []any{ (*Failure)(nil), // 0: kagent.api.v1.Failure (*Session)(nil), // 1: kagent.api.v1.Session @@ -1250,68 +1344,73 @@ var file_kagent_api_v1_sessions_proto_goTypes = []any{ (*ListSessionsResponse)(nil), // 8: kagent.api.v1.ListSessionsResponse (*UpdateSessionNameRequest)(nil), // 9: kagent.api.v1.UpdateSessionNameRequest (*UpdateSessionNameResponse)(nil), // 10: kagent.api.v1.UpdateSessionNameResponse - (*DeleteSessionRequest)(nil), // 11: kagent.api.v1.DeleteSessionRequest - (*DeleteSessionResponse)(nil), // 12: kagent.api.v1.DeleteSessionResponse - (*SessionShare)(nil), // 13: kagent.api.v1.SessionShare - (*CreateSessionShareRequest)(nil), // 14: kagent.api.v1.CreateSessionShareRequest - (*CreateSessionShareResponse)(nil), // 15: kagent.api.v1.CreateSessionShareResponse - (*ListSessionSharesRequest)(nil), // 16: kagent.api.v1.ListSessionSharesRequest - (*ListSessionSharesResponse)(nil), // 17: kagent.api.v1.ListSessionSharesResponse - (*RevokeSessionShareRequest)(nil), // 18: kagent.api.v1.RevokeSessionShareRequest - (*RevokeSessionShareResponse)(nil), // 19: kagent.api.v1.RevokeSessionShareResponse - (RuntimeState)(0), // 20: kagent.api.v1.RuntimeState - (RuntimeOperation)(0), // 21: kagent.api.v1.RuntimeOperation - (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp - (*ResourceReference)(nil), // 23: kagent.api.v1.ResourceReference - (*PageRequest)(nil), // 24: kagent.api.v1.PageRequest - (*PageResponse)(nil), // 25: kagent.api.v1.PageResponse - (*durationpb.Duration)(nil), // 26: google.protobuf.Duration + (*RevertSessionRequest)(nil), // 11: kagent.api.v1.RevertSessionRequest + (*RevertSessionResponse)(nil), // 12: kagent.api.v1.RevertSessionResponse + (*DeleteSessionRequest)(nil), // 13: kagent.api.v1.DeleteSessionRequest + (*DeleteSessionResponse)(nil), // 14: kagent.api.v1.DeleteSessionResponse + (*SessionShare)(nil), // 15: kagent.api.v1.SessionShare + (*CreateSessionShareRequest)(nil), // 16: kagent.api.v1.CreateSessionShareRequest + (*CreateSessionShareResponse)(nil), // 17: kagent.api.v1.CreateSessionShareResponse + (*ListSessionSharesRequest)(nil), // 18: kagent.api.v1.ListSessionSharesRequest + (*ListSessionSharesResponse)(nil), // 19: kagent.api.v1.ListSessionSharesResponse + (*RevokeSessionShareRequest)(nil), // 20: kagent.api.v1.RevokeSessionShareRequest + (*RevokeSessionShareResponse)(nil), // 21: kagent.api.v1.RevokeSessionShareResponse + (RuntimeState)(0), // 22: kagent.api.v1.RuntimeState + (RuntimeOperation)(0), // 23: kagent.api.v1.RuntimeOperation + (*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp + (*ResourceReference)(nil), // 25: kagent.api.v1.ResourceReference + (*PageRequest)(nil), // 26: kagent.api.v1.PageRequest + (*PageResponse)(nil), // 27: kagent.api.v1.PageResponse + (*durationpb.Duration)(nil), // 28: google.protobuf.Duration } var file_kagent_api_v1_sessions_proto_depIdxs = []int32{ 2, // 0: kagent.api.v1.Session.agent:type_name -> kagent.api.v1.AgentProvenance - 20, // 1: kagent.api.v1.Session.state:type_name -> kagent.api.v1.RuntimeState - 21, // 2: kagent.api.v1.Session.operation:type_name -> kagent.api.v1.RuntimeOperation + 22, // 1: kagent.api.v1.Session.state:type_name -> kagent.api.v1.RuntimeState + 23, // 2: kagent.api.v1.Session.operation:type_name -> kagent.api.v1.RuntimeOperation 0, // 3: kagent.api.v1.Session.failure:type_name -> kagent.api.v1.Failure - 22, // 4: kagent.api.v1.Session.created_at:type_name -> google.protobuf.Timestamp - 22, // 5: kagent.api.v1.Session.updated_at:type_name -> google.protobuf.Timestamp - 23, // 6: kagent.api.v1.AgentProvenance.ref:type_name -> kagent.api.v1.ResourceReference - 23, // 7: kagent.api.v1.CreateSessionRequest.agent:type_name -> kagent.api.v1.ResourceReference + 24, // 4: kagent.api.v1.Session.created_at:type_name -> google.protobuf.Timestamp + 24, // 5: kagent.api.v1.Session.updated_at:type_name -> google.protobuf.Timestamp + 25, // 6: kagent.api.v1.AgentProvenance.ref:type_name -> kagent.api.v1.ResourceReference + 25, // 7: kagent.api.v1.CreateSessionRequest.agent:type_name -> kagent.api.v1.ResourceReference 1, // 8: kagent.api.v1.CreateSessionResponse.session:type_name -> kagent.api.v1.Session 1, // 9: kagent.api.v1.GetSessionResponse.session:type_name -> kagent.api.v1.Session - 24, // 10: kagent.api.v1.ListSessionsRequest.page:type_name -> kagent.api.v1.PageRequest - 23, // 11: kagent.api.v1.ListSessionsRequest.agent:type_name -> kagent.api.v1.ResourceReference + 26, // 10: kagent.api.v1.ListSessionsRequest.page:type_name -> kagent.api.v1.PageRequest + 25, // 11: kagent.api.v1.ListSessionsRequest.agent:type_name -> kagent.api.v1.ResourceReference 1, // 12: kagent.api.v1.ListSessionsResponse.sessions:type_name -> kagent.api.v1.Session - 25, // 13: kagent.api.v1.ListSessionsResponse.page:type_name -> kagent.api.v1.PageResponse + 27, // 13: kagent.api.v1.ListSessionsResponse.page:type_name -> kagent.api.v1.PageResponse 1, // 14: kagent.api.v1.UpdateSessionNameResponse.session:type_name -> kagent.api.v1.Session - 1, // 15: kagent.api.v1.DeleteSessionResponse.session:type_name -> kagent.api.v1.Session - 22, // 16: kagent.api.v1.SessionShare.created_at:type_name -> google.protobuf.Timestamp - 22, // 17: kagent.api.v1.SessionShare.expires_at:type_name -> google.protobuf.Timestamp - 26, // 18: kagent.api.v1.CreateSessionShareRequest.ttl:type_name -> google.protobuf.Duration - 13, // 19: kagent.api.v1.CreateSessionShareResponse.share:type_name -> kagent.api.v1.SessionShare - 24, // 20: kagent.api.v1.ListSessionSharesRequest.page:type_name -> kagent.api.v1.PageRequest - 13, // 21: kagent.api.v1.ListSessionSharesResponse.shares:type_name -> kagent.api.v1.SessionShare - 25, // 22: kagent.api.v1.ListSessionSharesResponse.page:type_name -> kagent.api.v1.PageResponse - 3, // 23: kagent.api.v1.SessionService.CreateSession:input_type -> kagent.api.v1.CreateSessionRequest - 5, // 24: kagent.api.v1.SessionService.GetSession:input_type -> kagent.api.v1.GetSessionRequest - 7, // 25: kagent.api.v1.SessionService.ListSessions:input_type -> kagent.api.v1.ListSessionsRequest - 9, // 26: kagent.api.v1.SessionService.UpdateSessionName:input_type -> kagent.api.v1.UpdateSessionNameRequest - 11, // 27: kagent.api.v1.SessionService.DeleteSession:input_type -> kagent.api.v1.DeleteSessionRequest - 14, // 28: kagent.api.v1.SessionService.CreateSessionShare:input_type -> kagent.api.v1.CreateSessionShareRequest - 16, // 29: kagent.api.v1.SessionService.ListSessionShares:input_type -> kagent.api.v1.ListSessionSharesRequest - 18, // 30: kagent.api.v1.SessionService.RevokeSessionShare:input_type -> kagent.api.v1.RevokeSessionShareRequest - 4, // 31: kagent.api.v1.SessionService.CreateSession:output_type -> kagent.api.v1.CreateSessionResponse - 6, // 32: kagent.api.v1.SessionService.GetSession:output_type -> kagent.api.v1.GetSessionResponse - 8, // 33: kagent.api.v1.SessionService.ListSessions:output_type -> kagent.api.v1.ListSessionsResponse - 10, // 34: kagent.api.v1.SessionService.UpdateSessionName:output_type -> kagent.api.v1.UpdateSessionNameResponse - 12, // 35: kagent.api.v1.SessionService.DeleteSession:output_type -> kagent.api.v1.DeleteSessionResponse - 15, // 36: kagent.api.v1.SessionService.CreateSessionShare:output_type -> kagent.api.v1.CreateSessionShareResponse - 17, // 37: kagent.api.v1.SessionService.ListSessionShares:output_type -> kagent.api.v1.ListSessionSharesResponse - 19, // 38: kagent.api.v1.SessionService.RevokeSessionShare:output_type -> kagent.api.v1.RevokeSessionShareResponse - 31, // [31:39] is the sub-list for method output_type - 23, // [23:31] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 1, // 15: kagent.api.v1.RevertSessionResponse.session:type_name -> kagent.api.v1.Session + 1, // 16: kagent.api.v1.DeleteSessionResponse.session:type_name -> kagent.api.v1.Session + 24, // 17: kagent.api.v1.SessionShare.created_at:type_name -> google.protobuf.Timestamp + 24, // 18: kagent.api.v1.SessionShare.expires_at:type_name -> google.protobuf.Timestamp + 28, // 19: kagent.api.v1.CreateSessionShareRequest.ttl:type_name -> google.protobuf.Duration + 15, // 20: kagent.api.v1.CreateSessionShareResponse.share:type_name -> kagent.api.v1.SessionShare + 26, // 21: kagent.api.v1.ListSessionSharesRequest.page:type_name -> kagent.api.v1.PageRequest + 15, // 22: kagent.api.v1.ListSessionSharesResponse.shares:type_name -> kagent.api.v1.SessionShare + 27, // 23: kagent.api.v1.ListSessionSharesResponse.page:type_name -> kagent.api.v1.PageResponse + 3, // 24: kagent.api.v1.SessionService.CreateSession:input_type -> kagent.api.v1.CreateSessionRequest + 5, // 25: kagent.api.v1.SessionService.GetSession:input_type -> kagent.api.v1.GetSessionRequest + 7, // 26: kagent.api.v1.SessionService.ListSessions:input_type -> kagent.api.v1.ListSessionsRequest + 9, // 27: kagent.api.v1.SessionService.UpdateSessionName:input_type -> kagent.api.v1.UpdateSessionNameRequest + 11, // 28: kagent.api.v1.SessionService.RevertSession:input_type -> kagent.api.v1.RevertSessionRequest + 13, // 29: kagent.api.v1.SessionService.DeleteSession:input_type -> kagent.api.v1.DeleteSessionRequest + 16, // 30: kagent.api.v1.SessionService.CreateSessionShare:input_type -> kagent.api.v1.CreateSessionShareRequest + 18, // 31: kagent.api.v1.SessionService.ListSessionShares:input_type -> kagent.api.v1.ListSessionSharesRequest + 20, // 32: kagent.api.v1.SessionService.RevokeSessionShare:input_type -> kagent.api.v1.RevokeSessionShareRequest + 4, // 33: kagent.api.v1.SessionService.CreateSession:output_type -> kagent.api.v1.CreateSessionResponse + 6, // 34: kagent.api.v1.SessionService.GetSession:output_type -> kagent.api.v1.GetSessionResponse + 8, // 35: kagent.api.v1.SessionService.ListSessions:output_type -> kagent.api.v1.ListSessionsResponse + 10, // 36: kagent.api.v1.SessionService.UpdateSessionName:output_type -> kagent.api.v1.UpdateSessionNameResponse + 12, // 37: kagent.api.v1.SessionService.RevertSession:output_type -> kagent.api.v1.RevertSessionResponse + 14, // 38: kagent.api.v1.SessionService.DeleteSession:output_type -> kagent.api.v1.DeleteSessionResponse + 17, // 39: kagent.api.v1.SessionService.CreateSessionShare:output_type -> kagent.api.v1.CreateSessionShareResponse + 19, // 40: kagent.api.v1.SessionService.ListSessionShares:output_type -> kagent.api.v1.ListSessionSharesResponse + 21, // 41: kagent.api.v1.SessionService.RevokeSessionShare:output_type -> kagent.api.v1.RevokeSessionShareResponse + 33, // [33:42] is the sub-list for method output_type + 24, // [24:33] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_kagent_api_v1_sessions_proto_init() } @@ -1327,7 +1426,7 @@ func file_kagent_api_v1_sessions_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_kagent_api_v1_sessions_proto_rawDesc), len(file_kagent_api_v1_sessions_proto_rawDesc)), NumEnums: 0, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/go/api/gen/kagent/api/v1/sessions_grpc.pb.go b/go/api/gen/kagent/api/v1/sessions_grpc.pb.go index f5bfe3112d..4c262b9426 100644 --- a/go/api/gen/kagent/api/v1/sessions_grpc.pb.go +++ b/go/api/gen/kagent/api/v1/sessions_grpc.pb.go @@ -23,6 +23,7 @@ const ( SessionService_GetSession_FullMethodName = "/kagent.api.v1.SessionService/GetSession" SessionService_ListSessions_FullMethodName = "/kagent.api.v1.SessionService/ListSessions" SessionService_UpdateSessionName_FullMethodName = "/kagent.api.v1.SessionService/UpdateSessionName" + SessionService_RevertSession_FullMethodName = "/kagent.api.v1.SessionService/RevertSession" SessionService_DeleteSession_FullMethodName = "/kagent.api.v1.SessionService/DeleteSession" SessionService_CreateSessionShare_FullMethodName = "/kagent.api.v1.SessionService/CreateSessionShare" SessionService_ListSessionShares_FullMethodName = "/kagent.api.v1.SessionService/ListSessionShares" @@ -43,6 +44,8 @@ type SessionServiceClient interface { GetSession(ctx context.Context, in *GetSessionRequest, opts ...grpc.CallOption) (*GetSessionResponse, error) ListSessions(ctx context.Context, in *ListSessionsRequest, opts ...grpc.CallOption) (*ListSessionsResponse, error) UpdateSessionName(ctx context.Context, in *UpdateSessionNameRequest, opts ...grpc.CallOption) (*UpdateSessionNameResponse, error) + // Restores a crashed runtime to its last snapshot without replaying its turn. + RevertSession(ctx context.Context, in *RevertSessionRequest, opts ...grpc.CallOption) (*RevertSessionResponse, error) DeleteSession(ctx context.Context, in *DeleteSessionRequest, opts ...grpc.CallOption) (*DeleteSessionResponse, error) CreateSessionShare(ctx context.Context, in *CreateSessionShareRequest, opts ...grpc.CallOption) (*CreateSessionShareResponse, error) ListSessionShares(ctx context.Context, in *ListSessionSharesRequest, opts ...grpc.CallOption) (*ListSessionSharesResponse, error) @@ -97,6 +100,16 @@ func (c *sessionServiceClient) UpdateSessionName(ctx context.Context, in *Update return out, nil } +func (c *sessionServiceClient) RevertSession(ctx context.Context, in *RevertSessionRequest, opts ...grpc.CallOption) (*RevertSessionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RevertSessionResponse) + err := c.cc.Invoke(ctx, SessionService_RevertSession_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *sessionServiceClient) DeleteSession(ctx context.Context, in *DeleteSessionRequest, opts ...grpc.CallOption) (*DeleteSessionResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteSessionResponse) @@ -151,6 +164,8 @@ type SessionServiceServer interface { GetSession(context.Context, *GetSessionRequest) (*GetSessionResponse, error) ListSessions(context.Context, *ListSessionsRequest) (*ListSessionsResponse, error) UpdateSessionName(context.Context, *UpdateSessionNameRequest) (*UpdateSessionNameResponse, error) + // Restores a crashed runtime to its last snapshot without replaying its turn. + RevertSession(context.Context, *RevertSessionRequest) (*RevertSessionResponse, error) DeleteSession(context.Context, *DeleteSessionRequest) (*DeleteSessionResponse, error) CreateSessionShare(context.Context, *CreateSessionShareRequest) (*CreateSessionShareResponse, error) ListSessionShares(context.Context, *ListSessionSharesRequest) (*ListSessionSharesResponse, error) @@ -177,6 +192,9 @@ func (UnimplementedSessionServiceServer) ListSessions(context.Context, *ListSess func (UnimplementedSessionServiceServer) UpdateSessionName(context.Context, *UpdateSessionNameRequest) (*UpdateSessionNameResponse, error) { return nil, status.Error(codes.Unimplemented, "method UpdateSessionName not implemented") } +func (UnimplementedSessionServiceServer) RevertSession(context.Context, *RevertSessionRequest) (*RevertSessionResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RevertSession not implemented") +} func (UnimplementedSessionServiceServer) DeleteSession(context.Context, *DeleteSessionRequest) (*DeleteSessionResponse, error) { return nil, status.Error(codes.Unimplemented, "method DeleteSession not implemented") } @@ -282,6 +300,24 @@ func _SessionService_UpdateSessionName_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _SessionService_RevertSession_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevertSessionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SessionServiceServer).RevertSession(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SessionService_RevertSession_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SessionServiceServer).RevertSession(ctx, req.(*RevertSessionRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _SessionService_DeleteSession_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteSessionRequest) if err := dec(in); err != nil { @@ -377,6 +413,10 @@ var SessionService_ServiceDesc = grpc.ServiceDesc{ MethodName: "UpdateSessionName", Handler: _SessionService_UpdateSessionName_Handler, }, + { + MethodName: "RevertSession", + Handler: _SessionService_RevertSession_Handler, + }, { MethodName: "DeleteSession", Handler: _SessionService_DeleteSession_Handler, diff --git a/go/core/internal/grpcserver/policy.go b/go/core/internal/grpcserver/policy.go index 74b9a66dd8..e6a249d6f5 100644 --- a/go/core/internal/grpcserver/policy.go +++ b/go/core/internal/grpcserver/policy.go @@ -81,6 +81,7 @@ func DefaultMethodPolicies() MethodPolicies { // A rename is the only write on this service that is not a lifecycle // operation, and it must not inherit the read mode its neighbours carry. policies[apiv1.SessionService_UpdateSessionName_FullMethodName] = auth.AccessUpdate + policies[apiv1.SessionService_RevertSession_FullMethodName] = auth.AccessUpdate policies[apiv1.SessionService_DeleteSession_FullMethodName] = auth.AccessDelete policies[apiv1.SessionService_CreateSessionShare_FullMethodName] = auth.AccessCreate policies[apiv1.SessionService_ListSessionShares_FullMethodName] = auth.AccessRead diff --git a/go/core/internal/grpcserver/policy_test.go b/go/core/internal/grpcserver/policy_test.go index 712e2e243a..655c8e926c 100644 --- a/go/core/internal/grpcserver/policy_test.go +++ b/go/core/internal/grpcserver/policy_test.go @@ -38,6 +38,7 @@ func TestSessionServicePoliciesMatchTheirEffect(t *testing.T) { {name: "get", method: apiv1.SessionService_GetSession_FullMethodName, want: pkgauth.AccessRead}, {name: "list", method: apiv1.SessionService_ListSessions_FullMethodName, want: pkgauth.AccessRead}, {name: "rename is a write", method: apiv1.SessionService_UpdateSessionName_FullMethodName, want: pkgauth.AccessUpdate}, + {name: "revert", method: apiv1.SessionService_RevertSession_FullMethodName, want: pkgauth.AccessUpdate}, {name: "delete", method: apiv1.SessionService_DeleteSession_FullMethodName, want: pkgauth.AccessDelete}, {name: "create share", method: apiv1.SessionService_CreateSessionShare_FullMethodName, want: pkgauth.AccessCreate}, {name: "list shares", method: apiv1.SessionService_ListSessionShares_FullMethodName, want: pkgauth.AccessRead}, diff --git a/go/core/internal/grpcserver/protovalidate_test.go b/go/core/internal/grpcserver/protovalidate_test.go index 740ce014fe..64e1877b20 100644 --- a/go/core/internal/grpcserver/protovalidate_test.go +++ b/go/core/internal/grpcserver/protovalidate_test.go @@ -88,6 +88,7 @@ func TestInvalidSessionAndCheckpointIDsNeverReachHandlers(t *testing.T) { requests := []proto.Message{ &apiv1.GetSessionRequest{SessionId: "invalid"}, &apiv1.UpdateSessionNameRequest{SessionId: "invalid"}, + &apiv1.RevertSessionRequest{SessionId: "invalid"}, &apiv1.DeleteSessionRequest{SessionId: "invalid"}, &apiv1.CreateSessionShareRequest{SessionId: "invalid"}, &apiv1.ListSessionSharesRequest{SessionId: "invalid"}, diff --git a/go/core/internal/grpcserver/session.go b/go/core/internal/grpcserver/session.go index e84263bbf9..ed64db9edc 100644 --- a/go/core/internal/grpcserver/session.go +++ b/go/core/internal/grpcserver/session.go @@ -51,6 +51,14 @@ func (s *sessionServer) UpdateSessionName(ctx context.Context, request *apiv1.Up return &apiv1.UpdateSessionNameResponse{Session: session}, nil } +func (s *sessionServer) RevertSession(ctx context.Context, request *apiv1.RevertSessionRequest) (*apiv1.RevertSessionResponse, error) { + session, err := s.service.Revert(ctx, request.GetSessionId()) + if err != nil { + return nil, err + } + return &apiv1.RevertSessionResponse{Session: session}, nil +} + func (s *sessionServer) DeleteSession(ctx context.Context, request *apiv1.DeleteSessionRequest) (*apiv1.DeleteSessionResponse, error) { session, err := s.service.Delete(ctx, request.GetSessionId(), request.GetForce()) if err != nil { diff --git a/go/core/internal/grpcserver/session_recovery_test.go b/go/core/internal/grpcserver/session_recovery_test.go new file mode 100644 index 0000000000..d737b784b9 --- /dev/null +++ b/go/core/internal/grpcserver/session_recovery_test.go @@ -0,0 +1,221 @@ +package grpcserver + +import ( + "context" + "crypto/sha256" + "net" + "sync" + "testing" + "time" + + "github.com/a2aproject/a2a-go/v2/a2a" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + ateapipb "go.ate.dev/substrate/api/ateapi/v1" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/database" + "github.com/kagent-dev/kagent/go/core/internal/dbtest" + authimpl "github.com/kagent-dev/kagent/go/core/internal/httpserver/auth" + sessionsvc "github.com/kagent-dev/kagent/go/core/internal/service/session" + "github.com/kagent-dev/kagent/go/core/internal/substrate" + "github.com/kagent-dev/kagent/go/core/pkg/auth" +) + +type revertTestControl struct { + ateapipb.ControlClient + mu sync.Mutex + actors map[string]*ateapipb.Actor + revert func(*ateapipb.Actor) error +} + +func (c *revertTestControl) GetActor(_ context.Context, request *ateapipb.GetActorRequest, _ ...grpc.CallOption) (*ateapipb.Actor, error) { + c.mu.Lock() + defer c.mu.Unlock() + actor := c.actors[request.GetActor().GetName()] + if actor == nil { + return nil, status.Error(codes.NotFound, "actor not found") + } + return proto.CloneOf(actor), nil +} + +func (c *revertTestControl) RevertActor(_ context.Context, request *ateapipb.RevertActorRequest, _ ...grpc.CallOption) (*ateapipb.RevertActorResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + actor := c.actors[request.GetActor().GetName()] + if c.revert != nil { + if err := c.revert(actor); err != nil { + return nil, err + } + } + actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + return &ateapipb.RevertActorResponse{Actor: proto.CloneOf(actor)}, nil +} + +func TestRevertSessionThroughGRPC(t *testing.T) { + if testing.Short() { + t.Skip("requires PostgreSQL") + } + dsn := dbtest.StartT(context.WithoutCancel(t.Context()), t) + dbtest.MigrateT(t, dsn, false) + db, err := database.Connect(t.Context(), &database.PostgresConfig{Pool: dbtest.PoolConfig(dsn)}) + require.NoError(t, err) + t.Cleanup(db.Close) + store := database.NewClient(db, "public") + control := &revertTestControl{actors: map[string]*ateapipb.Actor{}} + actors := &substrate.Client{ControlClient: control} + workflow := sessionsvc.NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + listener := bufconn.Listen(DefaultMaxMessageSize) + server, err := New(Config{ + Listener: listener, Authenticator: &authimpl.InsecureAuthenticator{}, + SystemService: testSystemService(), ShareStore: store, + SessionService: sessionsvc.NewService(store, &auth.NoopAuthorizer{}, workflow), + }) + require.NoError(t, err) + ctx, stop := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- server.Start(ctx) }() + t.Cleanup(func() { stop(); require.NoError(t, <-done) }) + connection, err := grpc.NewClient("passthrough:///bufnet", grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { return listener.Dial() })) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, connection.Close()) }) + client := apiv1.NewSessionServiceClient(connection) + owner := metadata.NewOutgoingContext(t.Context(), metadata.Pairs("x-user-id", "alice")) + visitor := metadata.NewOutgoingContext(t.Context(), metadata.Pairs("x-user-id", "bob")) + _, err = client.RevertSession(owner, &apiv1.RevertSessionRequest{SessionId: "invalid"}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + + for _, test := range []struct { + name string + task a2a.TaskState + noSnapshot bool + lostResponse bool + }{ + {name: "interrupted turn", task: a2a.TaskStateWorking}, + {name: "completed turn", task: a2a.TaskStateCompleted}, + {name: "idle session"}, + {name: "published input wait", task: a2a.TaskStateInputRequired}, + {name: "published auth wait", task: a2a.TaskStateAuthRequired}, + {name: "first-turn crash", task: a2a.TaskStateWorking, noSnapshot: true}, + {name: "revert response lost", task: a2a.TaskStateWorking, lostResponse: true}, + } { + t.Run(test.name, func(t *testing.T) { + session := createTaskStoreSession(t, store) + name := substrate.ActorName(session.Id) + control.actors[name] = &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: name, Uid: "actor-uid"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: "team-a", Name: "assistant-kagent-revision"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + Snapshots: []*ateapipb.Snapshot{{Generation: 1, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapipb.ObjectSnapshot{SnapshotUri: "s3://snapshots/last-good"}}}}}, + } + var task *a2a.Task + if test.task != "" { + message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("original input")) + message.ContextID = session.ContextId + task = a2a.NewSubmittedTask(message, message) + task.Status.State = test.task + hash := sha256.Sum256([]byte(task.ID)) + version, err := store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") + require.NoError(t, err) + if test.task == a2a.TaskStateInputRequired || test.task == a2a.TaskStateAuthRequired { + require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) + } + } + request := &apiv1.RevertSessionRequest{SessionId: session.Id} + if test.noSnapshot { + control.actors[name].Status.Snapshots = nil + } + share, err := client.CreateSessionShare(owner, &apiv1.CreateSessionShareRequest{ + SessionId: session.Id, + }) + require.NoError(t, err) + readOnly := metadata.AppendToOutgoingContext(visitor, "x-share-token", share.Token) + _, err = client.RevertSession(readOnly, request) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + _, err = client.RevertSession(visitor, request) + require.Equal(t, codes.NotFound, status.Code(err)) + _, err = client.RevertSession(owner, request) + requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", "healthy runtimes cannot revert") + control.actors[name].Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED + before := proto.CloneOf(control.actors[name]) + control.actors[name].Metadata.Uid = "replacement-uid" + _, err = client.RevertSession(owner, request) + requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", "a same-name replacement is not this runtime") + control.actors[name].Metadata.Uid = before.Metadata.Uid + delete(control.actors, name) + _, err = client.RevertSession(owner, request) + requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", "a missing runtime cannot authorize recovery") + control.actors[name] = proto.CloneOf(before) + calls := 0 + failOnce := true + control.revert = func(actor *ateapipb.Actor) error { + calls++ + if failOnce { + failOnce = false + if test.lostResponse { + actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED + } + return status.Error(codes.Unavailable, "temporary revert outage") + } + return nil + } + _, err = client.RevertSession(owner, request) + require.Equal(t, codes.Unavailable, status.Code(err)) + failed, err := client.GetSession(owner, &apiv1.GetSessionRequest{SessionId: session.Id}) + require.NoError(t, err) + require.Equal(t, database.SessionRuntimeCrashed, failed.Session.Failure.Reason) + ready, err := client.RevertSession(owner, request) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, ready.Session.State) + require.Nil(t, ready.Session.Failure) + // A client whose success response was lost retries and learns that + // the runtime is already healthy, not that it cannot be recovered. + _, err = client.RevertSession(owner, request) + requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", "a recovered runtime has nothing to revert") + if test.lostResponse { + require.Equal(t, 1, calls, "a lost response cannot repeat the revert") + } else { + require.Equal(t, 2, calls) + } + after := control.actors[name] + require.Equal(t, before.Metadata.Uid, after.Metadata.Uid) + require.True(t, proto.Equal(substrate.LatestDurableActorSnapshot(before), substrate.LatestDurableActorSnapshot(after))) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, after.Status.State) + if task != nil { + stored, err := store.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) + require.NoError(t, err) + want := test.task + if want == a2a.TaskStateWorking || want == a2a.TaskStateInputRequired || want == a2a.TaskStateAuthRequired { + want = a2a.TaskStateFailed + } + require.Equal(t, want, stored.Status.State) + } + require.NoError(t, store.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), uuid.NewString())) + }) + } +} + +// requireRevertRefusal pins both the status code and the ErrorInfo reason, +// because the two FAILED_PRECONDITION refusals ask opposite things of a client. +func requireRevertRefusal(t *testing.T, err error, reason, message string) { + t.Helper() + refusal, ok := status.FromError(err) + require.True(t, ok, message) + require.Equal(t, codes.FailedPrecondition, refusal.Code(), message) + var reasons []string + for _, detail := range refusal.Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok && info.GetDomain() == "kagent.dev" { + reasons = append(reasons, info.GetReason()) + } + } + require.Equal(t, []string{reason}, reasons, message) +} diff --git a/go/core/internal/mcp/tasks_test.go b/go/core/internal/mcp/tasks_test.go index 5e3f1549e4..ad562ee138 100644 --- a/go/core/internal/mcp/tasks_test.go +++ b/go/core/internal/mcp/tasks_test.go @@ -568,6 +568,10 @@ func (*fakeSessionWorkflow) Create(_ context.Context, session *apiv1.Session) (* return session, nil } +func (*fakeSessionWorkflow) Revert(_ context.Context, session *apiv1.Session) (*apiv1.Session, error) { + return session, nil +} + func (*fakeSessionWorkflow) Delete(_ context.Context, session *apiv1.Session, _ bool) (*apiv1.Session, error) { return session, nil } diff --git a/go/core/internal/service/session/service.go b/go/core/internal/service/session/service.go index b7549a9304..a2afd9c2c2 100644 --- a/go/core/internal/service/session/service.go +++ b/go/core/internal/service/session/service.go @@ -11,6 +11,9 @@ import ( "time" "github.com/google/uuid" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" utilvalidation "k8s.io/apimachinery/pkg/util/validation" @@ -39,6 +42,7 @@ type store interface { type sessionWorkflow interface { Create(context.Context, *apiv1.Session) (*apiv1.Session, error) + Revert(context.Context, *apiv1.Session) (*apiv1.Session, error) Delete(context.Context, *apiv1.Session, bool) (*apiv1.Session, error) } @@ -280,6 +284,47 @@ func (s *Service) Delete(ctx context.Context, id string, force bool) (*apiv1.Ses return session, nil } +func (s *Service) Revert(ctx context.Context, id string) (*apiv1.Session, error) { + session, err := s.getAuthorized(ctx, id, auth.VerbUpdate) + if err != nil { + return nil, err + } + session, err = s.workflow.Revert(ctx, session) + if reason := revertRefusalReason(err); reason != "" { + // Both refusals are FAILED_PRECONDITION, but one means the runtime is + // already healthy and the other that no retry can restore it. The + // ErrorInfo reason lets a client tell them apart without parsing text. + refusal, detailErr := status.New(codes.FailedPrecondition, err.Error()).WithDetails(&errdetails.ErrorInfo{Domain: "kagent.dev", Reason: reason}) + if detailErr != nil { + return nil, detailErr + } + return nil, refusal.Err() + } + if errors.Is(err, database.ErrFailedPrecondition) { + return nil, serviceerrors.NewFailedPrecondition(err.Error(), err) + } + if errors.Is(err, database.ErrConflict) { + return nil, serviceerrors.NewAborted(err.Error(), err) + } + if err != nil { + return nil, serviceerrors.NewUnavailable("Failed to revert Session", err) + } + return session, nil +} + +// revertRefusalReason names the ErrorInfo reason for a revert that can never +// succeed against the Session as it stands, or returns "" for any other error. +func revertRefusalReason(err error) string { + switch { + case errors.Is(err, errRuntimeNotCrashed): + return "KAGENT_SESSION_RUNTIME_NOT_CRASHED" + case errors.Is(err, errRuntimeUnrecoverable): + return "KAGENT_SESSION_RUNTIME_UNRECOVERABLE" + default: + return "" + } +} + // CreateShare mints a read-only share of the caller's session. A positive ttl bounds how long its // token grants access and must not exceed the configured maximum; zero takes the maximum, // or leaves the share valid until it is revoked or the session deleted when none is set. diff --git a/go/core/internal/service/session/service_test.go b/go/core/internal/service/session/service_test.go index 69e0f97ce0..9a85d17ffb 100644 --- a/go/core/internal/service/session/service_test.go +++ b/go/core/internal/service/session/service_test.go @@ -13,6 +13,9 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" "github.com/kagent-dev/kagent/go/core/internal/database" @@ -129,6 +132,10 @@ func (w serviceTestWorkflow) Create(_ context.Context, session *apiv1.Session) ( return session, w.err } +func (w serviceTestWorkflow) Revert(_ context.Context, session *apiv1.Session) (*apiv1.Session, error) { + return session, w.err +} + func (w serviceTestWorkflow) Delete(_ context.Context, session *apiv1.Session, _ bool) (*apiv1.Session, error) { return session, w.err } @@ -201,23 +208,57 @@ func TestServiceCreateRejectsInvalidOrUnauthorizedRequests(t *testing.T) { } } -func TestServiceDeletePreservesContentionErrors(t *testing.T) { - for _, failure := range []struct { - cause error - code serviceerrors.Code +func TestServiceRevertNamesRefusalReason(t *testing.T) { + for _, test := range []struct { + cause error + reason string }{ - {database.ErrConflict, serviceerrors.CodeAborted}, - {database.ErrFailedPrecondition, serviceerrors.CodeFailedPrecondition}, + {errRuntimeNotCrashed, "KAGENT_SESSION_RUNTIME_NOT_CRASHED"}, + {fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable), "KAGENT_SESSION_RUNTIME_UNRECOVERABLE"}, } { - conflict := fmt.Errorf("Session has an operation in progress: %w", failure.cause) - service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: conflict}) - _, err := service.Delete(serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab", false) - if !serviceerrors.IsCode(err, failure.code) { - t.Fatalf("error = %v, want code %s", err, failure.code) - } - if serviceerrors.MessageOf(err) != conflict.Error() || !errors.Is(err, failure.cause) { - t.Fatalf("error = %v, want preserved contention reason", err) - } + service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: test.cause}) + _, err := service.Revert(serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab") + refusal, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.FailedPrecondition, refusal.Code()) + require.Equal(t, test.cause.Error(), refusal.Message()) + require.Len(t, refusal.Details(), 1) + info, ok := refusal.Details()[0].(*errdetails.ErrorInfo) + require.True(t, ok) + require.Equal(t, "kagent.dev", info.GetDomain()) + require.Equal(t, test.reason, info.GetReason()) + } +} + +func TestServiceLifecycleMethodsPreserveContentionErrors(t *testing.T) { + for _, test := range []struct { + name string + call func(*Service, context.Context, string) (*apiv1.Session, error) + }{ + {name: "revert", call: (*Service).Revert}, + {name: "delete", call: func(service *Service, ctx context.Context, id string) (*apiv1.Session, error) { + return service.Delete(ctx, id, false) + }}, + } { + t.Run(test.name, func(t *testing.T) { + for _, failure := range []struct { + cause error + code serviceerrors.Code + }{ + {database.ErrConflict, serviceerrors.CodeAborted}, + {database.ErrFailedPrecondition, serviceerrors.CodeFailedPrecondition}, + } { + conflict := fmt.Errorf("Session has an operation in progress: %w", failure.cause) + service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: conflict}) + _, err := test.call(service, serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab") + if !serviceerrors.IsCode(err, failure.code) { + t.Fatalf("error = %v, want code %s", err, failure.code) + } + if serviceerrors.MessageOf(err) != conflict.Error() || !errors.Is(err, failure.cause) { + t.Fatalf("error = %v, want preserved contention reason", err) + } + } + }) } } @@ -553,6 +594,7 @@ func TestServiceRejectsReadOnlyShareMutationsWithoutTransport(t *testing.T) { name string call func() error }{ + {name: "revert", call: func() error { _, err := service.Revert(ctx, id); return err }}, {name: "delete", call: func() error { _, err := service.Delete(ctx, id, false); return err }}, {name: "rename", call: func() error { _, err := service.Rename(ctx, id, "title"); return err }}, {name: "share listing", call: func() error { _, err := service.ListShares(ctx, id, 0, ""); return err }}, diff --git a/go/core/test/e2e/session_revert_test.go b/go/core/test/e2e/session_revert_test.go new file mode 100644 index 0000000000..c6a59464c3 --- /dev/null +++ b/go/core/test/e2e/session_revert_test.go @@ -0,0 +1,171 @@ +package e2e_test + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/a2aproject/a2a-go/v2/a2a" + a2apb "github.com/a2aproject/a2a-go/v2/a2apb/v1" + "github.com/kagent-dev/mockllm" + "github.com/stretchr/testify/require" + ateapipb "go.ate.dev/substrate/api/ateapi/v1" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" + "github.com/kagent-dev/kagent/go/core/internal/substrate" +) + +func TestSessionRevertCrashedRuntime(t *testing.T) { + target := interactionTarget(t) + forEachHarness(t, func(t *testing.T, harness testHarness) { + cfg, err := mockllm.LoadConfigFromFile("mocks/invoke_agent.json", interactionMocks) + require.NoError(t, err) + upstream, err := url.Parse(startMockLLMConfig(t, cfg)) + require.NoError(t, err) + proxy := httputil.NewSingleHostReverseProxy(upstream) + // Only an interaction runs a Session's Actor; quiescence suspends it once + // the turn settles. The gate holds the second turn's model call open so + // the Actor is still running when its worker is deleted. + var blockNext atomic.Bool + blocked := make(chan struct{}, 1) + release := make(chan struct{}) + gate := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && blockNext.CompareAndSwap(true, false) { + blocked <- struct{}{} + select { + case <-r.Context().Done(): + case <-release: + } + return + } + proxy.ServeHTTP(w, r) + })) + require.NoError(t, gate.Listener.Close()) + gate.Listener, err = net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + gate.Start() + t.Cleanup(gate.Close) + fixture := newInteractionFixture(t, harness, target, reachableModelURL(t, gate.URL)) + defer func() { + close(release) + gate.CloseClientConnections() + }() + _, _, task := fixture.send(t, "What is 2+2?") + require.Equal(t, a2a.TaskStateCompleted, task.Status.State) + assertActorSuspended(t, fixture) + getTask := &a2apb.GetTaskRequest{Tenant: fixture.tenant, Id: string(task.ID)} + before, err := fixture.client.GetTask(fixture.ctx, getTask) + require.NoError(t, err) + session, err := fixture.sessions.GetSession(fixture.ctx, &apiv1.GetSessionRequest{SessionId: fixture.sessionID}) + require.NoError(t, err) + namespace, name := session.GetSession().GetAgent().GetRef().GetNamespace(), substrate.ActorName(fixture.sessionID) + blockNext.Store(true) + _, turn := newMessageRequest(t, "What is 2+2?") + turn.Tenant, turn.Message.ContextId = fixture.tenant, fixture.sessionID + turnCtx, stopTurn := context.WithCancel(fixture.ctx) + defer stopTurn() + sendDone := make(chan error, 1) + go func() { _, err := sendMessageWithRetry(turnCtx, fixture.client, turn); sendDone <- err }() + select { + case <-blocked: + case <-time.After(time.Minute): + t.Fatal("second turn never reached the blocking model") + } + running, err := findSubstrateActor(fixture.ctx, fixture.system, namespace, name) + require.NoError(t, err) + require.NotNil(t, running) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_RUNNING, running.GetStatus().GetState()) + require.NotEmpty(t, substrate.SnapshotURI(substrate.LatestDurableActorSnapshot(running))) + request := &apiv1.RevertSessionRequest{SessionId: fixture.sessionID} + _, err = fixture.sessions.RevertSession(fixture.ctx, request) + require.Equal(t, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", revertRefusalReason(t, err), "healthy runtimes cannot revert") + assignment := running.GetStatus().GetWorkerAssignment() + require.Equal(t, namespace, assignment.GetWorkerNamespace()) + require.NotEmpty(t, assignment.GetWorkerPod()) + for token := ""; ; { + page, err := fixture.system.ListSubstrateActors(fixture.ctx, &apiv1.ListSubstrateActorsRequest{ + Page: &apiv1.PageRequest{Limit: 100, PageToken: token}, + }) + require.NoError(t, err) + require.Empty(t, page.GetAteApiError()) + for _, actor := range page.GetActors() { + other := actor.GetStatus().GetWorkerAssignment() + if actor.GetMetadata().GetUid() != running.GetMetadata().GetUid() { + require.False(t, other.GetWorkerNamespace() == assignment.GetWorkerNamespace() && other.GetWorkerPod() == assignment.GetWorkerPod(), + "refuse to delete a worker shared with another Actor") + } + } + token = page.GetPage().GetNextPageToken() + if token == "" { + break + } + } + kube := interactionKubeClient(t) + pod := &corev1.Pod{} + require.NoError(t, kube.Get(fixture.ctx, ctrlclient.ObjectKey{ + Namespace: assignment.GetWorkerNamespace(), Name: assignment.GetWorkerPod(), + }, pod)) + require.NotEmpty(t, pod.UID) + require.NoError(t, kube.Delete(fixture.ctx, pod, &ctrlclient.DeleteOptions{ + GracePeriodSeconds: new(int64(0)), Preconditions: &metav1.Preconditions{UID: &pod.UID}, + })) + err = wait.PollUntilContextTimeout(fixture.ctx, 100*time.Millisecond, time.Minute, true, func(ctx context.Context) (bool, error) { + actor, err := findSubstrateActor(ctx, fixture.system, namespace, name) + return actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED, err + }) + require.NoError(t, err) + reverted, err := fixture.sessions.RevertSession(fixture.ctx, request) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, reverted.Session.State) + // A client whose success response was lost retries and must learn that + // the runtime is healthy, not that it cannot be recovered. + _, err = fixture.sessions.RevertSession(fixture.ctx, request) + require.Equal(t, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", revertRefusalReason(t, err), "a recovered runtime has nothing to revert") + recovered, err := findSubstrateActor(fixture.ctx, fixture.system, namespace, name) + require.NoError(t, err) + require.Equal(t, running.GetMetadata().GetUid(), recovered.GetMetadata().GetUid()) + require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, recovered.GetStatus().GetState()) + require.Equal(t, substrate.SnapshotURI(substrate.LatestDurableActorSnapshot(running)), substrate.SnapshotURI(substrate.LatestActorSnapshot(recovered))) + retained, err := fixture.client.GetTask(fixture.ctx, getTask) + require.NoError(t, err) + require.True(t, proto.Equal(before, retained), "recovery preserves the completed task and its history") + stopTurn() + select { + case <-sendDone: + case <-time.After(10 * time.Second): + t.Fatal("interrupted caller did not stop") + } + _, _, next := fixture.send(t, "What is 2+2?") + require.Equal(t, a2a.TaskStateCompleted, next.Status.State) + require.Contains(t, taskText(next), "4") + }) +} + +// revertRefusalReason returns the kagent.dev ErrorInfo reason of a +// FAILED_PRECONDITION revert refusal. +func revertRefusalReason(t *testing.T, err error) string { + t.Helper() + refusal, ok := status.FromError(err) + require.True(t, ok, "revert error %v is not a gRPC status", err) + require.Equal(t, codes.FailedPrecondition, refusal.Code(), "revert error: %v", err) + for _, detail := range refusal.Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok && info.GetDomain() == "kagent.dev" { + return info.GetReason() + } + } + return "" +} diff --git a/proto/kagent/api/v1/sessions.proto b/proto/kagent/api/v1/sessions.proto index e000d269d5..93651ce5c0 100644 --- a/proto/kagent/api/v1/sessions.proto +++ b/proto/kagent/api/v1/sessions.proto @@ -20,6 +20,8 @@ service SessionService { rpc GetSession(GetSessionRequest) returns (GetSessionResponse); rpc ListSessions(ListSessionsRequest) returns (ListSessionsResponse); rpc UpdateSessionName(UpdateSessionNameRequest) returns (UpdateSessionNameResponse); + // Restores a crashed runtime to its last snapshot without replaying its turn. + rpc RevertSession(RevertSessionRequest) returns (RevertSessionResponse); rpc DeleteSession(DeleteSessionRequest) returns (DeleteSessionResponse); rpc CreateSessionShare(CreateSessionShareRequest) returns (CreateSessionShareResponse); rpc ListSessionShares(ListSessionSharesRequest) returns (ListSessionSharesResponse); @@ -107,6 +109,14 @@ message UpdateSessionNameResponse { Session session = 1; } +message RevertSessionRequest { + string session_id = 1 [(buf.validate.field).string.uuid = true]; +} + +message RevertSessionResponse { + Session session = 1; +} + message DeleteSessionRequest { string session_id = 1 [(buf.validate.field).string.uuid = true]; // Terminate ongoing turns instead of rejecting deletion while they are active. diff --git a/ui/src/generated/kagent/api/v1/sessions_pb.ts b/ui/src/generated/kagent/api/v1/sessions_pb.ts index 9c7abc8506..fdc14c14d4 100644 --- a/ui/src/generated/kagent/api/v1/sessions_pb.ts +++ b/ui/src/generated/kagent/api/v1/sessions_pb.ts @@ -17,7 +17,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file kagent/api/v1/sessions.proto. */ export const file_kagent_api_v1_sessions: GenFile = /*@__PURE__*/ - fileDesc("ChxrYWdlbnQvYXBpL3YxL3Nlc3Npb25zLnByb3RvEg1rYWdlbnQuYXBpLnYxIioKB0ZhaWx1cmUSDgoGcmVhc29uGAEgASgJEg8KB21lc3NhZ2UYAiABKAkikgMKB1Nlc3Npb24SCgoCaWQYASABKAkSDwoHY3JlYXRvchgCIAEoCRItCgVhZ2VudBgPIAEoCzIeLmthZ2VudC5hcGkudjEuQWdlbnRQcm92ZW5hbmNlEhkKEXByZXBhcmVkX3JldmlzaW9uGAUgASgJEhUKDWEyYV9hdXRob3JpdHkYBiABKAkSKgoFc3RhdGUYByABKA4yGy5rYWdlbnQuYXBpLnYxLlJ1bnRpbWVTdGF0ZRIyCglvcGVyYXRpb24YCCABKA4yHy5rYWdlbnQuYXBpLnYxLlJ1bnRpbWVPcGVyYXRpb24SJwoHZmFpbHVyZRgJIAEoCzIWLmthZ2VudC5hcGkudjEuRmFpbHVyZRIuCgpjcmVhdGVkX2F0GAogASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIuCgp1cGRhdGVkX2F0GAsgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIMCgRuYW1lGA0gASgJEhIKCmNvbnRleHRfaWQYDiABKAkiTQoPQWdlbnRQcm92ZW5hbmNlEi0KA3JlZhgBIAEoCzIgLmthZ2VudC5hcGkudjEuUmVzb3VyY2VSZWZlcmVuY2USCwoDdWlkGAIgASgJIrwBChRDcmVhdGVTZXNzaW9uUmVxdWVzdBI3CgVhZ2VudBgFIAEoCzIgLmthZ2VudC5hcGkudjEuUmVzb3VyY2VSZWZlcmVuY2VCBrpIA8gBARIeCgpyZXF1ZXN0X2lkGAMgASgJQgq6SAdyBRABGIABEksKBG5hbWUYBCABKAlCPbpIOnI4GMgBMjNeKD86JHxbXlxwe1p9XHB7Q2N9XSg/OlteXHB7Q2N9XSpbXlxwe1p9XHB7Q2N9XSk/KSQiQAoVQ3JlYXRlU2Vzc2lvblJlc3BvbnNlEicKB3Nlc3Npb24YASABKAsyFi5rYWdlbnQuYXBpLnYxLlNlc3Npb24iMQoRR2V0U2Vzc2lvblJlcXVlc3QSHAoKc2Vzc2lvbl9pZBgBIAEoCUIIukgFcgOwAQEiPQoSR2V0U2Vzc2lvblJlc3BvbnNlEicKB3Nlc3Npb24YASABKAsyFi5rYWdlbnQuYXBpLnYxLlNlc3Npb24ihgEKE0xpc3RTZXNzaW9uc1JlcXVlc3QSFAoMYWxsX2NyZWF0b3JzGAIgASgIEigKBHBhZ2UYAyABKAsyGi5rYWdlbnQuYXBpLnYxLlBhZ2VSZXF1ZXN0Ei8KBWFnZW50GAYgASgLMiAua2FnZW50LmFwaS52MS5SZXNvdXJjZVJlZmVyZW5jZSJrChRMaXN0U2Vzc2lvbnNSZXNwb25zZRIoCghzZXNzaW9ucxgBIAMoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbhIpCgRwYWdlGAIgASgLMhsua2FnZW50LmFwaS52MS5QYWdlUmVzcG9uc2UihQEKGFVwZGF0ZVNlc3Npb25OYW1lUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARJLCgRuYW1lGAIgASgJQj26SDpyOBjIATIzXig/OiR8W15ccHtafVxwe0NjfV0oPzpbXlxwe0NjfV0qW15ccHtafVxwe0NjfV0pPykkIkQKGVVwZGF0ZVNlc3Npb25OYW1lUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbiJDChREZWxldGVTZXNzaW9uUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARINCgVmb3JjZRgCIAEoCCJAChVEZWxldGVTZXNzaW9uUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbiKOAQoMU2Vzc2lvblNoYXJlEgoKAmlkGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkSLgoKY3JlYXRlZF9hdBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLgoKZXhwaXJlc19hdBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiawoZQ3JlYXRlU2Vzc2lvblNoYXJlUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARIwCgN0dGwYAiABKAsyGS5nb29nbGUucHJvdG9idWYuRHVyYXRpb25CCLpIBaoBAioAIlcKGkNyZWF0ZVNlc3Npb25TaGFyZVJlc3BvbnNlEioKBXNoYXJlGAEgASgLMhsua2FnZW50LmFwaS52MS5TZXNzaW9uU2hhcmUSDQoFdG9rZW4YAiABKAkiYgoYTGlzdFNlc3Npb25TaGFyZXNSZXF1ZXN0EhwKCnNlc3Npb25faWQYASABKAlCCLpIBXIDsAEBEigKBHBhZ2UYAiABKAsyGi5rYWdlbnQuYXBpLnYxLlBhZ2VSZXF1ZXN0InMKGUxpc3RTZXNzaW9uU2hhcmVzUmVzcG9uc2USKwoGc2hhcmVzGAEgAygLMhsua2FnZW50LmFwaS52MS5TZXNzaW9uU2hhcmUSKQoEcGFnZRgCIAEoCzIbLmthZ2VudC5hcGkudjEuUGFnZVJlc3BvbnNlIjcKGVJldm9rZVNlc3Npb25TaGFyZVJlcXVlc3QSGgoIc2hhcmVfaWQYASABKAlCCLpIBXIDsAEBIhwKGlJldm9rZVNlc3Npb25TaGFyZVJlc3BvbnNlMpoGCg5TZXNzaW9uU2VydmljZRJaCg1DcmVhdGVTZXNzaW9uEiMua2FnZW50LmFwaS52MS5DcmVhdGVTZXNzaW9uUmVxdWVzdBokLmthZ2VudC5hcGkudjEuQ3JlYXRlU2Vzc2lvblJlc3BvbnNlElEKCkdldFNlc3Npb24SIC5rYWdlbnQuYXBpLnYxLkdldFNlc3Npb25SZXF1ZXN0GiEua2FnZW50LmFwaS52MS5HZXRTZXNzaW9uUmVzcG9uc2USVwoMTGlzdFNlc3Npb25zEiIua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvbnNSZXF1ZXN0GiMua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvbnNSZXNwb25zZRJmChFVcGRhdGVTZXNzaW9uTmFtZRInLmthZ2VudC5hcGkudjEuVXBkYXRlU2Vzc2lvbk5hbWVSZXF1ZXN0Gigua2FnZW50LmFwaS52MS5VcGRhdGVTZXNzaW9uTmFtZVJlc3BvbnNlEloKDURlbGV0ZVNlc3Npb24SIy5rYWdlbnQuYXBpLnYxLkRlbGV0ZVNlc3Npb25SZXF1ZXN0GiQua2FnZW50LmFwaS52MS5EZWxldGVTZXNzaW9uUmVzcG9uc2USaQoSQ3JlYXRlU2Vzc2lvblNoYXJlEigua2FnZW50LmFwaS52MS5DcmVhdGVTZXNzaW9uU2hhcmVSZXF1ZXN0Gikua2FnZW50LmFwaS52MS5DcmVhdGVTZXNzaW9uU2hhcmVSZXNwb25zZRJmChFMaXN0U2Vzc2lvblNoYXJlcxInLmthZ2VudC5hcGkudjEuTGlzdFNlc3Npb25TaGFyZXNSZXF1ZXN0Gigua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvblNoYXJlc1Jlc3BvbnNlEmkKElJldm9rZVNlc3Npb25TaGFyZRIoLmthZ2VudC5hcGkudjEuUmV2b2tlU2Vzc2lvblNoYXJlUmVxdWVzdBopLmthZ2VudC5hcGkudjEuUmV2b2tlU2Vzc2lvblNoYXJlUmVzcG9uc2VCPVo7Z2l0aHViLmNvbS9rYWdlbnQtZGV2L2thZ2VudC9nby9hcGkvZ2VuL2thZ2VudC9hcGkvdjE7YXBpdjFiBnByb3RvMw", [file_buf_validate_validate, file_google_protobuf_duration, file_google_protobuf_timestamp, file_kagent_api_v1_common, file_kagent_api_v1_runtime]); + fileDesc("ChxrYWdlbnQvYXBpL3YxL3Nlc3Npb25zLnByb3RvEg1rYWdlbnQuYXBpLnYxIioKB0ZhaWx1cmUSDgoGcmVhc29uGAEgASgJEg8KB21lc3NhZ2UYAiABKAkikgMKB1Nlc3Npb24SCgoCaWQYASABKAkSDwoHY3JlYXRvchgCIAEoCRItCgVhZ2VudBgPIAEoCzIeLmthZ2VudC5hcGkudjEuQWdlbnRQcm92ZW5hbmNlEhkKEXByZXBhcmVkX3JldmlzaW9uGAUgASgJEhUKDWEyYV9hdXRob3JpdHkYBiABKAkSKgoFc3RhdGUYByABKA4yGy5rYWdlbnQuYXBpLnYxLlJ1bnRpbWVTdGF0ZRIyCglvcGVyYXRpb24YCCABKA4yHy5rYWdlbnQuYXBpLnYxLlJ1bnRpbWVPcGVyYXRpb24SJwoHZmFpbHVyZRgJIAEoCzIWLmthZ2VudC5hcGkudjEuRmFpbHVyZRIuCgpjcmVhdGVkX2F0GAogASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIuCgp1cGRhdGVkX2F0GAsgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIMCgRuYW1lGA0gASgJEhIKCmNvbnRleHRfaWQYDiABKAkiTQoPQWdlbnRQcm92ZW5hbmNlEi0KA3JlZhgBIAEoCzIgLmthZ2VudC5hcGkudjEuUmVzb3VyY2VSZWZlcmVuY2USCwoDdWlkGAIgASgJIrwBChRDcmVhdGVTZXNzaW9uUmVxdWVzdBI3CgVhZ2VudBgFIAEoCzIgLmthZ2VudC5hcGkudjEuUmVzb3VyY2VSZWZlcmVuY2VCBrpIA8gBARIeCgpyZXF1ZXN0X2lkGAMgASgJQgq6SAdyBRABGIABEksKBG5hbWUYBCABKAlCPbpIOnI4GMgBMjNeKD86JHxbXlxwe1p9XHB7Q2N9XSg/OlteXHB7Q2N9XSpbXlxwe1p9XHB7Q2N9XSk/KSQiQAoVQ3JlYXRlU2Vzc2lvblJlc3BvbnNlEicKB3Nlc3Npb24YASABKAsyFi5rYWdlbnQuYXBpLnYxLlNlc3Npb24iMQoRR2V0U2Vzc2lvblJlcXVlc3QSHAoKc2Vzc2lvbl9pZBgBIAEoCUIIukgFcgOwAQEiPQoSR2V0U2Vzc2lvblJlc3BvbnNlEicKB3Nlc3Npb24YASABKAsyFi5rYWdlbnQuYXBpLnYxLlNlc3Npb24ihgEKE0xpc3RTZXNzaW9uc1JlcXVlc3QSFAoMYWxsX2NyZWF0b3JzGAIgASgIEigKBHBhZ2UYAyABKAsyGi5rYWdlbnQuYXBpLnYxLlBhZ2VSZXF1ZXN0Ei8KBWFnZW50GAYgASgLMiAua2FnZW50LmFwaS52MS5SZXNvdXJjZVJlZmVyZW5jZSJrChRMaXN0U2Vzc2lvbnNSZXNwb25zZRIoCghzZXNzaW9ucxgBIAMoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbhIpCgRwYWdlGAIgASgLMhsua2FnZW50LmFwaS52MS5QYWdlUmVzcG9uc2UihQEKGFVwZGF0ZVNlc3Npb25OYW1lUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARJLCgRuYW1lGAIgASgJQj26SDpyOBjIATIzXig/OiR8W15ccHtafVxwe0NjfV0oPzpbXlxwe0NjfV0qW15ccHtafVxwe0NjfV0pPykkIkQKGVVwZGF0ZVNlc3Npb25OYW1lUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbiI0ChRSZXZlcnRTZXNzaW9uUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABASJAChVSZXZlcnRTZXNzaW9uUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbiJDChREZWxldGVTZXNzaW9uUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARINCgVmb3JjZRgCIAEoCCJAChVEZWxldGVTZXNzaW9uUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmthZ2VudC5hcGkudjEuU2Vzc2lvbiKOAQoMU2Vzc2lvblNoYXJlEgoKAmlkGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkSLgoKY3JlYXRlZF9hdBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLgoKZXhwaXJlc19hdBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiawoZQ3JlYXRlU2Vzc2lvblNoYXJlUmVxdWVzdBIcCgpzZXNzaW9uX2lkGAEgASgJQgi6SAVyA7ABARIwCgN0dGwYAiABKAsyGS5nb29nbGUucHJvdG9idWYuRHVyYXRpb25CCLpIBaoBAioAIlcKGkNyZWF0ZVNlc3Npb25TaGFyZVJlc3BvbnNlEioKBXNoYXJlGAEgASgLMhsua2FnZW50LmFwaS52MS5TZXNzaW9uU2hhcmUSDQoFdG9rZW4YAiABKAkiYgoYTGlzdFNlc3Npb25TaGFyZXNSZXF1ZXN0EhwKCnNlc3Npb25faWQYASABKAlCCLpIBXIDsAEBEigKBHBhZ2UYAiABKAsyGi5rYWdlbnQuYXBpLnYxLlBhZ2VSZXF1ZXN0InMKGUxpc3RTZXNzaW9uU2hhcmVzUmVzcG9uc2USKwoGc2hhcmVzGAEgAygLMhsua2FnZW50LmFwaS52MS5TZXNzaW9uU2hhcmUSKQoEcGFnZRgCIAEoCzIbLmthZ2VudC5hcGkudjEuUGFnZVJlc3BvbnNlIjcKGVJldm9rZVNlc3Npb25TaGFyZVJlcXVlc3QSGgoIc2hhcmVfaWQYASABKAlCCLpIBXIDsAEBIhwKGlJldm9rZVNlc3Npb25TaGFyZVJlc3BvbnNlMvYGCg5TZXNzaW9uU2VydmljZRJaCg1DcmVhdGVTZXNzaW9uEiMua2FnZW50LmFwaS52MS5DcmVhdGVTZXNzaW9uUmVxdWVzdBokLmthZ2VudC5hcGkudjEuQ3JlYXRlU2Vzc2lvblJlc3BvbnNlElEKCkdldFNlc3Npb24SIC5rYWdlbnQuYXBpLnYxLkdldFNlc3Npb25SZXF1ZXN0GiEua2FnZW50LmFwaS52MS5HZXRTZXNzaW9uUmVzcG9uc2USVwoMTGlzdFNlc3Npb25zEiIua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvbnNSZXF1ZXN0GiMua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvbnNSZXNwb25zZRJmChFVcGRhdGVTZXNzaW9uTmFtZRInLmthZ2VudC5hcGkudjEuVXBkYXRlU2Vzc2lvbk5hbWVSZXF1ZXN0Gigua2FnZW50LmFwaS52MS5VcGRhdGVTZXNzaW9uTmFtZVJlc3BvbnNlEloKDVJldmVydFNlc3Npb24SIy5rYWdlbnQuYXBpLnYxLlJldmVydFNlc3Npb25SZXF1ZXN0GiQua2FnZW50LmFwaS52MS5SZXZlcnRTZXNzaW9uUmVzcG9uc2USWgoNRGVsZXRlU2Vzc2lvbhIjLmthZ2VudC5hcGkudjEuRGVsZXRlU2Vzc2lvblJlcXVlc3QaJC5rYWdlbnQuYXBpLnYxLkRlbGV0ZVNlc3Npb25SZXNwb25zZRJpChJDcmVhdGVTZXNzaW9uU2hhcmUSKC5rYWdlbnQuYXBpLnYxLkNyZWF0ZVNlc3Npb25TaGFyZVJlcXVlc3QaKS5rYWdlbnQuYXBpLnYxLkNyZWF0ZVNlc3Npb25TaGFyZVJlc3BvbnNlEmYKEUxpc3RTZXNzaW9uU2hhcmVzEicua2FnZW50LmFwaS52MS5MaXN0U2Vzc2lvblNoYXJlc1JlcXVlc3QaKC5rYWdlbnQuYXBpLnYxLkxpc3RTZXNzaW9uU2hhcmVzUmVzcG9uc2USaQoSUmV2b2tlU2Vzc2lvblNoYXJlEigua2FnZW50LmFwaS52MS5SZXZva2VTZXNzaW9uU2hhcmVSZXF1ZXN0Gikua2FnZW50LmFwaS52MS5SZXZva2VTZXNzaW9uU2hhcmVSZXNwb25zZUI9WjtnaXRodWIuY29tL2thZ2VudC1kZXYva2FnZW50L2dvL2FwaS9nZW4va2FnZW50L2FwaS92MTthcGl2MWIGcHJvdG8z", [file_buf_validate_validate, file_google_protobuf_duration, file_google_protobuf_timestamp, file_kagent_api_v1_common, file_kagent_api_v1_runtime]); /** * @generated from message kagent.api.v1.Failure @@ -317,6 +317,40 @@ export type UpdateSessionNameResponse = Message<"kagent.api.v1.UpdateSessionName export const UpdateSessionNameResponseSchema: GenMessage = /*@__PURE__*/ messageDesc(file_kagent_api_v1_sessions, 10); +/** + * @generated from message kagent.api.v1.RevertSessionRequest + */ +export type RevertSessionRequest = Message<"kagent.api.v1.RevertSessionRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; +}; + +/** + * Describes the message kagent.api.v1.RevertSessionRequest. + * Use `create(RevertSessionRequestSchema)` to create a new message. + */ +export const RevertSessionRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_kagent_api_v1_sessions, 11); + +/** + * @generated from message kagent.api.v1.RevertSessionResponse + */ +export type RevertSessionResponse = Message<"kagent.api.v1.RevertSessionResponse"> & { + /** + * @generated from field: kagent.api.v1.Session session = 1; + */ + session?: Session | undefined; +}; + +/** + * Describes the message kagent.api.v1.RevertSessionResponse. + * Use `create(RevertSessionResponseSchema)` to create a new message. + */ +export const RevertSessionResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_kagent_api_v1_sessions, 12); + /** * @generated from message kagent.api.v1.DeleteSessionRequest */ @@ -340,7 +374,7 @@ export type DeleteSessionRequest = Message<"kagent.api.v1.DeleteSessionRequest"> * Use `create(DeleteSessionRequestSchema)` to create a new message. */ export const DeleteSessionRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 11); + messageDesc(file_kagent_api_v1_sessions, 13); /** * @generated from message kagent.api.v1.DeleteSessionResponse @@ -357,7 +391,7 @@ export type DeleteSessionResponse = Message<"kagent.api.v1.DeleteSessionResponse * Use `create(DeleteSessionResponseSchema)` to create a new message. */ export const DeleteSessionResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 12); + messageDesc(file_kagent_api_v1_sessions, 14); /** * Grants read-only access to a session, including A2A get, list, and subscribe. @@ -393,7 +427,7 @@ export type SessionShare = Message<"kagent.api.v1.SessionShare"> & { * Use `create(SessionShareSchema)` to create a new message. */ export const SessionShareSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 13); + messageDesc(file_kagent_api_v1_sessions, 15); /** * @generated from message kagent.api.v1.CreateSessionShareRequest @@ -419,7 +453,7 @@ export type CreateSessionShareRequest = Message<"kagent.api.v1.CreateSessionShar * Use `create(CreateSessionShareRequestSchema)` to create a new message. */ export const CreateSessionShareRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 14); + messageDesc(file_kagent_api_v1_sessions, 16); /** * @generated from message kagent.api.v1.CreateSessionShareResponse @@ -443,7 +477,7 @@ export type CreateSessionShareResponse = Message<"kagent.api.v1.CreateSessionSha * Use `create(CreateSessionShareResponseSchema)` to create a new message. */ export const CreateSessionShareResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 15); + messageDesc(file_kagent_api_v1_sessions, 17); /** * @generated from message kagent.api.v1.ListSessionSharesRequest @@ -465,7 +499,7 @@ export type ListSessionSharesRequest = Message<"kagent.api.v1.ListSessionSharesR * Use `create(ListSessionSharesRequestSchema)` to create a new message. */ export const ListSessionSharesRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 16); + messageDesc(file_kagent_api_v1_sessions, 18); /** * @generated from message kagent.api.v1.ListSessionSharesResponse @@ -487,7 +521,7 @@ export type ListSessionSharesResponse = Message<"kagent.api.v1.ListSessionShares * Use `create(ListSessionSharesResponseSchema)` to create a new message. */ export const ListSessionSharesResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 17); + messageDesc(file_kagent_api_v1_sessions, 19); /** * @generated from message kagent.api.v1.RevokeSessionShareRequest @@ -504,7 +538,7 @@ export type RevokeSessionShareRequest = Message<"kagent.api.v1.RevokeSessionShar * Use `create(RevokeSessionShareRequestSchema)` to create a new message. */ export const RevokeSessionShareRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 18); + messageDesc(file_kagent_api_v1_sessions, 20); /** * @generated from message kagent.api.v1.RevokeSessionShareResponse @@ -517,7 +551,7 @@ export type RevokeSessionShareResponse = Message<"kagent.api.v1.RevokeSessionSha * Use `create(RevokeSessionShareResponseSchema)` to create a new message. */ export const RevokeSessionShareResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_kagent_api_v1_sessions, 19); + messageDesc(file_kagent_api_v1_sessions, 21); /** * Lifecycle mutations execute inline. After a retryable error, repeat the same @@ -561,6 +595,16 @@ export const SessionService: GenService<{ input: typeof UpdateSessionNameRequestSchema; output: typeof UpdateSessionNameResponseSchema; }, + /** + * Restores a crashed runtime to its last snapshot without replaying its turn. + * + * @generated from rpc kagent.api.v1.SessionService.RevertSession + */ + revertSession: { + methodKind: "unary"; + input: typeof RevertSessionRequestSchema; + output: typeof RevertSessionResponseSchema; + }, /** * @generated from rpc kagent.api.v1.SessionService.DeleteSession */ From acb3ea0dcd2dbac652541d602b35364a02556281 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Wed, 4 Nov 2026 14:09:50 -0800 Subject: [PATCH 04/15] refactor(session): fence crash recovery with one read A manual revert ran 27 SQL statements before `FinishSessionOperation`. It now runs 19, with the same three Substrate calls and the same two transactions. - `SessionRecoveryFence` reads the lifecycle generation, the recorded Actor UID, and the Actor template in one statement, before the Actor is observed. It replaces `SessionLifecycleGeneration`, the observation's `GetRuntimeRevision`, and `verifyActor`'s post-observation `GetSessionForRuntime`. `observeCrash` checks the observed Actor against an `ActorBinding` built from the fence and compares UIDs. The recorded UID changes only through a lifecycle operation, which also replaces the generation, so admission's locked generation check still rejects a stale observation. - Admission reads the Actor-UID guard and the checkpoint-CREATING guard in one statement under the Session lock. It still evaluates them in their original order. - One CTE clears the history's pending quiescence marker and selects each interrupted task with its latest version and pending-boundary flag. That replaces a separate clear, a task-ID read, and two reads per task. A task is re-read only after a pending boundary was just settled, and a task with no events is still rejected. Signed-off-by: Ada Lundhe Co-Authored-By: Claude Opus 5.5 --- docs/architecture/runtime-and-lifecycle.md | 12 +- go/core/internal/database/session_recovery.go | 130 ++++++++++++------ .../database/session_recovery_test.go | 29 +++- go/core/internal/service/session/recovery.go | 61 ++++---- .../internal/service/session/recovery_test.go | 4 +- go/core/internal/service/session/workflow.go | 2 +- 6 files changed, 161 insertions(+), 77 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 0a54e9d2f2..8bbaa16d5c 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -241,10 +241,14 @@ crash, or fails nonterminal work absent from the retained snapshot. Waiting task captured by that snapshot remain available. Checkpoint creation excludes competing recovery. -The caller reads the Session's lifecycle generation before it observes Substrate. -Admission rejects the crash when another lifecycle operation began after that -read. A concurrent revert keeps the Actor UID and returns the Session to READY, -so only the generation exposes a stale observation. +The caller reads a recovery fence before it observes Substrate: the Session's +lifecycle generation, its recorded Actor UID, and the Actor template, in one +statement. The observed Actor must match that template and UID. Admission +rejects the crash when another lifecycle operation began after that read, and +re-checks the Actor UID under the Session lock. A concurrent revert keeps the +Actor UID and returns the Session to READY, so only the generation exposes a +stale observation. The recorded UID changes only through a lifecycle operation, +which also replaces the generation. An issued idle suspension does not exclude recovery. It cannot finish against the crashed Actor, and its worker keeps the claim when suspension fails. Admission diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index dbf78f3e52..549601d792 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -43,23 +43,50 @@ func (c *Client) ListSessionRecoveryCandidates(ctx context.Context, afterID stri return result, nil } -// SessionLifecycleGeneration returns the ID of the Session's latest lifecycle -// operation, or uuid.Nil when none is recorded. Every operation replaces it, and -// completion keeps it. Recovery reads it before observing Substrate. -func (c *Client) SessionLifecycleGeneration(ctx context.Context, id string) (uuid.UUID, error) { - generation, err := queryOne(ctx, c.db, `SELECT operation_id FROM session_record WHERE id = $1`, pgx.RowTo[*uuid.UUID], id) +// SessionRecoveryFence is what crash recovery reads before it observes Substrate. +// Generation is the ID of the Session's latest lifecycle operation, or uuid.Nil +// when none is recorded. Every operation replaces it, and completion keeps it. +// ActorUID is the recorded runtime Actor, or "" when none is recorded. The +// template identifies where that Actor lives and what it must run. +type SessionRecoveryFence struct { + Generation uuid.UUID + ActorUID string + TemplateAtespace string + TemplateName string +} + +// SessionRecoveryFence reads the lifecycle generation, recorded Actor UID, and +// Actor template of a live Session in one statement. Deleted or missing Sessions +// return ErrNotFound. Callers authorize access. The read takes no lock: admission +// re-checks the generation and Actor UID under the Session lock. +func (c *Client) SessionRecoveryFence(ctx context.Context, id string) (*SessionRecoveryFence, error) { + type fenceRow struct { + OperationID *uuid.UUID + ActorUID *string + ActorTemplateAtespace string + ActorTemplateName string + } + row, err := queryOne(ctx, c.db, ` + SELECT i.operation_id, i.actor_uid, r.actor_template_atespace, r.actor_template_name + FROM session_record i JOIN runtime_revision r ON r.revision = i.prepared_revision + WHERE i.id = $1 AND i.state <> 'RUNTIME_STATE_DELETED' + `, pgx.RowToStructByName[fenceRow], id) if err != nil { - return uuid.Nil, notFoundOr(err) + return nil, fmt.Errorf("read Session %s recovery fence: %w", id, notFoundOr(err)) } - if generation == nil { - return uuid.Nil, nil + fence := &SessionRecoveryFence{TemplateAtespace: row.ActorTemplateAtespace, TemplateName: row.ActorTemplateName} + if row.OperationID != nil { + fence.Generation = *row.OperationID } - return *generation, nil + if row.ActorUID != nil { + fence.ActorUID = *row.ActorUID + } + return fence, nil } // BeginCrashedSessionRecovery fences confirmed-crash recovery under the Session lock. // It preserves durable results; obsolete Actors and conflicting operations fail. -// generation is the caller's SessionLifecycleGeneration read from before it +// generation is the caller's SessionRecoveryFence generation, read before it // observed the crash. A lifecycle operation begun since then makes the observation // stale, because a concurrent revert keeps the Actor UID and returns to READY. func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, generation uuid.UUID, actorUID string, snapshot SessionTaskSnapshot) (*SessionOperation, error) { @@ -69,11 +96,21 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, gen if err != nil { return notFoundOr(err) } - matches, err := queryOne(ctx, tx, `SELECT actor_uid = $2 FROM session WHERE id = $1`, pgx.RowTo[bool], row.ID, actorUID) + // Both guards are read under the lock in one statement, and are evaluated in + // their original order: Actor identity first, checkpoint creation last. + type guards struct { + Matches bool + Busy bool + } + guard, err := queryOne(ctx, tx, ` + SELECT actor_uid = $2 AS matches, + EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') AS busy + FROM session WHERE id = $1 + `, pgx.RowToStructByName[guards], row.ID, actorUID) if err != nil { return err } - if !matches || actorUID == "" { + if !guard.Matches || actorUID == "" { return ErrConflict } session, err := toSession(row) @@ -100,13 +137,7 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, gen // retires the claim in this transaction; a late finish for it changes nothing. // The generation check keeps the crash observation current: without another // lifecycle operation, nothing returns the crashed Actor to service. - busy, err := queryOne(ctx, tx, ` - SELECT EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') - `, pgx.RowTo[bool], row.ID) - if err != nil { - return err - } - if busy { + if guard.Busy { return ErrConflict } row.State = apiv1.RuntimeState_RUNTIME_STATE_FAILED.String() @@ -136,45 +167,54 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, gen // settleCrashedSessionTask retains captured waits and fails discarded nonterminal work. // Callers hold the Session lock; recovery cannot checkpoint discarded state. func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { - if err := execSQL(ctx, tx, `UPDATE session_task_event SET quiescence_pending = FALSE WHERE history_id = $1 AND quiescence_pending`, session.HistoryID); err != nil { - return err + type interruptedTask struct { + ID string + Data []byte + Version *int64 + Pending bool } - taskIDs, err := queryMany(ctx, tx, ` - SELECT t.id FROM session_task t WHERE t.history_id = $1 + // The CTE clears the history's pending quiescence marker. The main query + // reads the pre-update snapshot, but pending only tests that a boundary was + // recorded, which clearing keeps true. A task without events is malformed. + tasks, err := queryMany(ctx, tx, ` + WITH cleared AS ( + UPDATE session_task_event SET quiescence_pending = FALSE + WHERE history_id = $1 AND quiescence_pending) + SELECT t.id, t.data, latest.sequence AS version, + COALESCE(NOT latest.published AND latest.quiescence_pending IS NOT NULL, FALSE) AS pending + FROM session_task t + LEFT JOIN LATERAL (SELECT sequence, published, quiescence_pending FROM session_task_event e + WHERE e.history_id = t.history_id AND e.task_id = t.id + ORDER BY sequence DESC LIMIT 1) latest ON TRUE + WHERE t.history_id = $1 AND (t.state IN ('TASK_STATE_SUBMITTED', 'TASK_STATE_WORKING') OR (t.state IN ('TASK_STATE_INPUT_REQUIRED', 'TASK_STATE_AUTH_REQUIRED') AND NOT EXISTS (SELECT 1 FROM session_task_event captured WHERE captured.history_id = t.history_id AND $2::text <> '' AND captured.snapshot_uri = $2 AND captured.snapshot_fidelity = $3 AND captured.snapshot_atespace = $4 - AND captured.sequence >= (SELECT MAX(sequence) FROM session_task_event latest - WHERE latest.history_id = t.history_id AND latest.task_id = t.id)))) - `, pgx.RowTo[string], session.HistoryID, snapshot.URI, snapshot.Fidelity, snapshot.Atespace) + AND captured.sequence >= latest.sequence))) + `, pgx.RowToStructByName[interruptedTask], session.HistoryID, snapshot.URI, snapshot.Fidelity, snapshot.Atespace) if err != nil { return fmt.Errorf("read interrupted Session tasks: %w", err) } - for _, taskID := range taskIDs { - version, err := taskVersion(ctx, tx, session.HistoryID, taskID) - if err != nil { - return err + for _, interrupted := range tasks { + taskID, data := interrupted.ID, interrupted.Data + if interrupted.Version == nil { + return fmt.Errorf("interrupted Session task %s has no events", taskID) } - pending, err := queryOne(ctx, tx, ` - SELECT EXISTS (SELECT 1 FROM session_task_event - WHERE sequence = $1 AND NOT published AND quiescence_pending IS NOT NULL) - `, pgx.RowTo[bool], version) - if err != nil { - return err - } - if pending { - if err := settleSessionTask(ctx, tx, session, taskID, version); err != nil { + if interrupted.Pending { + if err := settleSessionTask(ctx, tx, session, taskID, *interrupted.Version); err != nil { return err } + // Settlement rewrote the projection, so the selected data is stale. + stored, err := readSessionTask(ctx, tx, session.HistoryID, taskID) + if err != nil { + return err + } + data = stored.Data } - stored, err := readSessionTask(ctx, tx, session.HistoryID, taskID) - if err != nil { - return err - } - task, err := unmarshalSessionTask(stored.Data) + task, err := unmarshalSessionTask(data) if err != nil { return err } @@ -186,7 +226,7 @@ func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow if err := storeSessionTaskEvent(ctx, tx, session, task, event, false); err != nil { return err } - version, err = taskVersion(ctx, tx, session.HistoryID, taskID) + version, err := taskVersion(ctx, tx, session.HistoryID, taskID) if err != nil { return err } diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index 7cdb9ceade..131e7f9809 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -197,9 +197,9 @@ func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { func lifecycleGeneration(t *testing.T, client *Client, sessionID string) uuid.UUID { t.Helper() - generation, err := client.SessionLifecycleGeneration(t.Context(), sessionID) + fence, err := client.SessionRecoveryFence(t.Context(), sessionID) require.NoError(t, err) - return generation + return fence.Generation } // completeCrashedRecovery admits and finishes a revert as a successful concurrent @@ -276,3 +276,28 @@ func TestCrashedRecoveryNotePinsItsTimelinePosition(t *testing.T) { require.True(t, ok, "the crash note has a timeline position") require.True(t, position.Equal(*failed.Status.Timestamp), "position %s, status timestamp %s", position, failed.Status.Timestamp) } + +// The fence reports the recorded Actor and the generation a crash observation +// is valid for. Admitting a recovery replaces the generation and keeps the UID. +func TestSessionRecoveryFenceReadsRecordedActor(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, _ := waitingTaskFixture(t, client) + + fence, err := client.SessionRecoveryFence(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, "actor-"+session.Id, fence.ActorUID) + require.Equal(t, "team-a", fence.TemplateAtespace) + require.Equal(t, "revision-actor-template", fence.TemplateName) + require.NotEqual(t, uuid.Nil, fence.Generation, "creation recorded a lifecycle operation") + + operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, fence.Generation, fence.ActorUID, SessionTaskSnapshot{}) + require.NoError(t, err) + admitted, err := client.SessionRecoveryFence(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, operation.ID, admitted.Generation) + require.Equal(t, fence.ActorUID, admitted.ActorUID) + + _, err = client.SessionRecoveryFence(t.Context(), uuid.NewString()) + require.ErrorIs(t, err, ErrNotFound) +} diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go index de296fc141..80d0cb0dd1 100644 --- a/go/core/internal/service/session/recovery.go +++ b/go/core/internal/service/session/recovery.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/google/uuid" ateapipb "go.ate.dev/substrate/api/ateapi/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -40,32 +41,52 @@ func lifecycleIdle(session *apiv1.Session) bool { return session.State == apiv1.RuntimeState_RUNTIME_STATE_READY && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE } +// crashObservation is the Substrate state of a Session's recorded Actor, taken +// after the lifecycle generation it is valid for. +type crashObservation struct { + generation uuid.UUID + actor *ateapipb.Actor +} + +func (o crashObservation) crashed() bool { + return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED +} + // observeCrash reads the Session's recorded Actor and reports whether Substrate // confirms it crashed. Revert admits recovery only through this check. An Actor // that is missing, replaced, or from another template is an error, never a // crash: reverting it would restore the wrong runtime. Those errors wrap // errRuntimeUnrecoverable; transient read failures do not. -func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session) (*ateapipb.Actor, bool, error) { - revision, err := w.store.GetRuntimeRevision(ctx, session.PreparedRevision) +// +// The fence is read before the Actor. A concurrent revert keeps the Actor UID +// and returns to READY, so only the generation shows that a crash observation +// predates it. The recorded UID changes only through a lifecycle operation, +// which also replaces the generation, so admission's locked generation check +// rejects an observation compared against a stale UID. +func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session) (crashObservation, error) { + fence, err := w.store.SessionRecoveryFence(ctx, session.Id) + if errors.Is(err, database.ErrNotFound) { + return crashObservation{}, fmt.Errorf("runtime Session was deleted: %w", errRuntimeUnrecoverable) + } if err != nil { - return nil, false, err + return crashObservation{}, err } - actor, err := w.actors.GetActor(ctx, revision.ActorTemplateAtespace, substrate.ActorName(session.Id)) + binding := substrate.ActorBinding{Atespace: fence.TemplateAtespace, Name: substrate.ActorName(session.Id), + TemplateAtespace: fence.TemplateAtespace, TemplateName: fence.TemplateName} + actor, err := w.actors.GetActor(ctx, binding.Atespace, binding.Name) if status.Code(err) == codes.NotFound { - return nil, false, fmt.Errorf("runtime Actor is missing: %w", errRuntimeUnrecoverable) + return crashObservation{}, fmt.Errorf("runtime Actor is missing: %w", errRuntimeUnrecoverable) } if err != nil { - return nil, false, err + return crashObservation{}, err } - if !validActorIdentity(actor, revision, substrate.ActorName(session.Id)) { - return nil, false, fmt.Errorf("runtime Actor identity or template changed: %w", errRuntimeUnrecoverable) + if !binding.Matches(actor) { + return crashObservation{}, fmt.Errorf("runtime Actor identity or template changed: %w", errRuntimeUnrecoverable) } - if err := w.verifyActor(ctx, session, revision, actor); errors.Is(err, database.ErrNotFound) { - return nil, false, fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) - } else if err != nil { - return nil, false, err + if actor.GetMetadata().GetUid() != fence.ActorUID { + return crashObservation{}, fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) } - return actor, actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED, nil + return crashObservation{generation: fence.Generation, actor: actor}, nil } // scanRecovery runs one recovery interval and returns the cursor for the next. @@ -136,22 +157,16 @@ func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*ap if !lifecycleIdle(session) { return nil, fmt.Errorf("runtime state is %s: %w", session.State, errRuntimeNotCrashed) } - // Read before observing Substrate. A concurrent revert keeps the Actor UID and - // returns to READY, so only the generation shows that this crash observation - // predates it. - generation, err := w.store.SessionLifecycleGeneration(ctx, session.Id) - if err != nil { - return nil, err - } - actor, crashed, err := w.observeCrash(ctx, session) + observed, err := w.observeCrash(ctx, session) if err != nil { return nil, err } - if !crashed { + if !observed.crashed() { return nil, errRuntimeNotCrashed } + actor := observed.actor snapshot := substrate.LatestDurableActorSnapshot(actor) - operation, err := w.store.BeginCrashedSessionRecovery(ctx, session.Id, generation, actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{ + operation, err := w.store.BeginCrashedSessionRecovery(ctx, session.Id, observed.generation, actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{ Atespace: actor.GetMetadata().GetAtespace(), URI: substrate.SnapshotURI(snapshot), Fidelity: snapshot.GetFidelity().String(), }) diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 16515f7cee..3d8ae51f58 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -66,9 +66,9 @@ func (a *recoveryTestActors) GetActor(ctx context.Context, atespace, name string func revertGeneration(t *testing.T, store *lifecycleTestStore, sessionID string) uuid.UUID { t.Helper() - generation, err := store.SessionLifecycleGeneration(t.Context(), sessionID) + fence, err := store.SessionRecoveryFence(t.Context(), sessionID) require.NoError(t, err) - return generation + return fence.Generation } func crashedTurnFixture(t *testing.T) (*lifecycleTestStore, *lifecycleTestActors, *apiv1.Session, *a2a.Task, int64) { diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 20a758b435..e77dd91389 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -16,7 +16,7 @@ import ( type workflowStore interface { ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) - SessionLifecycleGeneration(context.Context, string) (uuid.UUID, error) + SessionRecoveryFence(context.Context, string) (*database.SessionRecoveryFence, error) BeginCrashedSessionRecovery(context.Context, string, uuid.UUID, string, database.SessionTaskSnapshot) (*database.SessionOperation, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) From f3eb93a7d985aff7515008229ebc332de8ce7cbe Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 08:08:31 -0800 Subject: [PATCH 05/15] refactor(session): name the recovery read a generation and share the admitted check Review follow-ups on #3395: - SessionRecoveryFence becomes SessionGeneration, read by GetSessionGeneration, matching the lifecycle code's name for operation_id. BeginCrashedSessionRecovery takes the whole read back, so the store compares both the generation and the recorded Actor UID, and the caller only holds onto it. - database.CrashRecoveryAdmitted replaces three copies of the admitted-recovery check, and every caller now requires the recovery's RESUME to be in flight. FinishSessionOperation replaces the failure on every finish, so the copies only differed while a Delete supersedes the recovery, where both return a conflict. - Revert adds its timeout only on the admission path. run already bounds a joined attempt. No query changes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 8 +-- go/core/internal/database/session_recovery.go | 67 +++++++++++-------- .../database/session_recovery_test.go | 58 +++++++++------- go/core/internal/service/session/recovery.go | 30 ++++----- .../internal/service/session/recovery_test.go | 14 ++-- go/core/internal/service/session/workflow.go | 17 ++--- 6 files changed, 103 insertions(+), 91 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 8bbaa16d5c..1a28d3522b 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -241,10 +241,10 @@ crash, or fails nonterminal work absent from the retained snapshot. Waiting task captured by that snapshot remain available. Checkpoint creation excludes competing recovery. -The caller reads a recovery fence before it observes Substrate: the Session's -lifecycle generation, its recorded Actor UID, and the Actor template, in one -statement. The observed Actor must match that template and UID. Admission -rejects the crash when another lifecycle operation began after that read, and +The caller reads the Session's generation before it observes Substrate: its +lifecycle generation, recorded Actor UID, and Actor template, in one statement. +The observed Actor must match that template and UID. Admission takes the read +back, rejects the crash when another lifecycle operation began after it, and re-checks the Actor UID under the Session lock. A concurrent revert keeps the Actor UID and returns the Session to READY, so only the generation exposes a stale observation. The recorded UID changes only through a lifecycle operation, diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 549601d792..cac3b1b8e4 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -18,6 +18,16 @@ import ( // that the recorded Actor crashed. Transport failures cannot admit this work. const SessionRuntimeCrashed = "RuntimeCrashed" +// CrashRecoveryAdmitted reports a Session whose crash recovery was admitted and +// has not finished: FAILED for a confirmed crash, with the recovery's RESUME +// still in flight. A retry joins that operation instead of observing Substrate +// again, and execution reverts the recorded Actor in place instead of resuming it. +func CrashRecoveryAdmitted(session *apiv1.Session) bool { + return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && + session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && + session.Failure.GetReason() == SessionRuntimeCrashed +} + // ListSessionRecoveryCandidates finds recoveries admitted by an explicit request. // Retrying durable work cannot admit recovery for a READY Session. func (c *Client) ListSessionRecoveryCandidates(ctx context.Context, afterID string, limit int) ([]*apiv1.Session, error) { @@ -43,24 +53,26 @@ func (c *Client) ListSessionRecoveryCandidates(ctx context.Context, afterID stri return result, nil } -// SessionRecoveryFence is what crash recovery reads before it observes Substrate. -// Generation is the ID of the Session's latest lifecycle operation, or uuid.Nil -// when none is recorded. Every operation replaces it, and completion keeps it. -// ActorUID is the recorded runtime Actor, or "" when none is recorded. The -// template identifies where that Actor lives and what it must run. -type SessionRecoveryFence struct { - Generation uuid.UUID +// SessionGeneration is a live Session's lifecycle generation and the runtime it +// recorded, read together before crash recovery observes Substrate. OperationID +// is the Session's latest lifecycle operation, or uuid.Nil when none is recorded: +// every operation replaces it, and completion keeps it. ActorUID is the recorded +// runtime Actor, or "" when none is recorded. The template identifies where that +// Actor lives and what it must run. +type SessionGeneration struct { + SessionID string + OperationID uuid.UUID ActorUID string TemplateAtespace string TemplateName string } -// SessionRecoveryFence reads the lifecycle generation, recorded Actor UID, and +// GetSessionGeneration reads the lifecycle generation, recorded Actor UID, and // Actor template of a live Session in one statement. Deleted or missing Sessions // return ErrNotFound. Callers authorize access. The read takes no lock: admission // re-checks the generation and Actor UID under the Session lock. -func (c *Client) SessionRecoveryFence(ctx context.Context, id string) (*SessionRecoveryFence, error) { - type fenceRow struct { +func (c *Client) GetSessionGeneration(ctx context.Context, id string) (*SessionGeneration, error) { + type generationRow struct { OperationID *uuid.UUID ActorUID *string ActorTemplateAtespace string @@ -70,29 +82,30 @@ func (c *Client) SessionRecoveryFence(ctx context.Context, id string) (*SessionR SELECT i.operation_id, i.actor_uid, r.actor_template_atespace, r.actor_template_name FROM session_record i JOIN runtime_revision r ON r.revision = i.prepared_revision WHERE i.id = $1 AND i.state <> 'RUNTIME_STATE_DELETED' - `, pgx.RowToStructByName[fenceRow], id) + `, pgx.RowToStructByName[generationRow], id) if err != nil { - return nil, fmt.Errorf("read Session %s recovery fence: %w", id, notFoundOr(err)) + return nil, fmt.Errorf("read Session %s generation: %w", id, notFoundOr(err)) } - fence := &SessionRecoveryFence{TemplateAtespace: row.ActorTemplateAtespace, TemplateName: row.ActorTemplateName} + generation := &SessionGeneration{SessionID: id, TemplateAtespace: row.ActorTemplateAtespace, TemplateName: row.ActorTemplateName} if row.OperationID != nil { - fence.Generation = *row.OperationID + generation.OperationID = *row.OperationID } if row.ActorUID != nil { - fence.ActorUID = *row.ActorUID + generation.ActorUID = *row.ActorUID } - return fence, nil + return generation, nil } -// BeginCrashedSessionRecovery fences confirmed-crash recovery under the Session lock. -// It preserves durable results; obsolete Actors and conflicting operations fail. -// generation is the caller's SessionRecoveryFence generation, read before it -// observed the crash. A lifecycle operation begun since then makes the observation -// stale, because a concurrent revert keeps the Actor UID and returns to READY. -func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, generation uuid.UUID, actorUID string, snapshot SessionTaskSnapshot) (*SessionOperation, error) { +// BeginCrashedSessionRecovery admits confirmed-crash recovery under the Session +// lock. It preserves durable results; obsolete Actors and conflicting operations +// fail. generation is the caller's GetSessionGeneration read, taken before it +// observed the crash. The Session must still record its Actor UID, and a +// lifecycle operation begun since then makes the observation stale, because a +// concurrent revert keeps the Actor UID and returns to READY. +func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *SessionGeneration, snapshot SessionTaskSnapshot) (*SessionOperation, error) { var operation *SessionOperation err := c.withTx(ctx, func(tx pgx.Tx) error { - row, err := lockSession(ctx, tx, id) + row, err := lockSession(ctx, tx, generation.SessionID) if err != nil { return notFoundOr(err) } @@ -106,18 +119,18 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, gen SELECT actor_uid = $2 AS matches, EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') AS busy FROM session WHERE id = $1 - `, pgx.RowToStructByName[guards], row.ID, actorUID) + `, pgx.RowToStructByName[guards], row.ID, generation.ActorUID) if err != nil { return err } - if !guard.Matches || actorUID == "" { + if !guard.Matches || generation.ActorUID == "" { return ErrConflict } session, err := toSession(row) if err != nil { return err } - if session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && session.Failure.GetReason() == SessionRuntimeCrashed { + if CrashRecoveryAdmitted(session) { operation, err = toSessionOperation(row) return err } @@ -128,7 +141,7 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, id string, gen if row.OperationID != nil { current = *row.OperationID } - if current != generation { + if current != generation.OperationID { return ErrConflict } // An issued idle suspension does not exclude recovery here. It cannot finish diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index 131e7f9809..b321b7f7ee 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -56,7 +56,7 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { require.NoError(t, err) require.Empty(t, page) - operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, snapshot) + operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), snapshot) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, operation.Instance.State) require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) @@ -71,7 +71,7 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { } _, settledVersion, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) - replayed, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, snapshot) + replayed, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), snapshot) require.NoError(t, err) require.Equal(t, operation.ID, replayed.ID) _, replayedVersion, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) @@ -111,7 +111,7 @@ func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) { session, task := waitingTaskFixture(t, client) uid := "actor-" + session.Id // Callers read the generation before they observe the crash. - generation := lifecycleGeneration(t, client, session.Id) + generation := sessionGeneration(t, client, session.Id) switch { case test.actorUID != "": uid = test.actorUID @@ -136,7 +136,8 @@ func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) { } before, err := client.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) - _, err = client.BeginCrashedSessionRecovery(t.Context(), session.Id, generation, uid, SessionTaskSnapshot{}) + generation.ActorUID = uid + _, err = client.BeginCrashedSessionRecovery(t.Context(), generation, SessionTaskSnapshot{}) require.ErrorIs(t, err, ErrConflict) after, err := client.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) @@ -169,7 +170,7 @@ func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { require.NoError(t, err) uid := "actor-" + session.Id - operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), uid, SessionTaskSnapshot{}) + operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, uid), SessionTaskSnapshot{}) require.NoError(t, err) require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) pending, err := queryOne(t.Context(), client.db, `SELECT quiescence_pending FROM session_task_event WHERE sequence = $1`, pgx.RowTo[bool], work.Version) @@ -195,18 +196,27 @@ func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { require.NoError(t, err, "the retired claim no longer blocks a new turn") } -func lifecycleGeneration(t *testing.T, client *Client, sessionID string) uuid.UUID { +// sessionGeneration reads what a caller reads before it observes a crash. +func sessionGeneration(t *testing.T, client *Client, sessionID string) *SessionGeneration { t.Helper() - fence, err := client.SessionRecoveryFence(t.Context(), sessionID) + generation, err := client.GetSessionGeneration(t.Context(), sessionID) require.NoError(t, err) - return fence.Generation + return generation +} + +// observedGeneration is a current generation from a caller that observed actorUID. +func observedGeneration(t *testing.T, client *Client, sessionID, actorUID string) *SessionGeneration { + t.Helper() + generation := sessionGeneration(t, client, sessionID) + generation.ActorUID = actorUID + return generation } // completeCrashedRecovery admits and finishes a revert as a successful concurrent // caller would, leaving the Session READY on the same Actor UID. func completeCrashedRecovery(t *testing.T, client *Client, sessionID, actorUID string) { t.Helper() - operation, err := client.BeginCrashedSessionRecovery(t.Context(), sessionID, lifecycleGeneration(t, client, sessionID), actorUID, SessionTaskSnapshot{}) + operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, sessionID, actorUID), SessionTaskSnapshot{}) require.NoError(t, err) finishCrashedRecovery(t, client, sessionID, operation, actorUID) } @@ -241,7 +251,7 @@ func TestCrashedRecoveryAdmissionRollsBackOnMalformedTask(t *testing.T) { // A corrupt projection makes settlement fail after the lifecycle write. _, err = client.db.Exec(t.Context(), `UPDATE session_task SET data = $2 WHERE history_id = $1 AND id = $3`, row.HistoryID, []byte{0xff}, string(task.ID)) require.NoError(t, err) - _, err = client.BeginCrashedSessionRecovery(t.Context(), session.Id, lifecycleGeneration(t, client, session.Id), "actor-"+session.Id, SessionTaskSnapshot{}) + _, err = client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), SessionTaskSnapshot{}) require.Error(t, err) current, err := client.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) @@ -277,27 +287,29 @@ func TestCrashedRecoveryNotePinsItsTimelinePosition(t *testing.T) { require.True(t, position.Equal(*failed.Status.Timestamp), "position %s, status timestamp %s", position, failed.Status.Timestamp) } -// The fence reports the recorded Actor and the generation a crash observation -// is valid for. Admitting a recovery replaces the generation and keeps the UID. -func TestSessionRecoveryFenceReadsRecordedActor(t *testing.T) { +// The generation reports the recorded Actor and the operation a crash +// observation is valid for. Admitting a recovery replaces the operation and keeps +// the UID. +func TestSessionGenerationReadsRecordedActor(t *testing.T) { client := NewClient(setupTestDB(t), "public") sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") session, _ := waitingTaskFixture(t, client) - fence, err := client.SessionRecoveryFence(t.Context(), session.Id) + generation, err := client.GetSessionGeneration(t.Context(), session.Id) require.NoError(t, err) - require.Equal(t, "actor-"+session.Id, fence.ActorUID) - require.Equal(t, "team-a", fence.TemplateAtespace) - require.Equal(t, "revision-actor-template", fence.TemplateName) - require.NotEqual(t, uuid.Nil, fence.Generation, "creation recorded a lifecycle operation") + require.Equal(t, session.Id, generation.SessionID) + require.Equal(t, "actor-"+session.Id, generation.ActorUID) + require.Equal(t, "team-a", generation.TemplateAtespace) + require.Equal(t, "revision-actor-template", generation.TemplateName) + require.NotEqual(t, uuid.Nil, generation.OperationID, "creation recorded a lifecycle operation") - operation, err := client.BeginCrashedSessionRecovery(t.Context(), session.Id, fence.Generation, fence.ActorUID, SessionTaskSnapshot{}) + operation, err := client.BeginCrashedSessionRecovery(t.Context(), generation, SessionTaskSnapshot{}) require.NoError(t, err) - admitted, err := client.SessionRecoveryFence(t.Context(), session.Id) + admitted, err := client.GetSessionGeneration(t.Context(), session.Id) require.NoError(t, err) - require.Equal(t, operation.ID, admitted.Generation) - require.Equal(t, fence.ActorUID, admitted.ActorUID) + require.Equal(t, operation.ID, admitted.OperationID) + require.Equal(t, generation.ActorUID, admitted.ActorUID) - _, err = client.SessionRecoveryFence(t.Context(), uuid.NewString()) + _, err = client.GetSessionGeneration(t.Context(), uuid.NewString()) require.ErrorIs(t, err, ErrNotFound) } diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go index 80d0cb0dd1..1f5d8dafe5 100644 --- a/go/core/internal/service/session/recovery.go +++ b/go/core/internal/service/session/recovery.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "github.com/google/uuid" ateapipb "go.ate.dev/substrate/api/ateapi/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -29,12 +28,6 @@ var ( errRuntimeUnrecoverable = fmt.Errorf("runtime cannot be recovered: %w", database.ErrFailedPrecondition) ) -// crashRecoveryAdmitted reports a revert already admitted for a confirmed crash. -// Revert continues it without observing Substrate again. -func crashRecoveryAdmitted(session *apiv1.Session) bool { - return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Failure.GetReason() == database.SessionRuntimeCrashed -} - // lifecycleIdle reports a Session with no lifecycle operation in flight. Only an // idle Session can admit a new revert; any other operation owns the Actor. func lifecycleIdle(session *apiv1.Session) bool { @@ -44,7 +37,7 @@ func lifecycleIdle(session *apiv1.Session) bool { // crashObservation is the Substrate state of a Session's recorded Actor, taken // after the lifecycle generation it is valid for. type crashObservation struct { - generation uuid.UUID + generation *database.SessionGeneration actor *ateapipb.Actor } @@ -58,21 +51,21 @@ func (o crashObservation) crashed() bool { // crash: reverting it would restore the wrong runtime. Those errors wrap // errRuntimeUnrecoverable; transient read failures do not. // -// The fence is read before the Actor. A concurrent revert keeps the Actor UID +// The generation is read before the Actor. A concurrent revert keeps the Actor UID // and returns to READY, so only the generation shows that a crash observation // predates it. The recorded UID changes only through a lifecycle operation, // which also replaces the generation, so admission's locked generation check // rejects an observation compared against a stale UID. func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session) (crashObservation, error) { - fence, err := w.store.SessionRecoveryFence(ctx, session.Id) + generation, err := w.store.GetSessionGeneration(ctx, session.Id) if errors.Is(err, database.ErrNotFound) { return crashObservation{}, fmt.Errorf("runtime Session was deleted: %w", errRuntimeUnrecoverable) } if err != nil { return crashObservation{}, err } - binding := substrate.ActorBinding{Atespace: fence.TemplateAtespace, Name: substrate.ActorName(session.Id), - TemplateAtespace: fence.TemplateAtespace, TemplateName: fence.TemplateName} + binding := substrate.ActorBinding{Atespace: generation.TemplateAtespace, Name: substrate.ActorName(session.Id), + TemplateAtespace: generation.TemplateAtespace, TemplateName: generation.TemplateName} actor, err := w.actors.GetActor(ctx, binding.Atespace, binding.Name) if status.Code(err) == codes.NotFound { return crashObservation{}, fmt.Errorf("runtime Actor is missing: %w", errRuntimeUnrecoverable) @@ -83,10 +76,10 @@ func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session if !binding.Matches(actor) { return crashObservation{}, fmt.Errorf("runtime Actor identity or template changed: %w", errRuntimeUnrecoverable) } - if actor.GetMetadata().GetUid() != fence.ActorUID { + if actor.GetMetadata().GetUid() != generation.ActorUID { return crashObservation{}, fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) } - return crashObservation{generation: fence.Generation, actor: actor}, nil + return crashObservation{generation: generation, actor: actor}, nil } // scanRecovery runs one recovery interval and returns the cursor for the next. @@ -142,11 +135,12 @@ func (w *ActorWorkflow) recoverSessionCandidate(ctx context.Context, session *ap // Revert restores a confirmed crashed runtime without replaying its interrupted turn. // Retries join the same durable operation and preserve the Actor identity. func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*apiv1.Session, error) { - ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) - defer cancel() - if crashRecoveryAdmitted(session) { + if database.CrashRecoveryAdmitted(session) { return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, false) } + // run bounds a joined attempt. Admission bounds its own reads and execution. + ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) + defer cancel() // Only an active lifecycle operation is contention: a retry converges once it // finishes. A Session at rest in another state, such as one suspended by // deadline cleanup, stays there until an interaction resumes it, so a retry @@ -166,7 +160,7 @@ func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*ap } actor := observed.actor snapshot := substrate.LatestDurableActorSnapshot(actor) - operation, err := w.store.BeginCrashedSessionRecovery(ctx, session.Id, observed.generation, actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{ + operation, err := w.store.BeginCrashedSessionRecovery(ctx, observed.generation, database.SessionTaskSnapshot{ Atespace: actor.GetMetadata().GetAtespace(), URI: substrate.SnapshotURI(snapshot), Fidelity: snapshot.GetFidelity().String(), }) diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 3d8ae51f58..5029153de1 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -64,11 +64,13 @@ func (a *recoveryTestActors) GetActor(ctx context.Context, atespace, name string return actor, err } -func revertGeneration(t *testing.T, store *lifecycleTestStore, sessionID string) uuid.UUID { +// revertGeneration is a current generation from a caller that observed actorUID. +func revertGeneration(t *testing.T, store *lifecycleTestStore, sessionID, actorUID string) *database.SessionGeneration { t.Helper() - fence, err := store.SessionRecoveryFence(t.Context(), sessionID) + generation, err := store.GetSessionGeneration(t.Context(), sessionID) require.NoError(t, err) - return fence.Generation + generation.ActorUID = actorUID + return generation } func crashedTurnFixture(t *testing.T) (*lifecycleTestStore, *lifecycleTestActors, *apiv1.Session, *a2a.Task, int64) { @@ -295,7 +297,7 @@ func TestSessionRecoveryCandidatePolicy(t *testing.T) { require.Empty(t, page, "manual recovery does not inspect idle or active runtimes") actor, err := actors.GetActor(t.Context(), "team-a", substrate.ActorName(active.Id)) require.NoError(t, err) - _, err = store.BeginCrashedSessionRecovery(t.Context(), active.Id, revertGeneration(t, store, active.Id), actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + _, err = store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, active.Id, actor.GetMetadata().GetUid()), database.SessionTaskSnapshot{}) require.NoError(t, err) page, err = store.ListSessionRecoveryCandidates(t.Context(), "", 100) require.NoError(t, err) @@ -549,7 +551,7 @@ func TestSessionRecoveryPagesPastUnavailableActors(t *testing.T) { for _, id := range ids { actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(id)) require.NoError(t, err) - _, err = store.BeginCrashedSessionRecovery(t.Context(), id, revertGeneration(t, store, id), actor.GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + _, err = store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, id, actor.GetMetadata().GetUid()), database.SessionTaskSnapshot{}) require.NoError(t, err) } slices.Sort(ids) @@ -640,7 +642,7 @@ func TestDeleteSupersedesUnissuedCrashRecovery(t *testing.T) { t.Run(test.name, func(t *testing.T) { store, base, session, _, _ := crashedTurnFixture(t) key := actorKey("team-a", substrate.ActorName(session.Id)) - _, err := store.BeginCrashedSessionRecovery(t.Context(), session.Id, revertGeneration(t, store, session.Id), base.actors[key].GetMetadata().GetUid(), database.SessionTaskSnapshot{}) + _, err := store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, session.Id, base.actors[key].GetMetadata().GetUid()), database.SessionTaskSnapshot{}) require.NoError(t, err) if test.gone { delete(base.actors, key) diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index e77dd91389..acd45136a3 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -16,8 +16,8 @@ import ( type workflowStore interface { ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) - SessionRecoveryFence(context.Context, string) (*database.SessionRecoveryFence, error) - BeginCrashedSessionRecovery(context.Context, string, uuid.UUID, string, database.SessionTaskSnapshot) (*database.SessionOperation, error) + GetSessionGeneration(context.Context, string) (*database.SessionGeneration, error) + BeginCrashedSessionRecovery(context.Context, *database.SessionGeneration, database.SessionTaskSnapshot) (*database.SessionOperation, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) FinishSessionQuiescence(context.Context, *database.SessionQuiescence, *database.SessionTaskSnapshot) error @@ -162,7 +162,7 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session } var transition *substrate.ActorTransition switch { - case crashRecovery(operation): + case database.CrashRecoveryAdmitted(operation.Instance): transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding) case operation.ExecutorID != uuid.Nil: transition, err = substrate.PrepareActorRetry(ctx, w.actors, binding, kind, creation) @@ -221,7 +221,7 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. if admitted.ExecutorID != uuid.Nil { return nil, cause // An earlier attempt may have issued work; retain its intent. } - if crashRecovery(admitted) { + if database.CrashRecoveryAdmitted(admitted.Instance) { // Admission already failed the interrupted turn, so clearing the operation // would leave a FAILED Session that nothing retries. Keep the intent for the // next scan. No claim was taken, so Delete can still supersede it. @@ -240,15 +240,6 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. return nil, errors.Join(cause, err) } -// crashRecovery reports a RESUME admitted after Substrate confirmed a crash. -// It reverts the recorded Actor in place instead of resuming it. -func crashRecovery(operation *database.SessionOperation) bool { - session := operation.Instance - return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && - session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && - session.Failure.GetReason() == database.SessionRuntimeCrashed -} - func operationOutcome(operation *database.SessionOperation) (*apiv1.Session, error) { if operation.Instance.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { return operation.Instance, nil From 7a1ae645eb6455b1623e953c95820474068abbad Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 14:47:09 -0800 Subject: [PATCH 06/15] refactor(session): record crash recovery as a REVERT operation Review follow-ups on #3395: - RUNTIME_OPERATION_REVERT replaces the revert flag on ActorTransition and the FAILED + RESUME + RuntimeCrashed encoding. CrashRecoveryAdmitted is now Operation == REVERT, revert is a case in ApplyActorTransition, and BeginSessionOperation can join a REVERT but never start one. The runtime_instance operation CHECK accepts the new value. - Recovery no longer runs on a background ticker. An admitted recovery is a durable REVERT operation that a retried RevertSession joins, the way Create and Delete retries work. ListSessionRecoveryCandidates, the scan functions, and the recovery goroutine in Start are removed. - Admission pins the durable snapshot URI it settled waits against, keyed by the REVERT operation ID. PrepareActorRevert requires the Actor to still report that snapshot, so a snapshot uploaded after admission cannot bring back a wait admission already failed. - A recovery whose Actor is missing, replaced, or holds another snapshot is abandoned even after an earlier attempt claimed it: no revert of the recorded runtime can still be in progress. The Session stays FAILED with reason RuntimeUnrecoverable and no operation, so Delete proceeds. A live claim still wins. - RevertSession returns the Session when it is READY and its recorded Actor is SUSPENDED, so a retry after a lost success response succeeds instead of reporting KAGENT_SESSION_RUNTIME_NOT_CRASHED. Docs describe what a client sees while a recovery is pending. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 47 +++-- docs/lifecycle-retries.md | 11 +- go/api/gen/kagent/api/v1/runtime.pb.go | 10 +- .../internal/database/runtime_instances.go | 14 +- go/core/internal/database/session_recovery.go | 108 +++++++--- .../database/session_recovery_test.go | 61 +++++- .../grpcserver/session_recovery_test.go | 9 +- .../internal/service/session/quiescence.go | 28 +-- .../service/session/quiescence_test.go | 4 - go/core/internal/service/session/recovery.go | 84 +++----- .../internal/service/session/recovery_test.go | 196 ++++-------------- go/core/internal/service/session/workflow.go | 43 +++- go/core/internal/substrate/lifecycle.go | 43 ++-- go/core/internal/substrate/lifecycle_test.go | 12 +- .../pkg/migrations/core/000001_initial.sql | 9 +- go/core/test/e2e/session_revert_test.go | 9 +- proto/kagent/api/v1/runtime.proto | 3 + ui/src/api/domain/agentInstances.ts | 1 + ui/src/api/grpc/operations.ts | 1 + .../agent-instances/instanceLabels.ts | 1 + ui/src/components/chat/lifecycleReading.ts | 9 + ui/src/generated/kagent/api/v1/runtime_pb.ts | 10 +- ui/src/mocks/transport.ts | 1 + 23 files changed, 370 insertions(+), 344 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 1a28d3522b..8a6c325457 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -233,13 +233,23 @@ returns `FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`, because no retry restores it. A transport failure returns `UNAVAILABLE`. Only an active lifecycle operation returns `ABORTED`. A Session at rest outside READY returns `KAGENT_SESSION_RUNTIME_NOT_CRASHED`, so a client does not retry -it until its deadline. - -Admission records FAILED/RESUME with a RuntimeCrashed failure under the Session -lock. The same transaction publishes a durable terminal result saved before the -crash, or fails nonterminal work absent from the retained snapshot. Waiting tasks -captured by that snapshot remain available. Checkpoint creation excludes competing -recovery. +it until its deadline. A READY Session whose recorded Actor is SUSPENDED has +nothing to recover, so RevertSession returns it. That is also what a retry sees +after a revert whose response was lost. + +Admission records FAILED/REVERT with a RuntimeCrashed failure under the Session +lock. REVERT is its own `RuntimeOperation`: only a confirmed crash starts one, and +`BeginSessionOperation` can join a REVERT but never start one. The same transaction +publishes a durable terminal result saved before the crash, or fails nonterminal +work absent from the retained snapshot. Waiting tasks captured by that snapshot +remain available. Checkpoint creation excludes competing recovery. + +Admission also pins the durable snapshot URI it judged waits against on the +Session, keyed by the REVERT operation ID. Substrate's revert restores the latest +durable snapshot it holds when the call runs, so execution first requires the +Actor to still report the pinned snapshot, then requires the reverted Actor to +report it too. Otherwise a snapshot uploaded after admission could bring back a +wait that admission had already failed. The caller reads the Session's generation before it observes Substrate: its lifecycle generation, recorded Actor UID, and Actor template, in one statement. @@ -254,18 +264,29 @@ An issued idle suspension does not exclude recovery. It cannot finish against th crashed Actor, and its worker keeps the claim when suspension fails. Admission retires the claim in its transaction, so a late finish records nothing. -The worker reverts the same Actor to its last durable snapshot without replaying +Execution reverts the same Actor to its last durable snapshot without replaying input. A runtime without a durable snapshot returns to its initial state. An interrupted revert continues from REVERTING; a lost response can be reconciled -from SUSPENDED. Already admitted recoveries survive API restarts and retry on the -existing recovery interval in ID-ordered pages of at most 100. The worker does -not inspect READY Sessions or initiate a new recovery. +from SUSPENDED. Nothing retries a recovery in the background. An admitted +recovery survives API restarts as a durable REVERT operation, and the client +continues it by retrying RevertSession, as with Create and Delete. While it is +pending, `GetSession` reports FAILED/REVERT with failure reason `RuntimeCrashed`, +and interactions are refused because a lifecycle operation is in progress. + +A failed Actor read before the claim leaves the operation unclaimed, so Delete +can supersede it. If an attempt claimed the revert and the Actor is then missing, +replaced, or reports a different durable snapshot, no revert of the recorded +runtime can still be in progress. The next attempt abandons the operation: the +Session stays FAILED with reason `RuntimeUnrecoverable` and no operation, so +Delete can proceed. A live claim still wins, so an attempt that is issuing the +revert finishes first. A successful revert returns the Session to READY with the same Actor suspended. The next authorized interaction resumes it through Substrate ingress. Runtime changes after the retained snapshot are lost; the discarded turn receives no -checkpoint at that older snapshot. Another revert against the now-healthy Actor -returns `FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. +checkpoint at that older snapshot. Another revert while the Actor stays +suspended returns the Session. Once an interaction resumes it, a revert returns +`FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. ```mermaid sequenceDiagram diff --git a/docs/lifecycle-retries.md b/docs/lifecycle-retries.md index 3a9b129686..6ccb6694f3 100644 --- a/docs/lifecycle-retries.md +++ b/docs/lifecycle-retries.md @@ -48,8 +48,11 @@ It preserves durable terminal results and returns the Session to READY with its It discards runtime changes since that snapshot and does not replay input. A runtime without a snapshot restarts from its initial state. -Retry an interrupted revert against the same Session ID. Pending recovery retains its operation across API restarts. -The lifecycle worker retries already admitted recoveries; it does not initiate recovery for other crashed Sessions. +Retry an interrupted revert against the same Session ID; nothing retries it in the background. +Pending recovery retains its operation across API restarts, and a retry joins it. +While recovery is pending, `GetSession` reports state `FAILED`, operation `REVERT`, and failure reason `RuntimeCrashed`. +Interactions fail because a lifecycle operation is in progress. Call `RevertSession` again to finish the recovery. +A retry after a revert whose response was lost returns the recovered Session. Checkpoint creation returns `ABORTED` until it finishes. A revert that observed the crash before another lifecycle operation began also returns `ABORTED`. So does a revert while another lifecycle operation is active. @@ -59,8 +62,8 @@ Its `google.rpc.ErrorInfo` detail, in domain `kagent.dev`, names one of two reas | Reason | Meaning | Client action | | --- | --- | --- | -| `KAGENT_SESSION_RUNTIME_NOT_CRASHED` | The runtime is healthy or suspended. A retry after a revert whose response was lost lands here. | Treat as recovered and continue the conversation. | -| `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` | The recorded Actor is missing, or a same-name Actor replaced it. | Stop retrying. Start another Session, or fork one from a checkpoint. | +| `KAGENT_SESSION_RUNTIME_NOT_CRASHED` | The runtime is running, or the Session is at rest outside READY. | Continue the conversation. | +| `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` | The recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. The Session stays `FAILED` with failure reason `RuntimeUnrecoverable`. | Stop retrying. Delete the Session, then start another or fork one from a checkpoint. | A `FAILED_PRECONDITION` without either reason follows the general table below. Read the reason rather than the message text, which is not stable. diff --git a/go/api/gen/kagent/api/v1/runtime.pb.go b/go/api/gen/kagent/api/v1/runtime.pb.go index 88c28e575d..77fe6e91f0 100644 --- a/go/api/gen/kagent/api/v1/runtime.pb.go +++ b/go/api/gen/kagent/api/v1/runtime.pb.go @@ -95,6 +95,9 @@ const ( RuntimeOperation_RUNTIME_OPERATION_DELETE RuntimeOperation = 4 // No lifecycle operation is active. RuntimeOperation_RUNTIME_OPERATION_NONE RuntimeOperation = 5 + // Restores a crashed runtime to its last durable snapshot. Only a confirmed + // crash admits it; the Session stays FAILED until it finishes. + RuntimeOperation_RUNTIME_OPERATION_REVERT RuntimeOperation = 6 ) // Enum value maps for RuntimeOperation. @@ -106,6 +109,7 @@ var ( 3: "RUNTIME_OPERATION_RESUME", 4: "RUNTIME_OPERATION_DELETE", 5: "RUNTIME_OPERATION_NONE", + 6: "RUNTIME_OPERATION_REVERT", } RuntimeOperation_value = map[string]int32{ "RUNTIME_OPERATION_UNSPECIFIED": 0, @@ -114,6 +118,7 @@ var ( "RUNTIME_OPERATION_RESUME": 3, "RUNTIME_OPERATION_DELETE": 4, "RUNTIME_OPERATION_NONE": 5, + "RUNTIME_OPERATION_REVERT": 6, } ) @@ -156,14 +161,15 @@ const file_kagent_api_v1_runtime_proto_rawDesc = "" + "\x17RUNTIME_STATE_SUSPENDED\x10\x03\x12\x18\n" + "\x14RUNTIME_STATE_FAILED\x10\x04\x12\x1a\n" + "\x16RUNTIME_STATE_DELETING\x10\x05\x12\x19\n" + - "\x15RUNTIME_STATE_DELETED\x10\x06*\xca\x01\n" + + "\x15RUNTIME_STATE_DELETED\x10\x06*\xe8\x01\n" + "\x10RuntimeOperation\x12!\n" + "\x1dRUNTIME_OPERATION_UNSPECIFIED\x10\x00\x12\x1c\n" + "\x18RUNTIME_OPERATION_CREATE\x10\x01\x12\x1d\n" + "\x19RUNTIME_OPERATION_SUSPEND\x10\x02\x12\x1c\n" + "\x18RUNTIME_OPERATION_RESUME\x10\x03\x12\x1c\n" + "\x18RUNTIME_OPERATION_DELETE\x10\x04\x12\x1a\n" + - "\x16RUNTIME_OPERATION_NONE\x10\x05B=Z;github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1;apiv1b\x06proto3" + "\x16RUNTIME_OPERATION_NONE\x10\x05\x12\x1c\n" + + "\x18RUNTIME_OPERATION_REVERT\x10\x06B=Z;github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1;apiv1b\x06proto3" var ( file_kagent_api_v1_runtime_proto_rawDescOnce sync.Once diff --git a/go/core/internal/database/runtime_instances.go b/go/core/internal/database/runtime_instances.go index f57ded3066..dfdc24570c 100644 --- a/go/core/internal/database/runtime_instances.go +++ b/go/core/internal/database/runtime_instances.go @@ -68,7 +68,11 @@ func runtimeOperationNeeded(row runtimeInstanceRow, requested apiv1.RuntimeOpera if err != nil { return false, err } - if requested < apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE || requested > apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE { + switch requested { + case apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, + apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, + apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT: + default: return false, fmt.Errorf("invalid runtime operation %s: %w", requested, ErrFailedPrecondition) } if state == apiv1.RuntimeState_RUNTIME_STATE_DELETED { @@ -86,6 +90,11 @@ func runtimeOperationNeeded(row runtimeInstanceRow, requested apiv1.RuntimeOpera } operation = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE } + if requested == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT { + // Only BeginCrashedSessionRecovery admits a revert, after Substrate confirms + // the crash. This path can join one but never start one. + return false, ErrConflict + } var expected, target apiv1.RuntimeState expectedOperation := apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE switch requested { @@ -178,7 +187,8 @@ func completeRuntimeOperation(row runtimeInstanceRow, operationID, executorID uu return row, ErrConflict } switch operation { - case apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME: + case apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, + apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT: row.State = apiv1.RuntimeState_RUNTIME_STATE_READY.String() case apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND: row.State = apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED.String() diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index cac3b1b8e4..5e0806a23a 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -18,39 +18,17 @@ import ( // that the recorded Actor crashed. Transport failures cannot admit this work. const SessionRuntimeCrashed = "RuntimeCrashed" +// SessionRuntimeUnrecoverable identifies a Session whose admitted crash +// recovery was abandoned because its recorded Actor is gone, was replaced, or no +// longer holds the admitted snapshot. No retry restores it; Delete still can. +const SessionRuntimeUnrecoverable = "RuntimeUnrecoverable" + // CrashRecoveryAdmitted reports a Session whose crash recovery was admitted and -// has not finished: FAILED for a confirmed crash, with the recovery's RESUME -// still in flight. A retry joins that operation instead of observing Substrate -// again, and execution reverts the recorded Actor in place instead of resuming it. +// has not finished. A retry joins that operation instead of observing Substrate +// again. Only BeginCrashedSessionRecovery starts a REVERT, so the operation alone +// identifies it. func CrashRecoveryAdmitted(session *apiv1.Session) bool { - return session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && - session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME && - session.Failure.GetReason() == SessionRuntimeCrashed -} - -// ListSessionRecoveryCandidates finds recoveries admitted by an explicit request. -// Retrying durable work cannot admit recovery for a READY Session. -func (c *Client) ListSessionRecoveryCandidates(ctx context.Context, afterID string, limit int) ([]*apiv1.Session, error) { - rows, err := queryMany(ctx, c.db, ` - SELECT id, user_id, prepared_revision, state, data, operation, context_id, - source_checkpoint_id, history_id, operation_id, executor_id - FROM session_record i - WHERE ($1::text = '' OR i.id > $1::uuid) - AND i.state = 'RUNTIME_STATE_FAILED' AND i.operation = 'RUNTIME_OPERATION_RESUME' - ORDER BY i.id LIMIT $2 - `, pgx.RowToStructByName[sessionRow], afterID, limit) - if err != nil { - return nil, err - } - result := make([]*apiv1.Session, 0, len(rows)) - for _, row := range rows { - session, err := toSession(row) - if err != nil { - return nil, err - } - result = append(result, session) - } - return result, nil + return session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT } // SessionGeneration is a live Session's lifecycle generation and the runtime it @@ -154,18 +132,24 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se return ErrConflict } row.State = apiv1.RuntimeState_RUNTIME_STATE_FAILED.String() - row.runtimeInstanceRow, err = beginRuntimeOperation(ctx, tx, row.runtimeInstanceRow, runtimeKindAgent, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME) + row.runtimeInstanceRow, err = beginRuntimeOperation(ctx, tx, row.runtimeInstanceRow, runtimeKindAgent, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT) if err != nil { return err } - session.State, session.Operation = apiv1.RuntimeState_RUNTIME_STATE_FAILED, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME + session.State, session.Operation = apiv1.RuntimeState_RUNTIME_STATE_FAILED, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT session.Failure = &apiv1.Failure{Reason: SessionRuntimeCrashed, Message: "Runtime crashed; recovery to its last snapshot is pending"} session.UpdatedAt = timestamppb.Now() row.Data, err = marshalSession(session) if err != nil { return err } - if err := execSQL(ctx, tx, `UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL WHERE id = $1`, row.ID, row.Data); err != nil { + // Execution must restore the snapshot settlement below judged waits against, + // so the operation pins it rather than letting Substrate choose at revert time. + if err := execSQL(ctx, tx, ` + UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL, + recovery_operation_id = $3, recovery_snapshot_uri = $4 + WHERE id = $1 + `, row.ID, row.Data, row.OperationID, snapshot.URI); err != nil { return err } if err := settleCrashedSessionTask(ctx, tx, row, snapshot); err != nil { @@ -177,6 +161,62 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se return operation, err } +// GetSessionRecoverySnapshot returns the durable snapshot URI pinned when the +// Session's REVERT operation was admitted. The URI is empty when the crashed +// Actor had no durable snapshot. Any other operation returns ErrNotFound. +func (c *Client) GetSessionRecoverySnapshot(ctx context.Context, sessionID string, operationID uuid.UUID) (string, error) { + uri, err := queryOne(ctx, c.db, ` + SELECT s.recovery_snapshot_uri FROM session s JOIN runtime_instance r USING (id) + WHERE s.id = $1 AND s.recovery_operation_id = $2 + AND r.operation_id = $2 AND r.operation = 'RUNTIME_OPERATION_REVERT' + `, pgx.RowTo[string], sessionID, operationID) + if err != nil { + return "", fmt.Errorf("read Session %s recovery snapshot: %w", sessionID, notFoundOr(err)) + } + return uri, nil +} + +// AbandonSessionRecovery ends an admitted REVERT whose Actor can no longer be +// restored: it is missing, was replaced, or no longer holds the pinned snapshot. +// Unlike a preparation failure it may end claimed work, because those +// observations show that no revert of the recorded Actor can still succeed. A +// live claim still wins, so an attempt that is issuing the revert finishes first. +// The Session stays FAILED with no operation, so Delete can proceed. +func (c *Client) AbandonSessionRecovery(ctx context.Context, sessionID string, operationID uuid.UUID, cause string) (*apiv1.Session, error) { + var result *apiv1.Session + err := c.withTx(ctx, func(tx pgx.Tx) error { + row, err := lockSession(ctx, tx, sessionID) + if err != nil { + return notFoundOr(err) + } + tag, err := tx.Exec(ctx, ` + UPDATE runtime_instance SET operation = 'RUNTIME_OPERATION_NONE', executor_id = NULL, executor_expires_at = NULL + WHERE id = $1 AND kind = 'agent' AND operation_id = $2 AND operation = 'RUNTIME_OPERATION_REVERT' + AND (executor_id IS NULL OR executor_expires_at <= clock_timestamp()) + `, row.ID, operationID) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrConflict + } + row.Operation, row.ExecutorID = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE.String(), nil + result, err = toSession(row) + if err != nil { + return err + } + result.Operation = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE + result.Failure = &apiv1.Failure{Reason: SessionRuntimeUnrecoverable, Message: "Runtime cannot be recovered: " + cause} + result.UpdatedAt = timestamppb.Now() + row.Data, err = marshalSession(result) + if err != nil { + return err + } + return execSQL(ctx, tx, `UPDATE session SET data = $2 WHERE id = $1`, row.ID, row.Data) + }) + return result, err +} + // settleCrashedSessionTask retains captured waits and fails discarded nonterminal work. // Callers hold the Session lock; recovery cannot checkpoint discarded state. func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index b321b7f7ee..0c443a17d6 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -52,14 +52,13 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { } before, _, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) - page, err := client.ListSessionRecoveryCandidates(t.Context(), "", 100) - require.NoError(t, err) - require.Empty(t, page) + _, err = client.GetSessionRecoverySnapshot(t.Context(), session.Id, uuid.New()) + require.ErrorIs(t, err, ErrNotFound) operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), snapshot) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, operation.Instance.State) - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, operation.Instance.Operation) require.Equal(t, SessionRuntimeCrashed, operation.Instance.Failure.Reason) stored, err := client.GetSettledSessionTask(t.Context(), session.Id, string(task.ID), nil) require.NoError(t, err) @@ -86,10 +85,9 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { require.ErrorIs(t, client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "next"), ErrConflict) _, _, err = client.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID)}, session.Creator, uuid.NewString()) require.ErrorIs(t, err, ErrConflict) - page, err = client.ListSessionRecoveryCandidates(t.Context(), "", 100) + pinned, err := client.GetSessionRecoverySnapshot(t.Context(), session.Id, operation.ID) require.NoError(t, err) - require.Len(t, page, 1) - require.Equal(t, session.Id, page[0].Id) + require.Equal(t, snapshot.URI, pinned, "the operation pins the snapshot that settlement judged waits against") }) } } @@ -172,7 +170,7 @@ func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, uid), SessionTaskSnapshot{}) require.NoError(t, err) - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, operation.Instance.Operation) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, operation.Instance.Operation) pending, err := queryOne(t.Context(), client.db, `SELECT quiescence_pending FROM session_task_event WHERE sequence = $1`, pgx.RowTo[bool], work.Version) require.NoError(t, err) require.False(t, pending, "admission retires the claim in its own transaction") @@ -313,3 +311,50 @@ func TestSessionGenerationReadsRecordedActor(t *testing.T) { _, err = client.GetSessionGeneration(t.Context(), uuid.NewString()) require.ErrorIs(t, err, ErrNotFound) } + +func TestAbandonedRecoveryLetsDeleteProceed(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, _ := waitingTaskFixture(t, client) + uid := "actor-" + session.Id + operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, uid), SessionTaskSnapshot{}) + require.NoError(t, err) + executor := uuid.New() + claimed, err := client.ClaimSessionOperation(t.Context(), session.Id, operation.ID, executor) + require.NoError(t, err) + require.True(t, claimed) + + _, err = client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") + require.ErrorIs(t, err, ErrConflict, "a live claim may still be issuing the revert") + _, err = client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, false) + require.ErrorIs(t, err, ErrConflict, "a claimed recovery blocks Delete") + + require.NoError(t, client.ReleaseRuntimeOperation(t.Context(), session.Id, operation.ID, executor)) + abandoned, err := client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, abandoned.State) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, abandoned.Operation) + require.Equal(t, SessionRuntimeUnrecoverable, abandoned.Failure.GetReason()) + stored, err := client.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.True(t, proto.Equal(abandoned, stored)) + _, err = client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") + require.ErrorIs(t, err, ErrConflict) + + deletion, err := client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, false) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, deletion.Instance.Operation) +} + +func TestRevertCanOnlyJoinAnAdmittedRecovery(t *testing.T) { + client := NewClient(setupTestDB(t), "public") + sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") + session, _ := waitingTaskFixture(t, client) + _, err := client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, false) + require.ErrorIs(t, err, ErrConflict, "only a confirmed crash admits a revert") + admitted, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), SessionTaskSnapshot{}) + require.NoError(t, err) + joined, err := client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, false) + require.NoError(t, err) + require.Equal(t, admitted.ID, joined.ID) +} diff --git a/go/core/internal/grpcserver/session_recovery_test.go b/go/core/internal/grpcserver/session_recovery_test.go index d737b784b9..e334afb469 100644 --- a/go/core/internal/grpcserver/session_recovery_test.go +++ b/go/core/internal/grpcserver/session_recovery_test.go @@ -177,10 +177,11 @@ func TestRevertSessionThroughGRPC(t *testing.T) { require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, ready.Session.State) require.Nil(t, ready.Session.Failure) - // A client whose success response was lost retries and learns that - // the runtime is already healthy, not that it cannot be recovered. - _, err = client.RevertSession(owner, request) - requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", "a recovered runtime has nothing to revert") + // A client whose success response was lost retries and gets the + // recovered Session back, as Create and Delete retries do. + retried, err := client.RevertSession(owner, request) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, retried.Session.State) if test.lostResponse { require.Equal(t, 1, calls, "a lost response cannot repeat the revert") } else { diff --git a/go/core/internal/service/session/quiescence.go b/go/core/internal/service/session/quiescence.go index 97e37a5a2a..283e70c7cc 100644 --- a/go/core/internal/service/session/quiescence.go +++ b/go/core/internal/service/session/quiescence.go @@ -28,15 +28,11 @@ func (w *ActorWorkflow) wakeQuiescence() { } } -// Start suspends idle sessions to VOLUMES independently of task publication and -// retries crash recoveries admitted by explicit requests. -// Startup and periodic scans discover idle work missed by local signals. Workers -// drain claims with bounded concurrency. Crash recovery scans on its own ticker, so -// a slow idle suspension cannot delay a recovery retry and a slow revert cannot -// hold up idle work. +// Start suspends idle sessions to VOLUMES independently of task publication. +// Startup and periodic recovery scans discover work missed by local signals. +// Workers drain claims with bounded concurrency; task reads never wait for them. func (w *ActorWorkflow) Start(ctx context.Context) error { var workers sync.WaitGroup - workers.Go(func() { w.recoverCrashedSessions(ctx) }) for range 4 { workers.Go(func() { timer := time.NewTimer(w.recoveryInterval) @@ -76,24 +72,6 @@ func (w *ActorWorkflow) Start(ctx context.Context) error { return nil } -// recoverCrashedSessions runs one recovery scan per tick. -// The cursor carries across ticks, so a large backlog cannot starve the Sessions -// after its first page, and an Actor that stays unreadable is revisited when the -// scan wraps. -func (w *ActorWorkflow) recoverCrashedSessions(ctx context.Context) { - ticker := time.NewTicker(w.recoveryInterval) - defer ticker.Stop() - var after string - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - after = w.scanRecovery(ctx, after) - } -} - func (w *ActorWorkflow) quiesceIdleSession(ctx context.Context, work *database.SessionQuiescence) { runtimeCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) snapshot, err := w.Quiesce(runtimeCtx, work.Session) diff --git a/go/core/internal/service/session/quiescence_test.go b/go/core/internal/service/session/quiescence_test.go index d6b9af4991..99e8852d72 100644 --- a/go/core/internal/service/session/quiescence_test.go +++ b/go/core/internal/service/session/quiescence_test.go @@ -32,10 +32,6 @@ type idleQuiescenceStore struct { var _ workflowStore = (*idleQuiescenceStore)(nil) -func (s *idleQuiescenceStore) ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) { - return nil, nil -} - func (s *idleQuiescenceStore) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) { s.claims.Add(1) if s.claimErr != nil { diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go index 1f5d8dafe5..33a15f9fae 100644 --- a/go/core/internal/service/session/recovery.go +++ b/go/core/internal/service/session/recovery.go @@ -12,16 +12,17 @@ import ( apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" "github.com/kagent-dev/kagent/go/core/internal/database" "github.com/kagent-dev/kagent/go/core/internal/substrate" - "github.com/kagent-dev/kagent/go/pkg/logging" ) // Revert refuses a Session with nothing to act on for one of two reasons. They // ask opposite things of a client, so the API reports them with distinct reasons. -// errRuntimeNotCrashed means nothing needs recovering: the runtime is healthy or -// suspended. A retry after a revert whose response was lost lands here, and the -// next interaction proceeds. -// errRuntimeUnrecoverable means the recorded Actor is gone or was replaced. No -// retry restores it, and the client has to start another Session. Transient +// errRuntimeNotCrashed means nothing needs recovering: the runtime is running or +// the Session is at rest in another state. A READY Session whose Actor is +// suspended is not refused; Revert returns it, which also covers a retry after a +// revert whose response was lost. +// errRuntimeUnrecoverable means the recorded Actor is gone, was replaced, or no +// longer holds the admitted snapshot. No retry restores it, and the client has +// to start another Session. Transient // Substrate and database failures wrap neither, so they stay retryable. var ( errRuntimeNotCrashed = fmt.Errorf("runtime is not crashed: %w", database.ErrFailedPrecondition) @@ -45,6 +46,11 @@ func (o crashObservation) crashed() bool { return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED } +// reverted reports a recorded Actor at rest, as a completed revert leaves it. +func (o crashObservation) reverted() bool { + return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED +} + // observeCrash reads the Session's recorded Actor and reports whether Substrate // confirms it crashed. Revert admits recovery only through this check. An Actor // that is missing, replaced, or from another template is an error, never a @@ -82,61 +88,17 @@ func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session return crashObservation{generation: generation, actor: actor}, nil } -// scanRecovery runs one recovery interval and returns the cursor for the next. -// Admitted recoveries page through PostgreSQL in ID order. -func (w *ActorWorkflow) scanRecovery(ctx context.Context, afterID string) string { - scanCtx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) - after, err := w.recoverSessionPage(scanCtx, afterID) - cancel() - if err != nil && ctx.Err() == nil { - logging.FromContext(ctx).ErrorContext(ctx, "scan interrupted sessions", "error", err) - } - return after -} - -// recoverSessionPage retries one ID-ordered page of admitted recoveries. -// READY Sessions are excluded, so a background retry cannot admit new work. -func (w *ActorWorkflow) recoverSessionPage(ctx context.Context, afterID string) (string, error) { - const pageSize = 100 - sessions, err := w.store.ListSessionRecoveryCandidates(ctx, afterID, pageSize) - if err != nil { - return afterID, err - } - for _, session := range sessions { - w.recoverCandidate(ctx, session) - afterID = session.Id - if ctx.Err() != nil { - return afterID, ctx.Err() - } - } - if len(sessions) < pageSize { - afterID = "" - } - return afterID, nil -} - -func (w *ActorWorkflow) recoverCandidate(ctx context.Context, session *apiv1.Session) { - attemptCtx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) - err := w.recoverSessionCandidate(attemptCtx, session) - cancel() - if err != nil && !errors.Is(err, database.ErrConflict) && ctx.Err() == nil { - logging.FromContext(ctx).ErrorContext(ctx, "recover crashed session", "session_id", session.Id, "error", err) - } -} - -func (w *ActorWorkflow) recoverSessionCandidate(ctx context.Context, session *apiv1.Session) error { - _, err := w.Revert(ctx, session) - if errors.Is(err, database.ErrFailedPrecondition) { - return nil - } - return err -} - // Revert restores a confirmed crashed runtime without replaying its interrupted turn. -// Retries join the same durable operation and preserve the Actor identity. +// Retries join the same durable operation and preserve the Actor identity. Nothing +// retries an admitted recovery in the background: the caller retries RevertSession +// until it succeeds, as with CreateSession and DeleteSession. func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*apiv1.Session, error) { if database.CrashRecoveryAdmitted(session) { - return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, false) + return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, false) + } + if session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE && + session.Failure.GetReason() == database.SessionRuntimeUnrecoverable { + return nil, fmt.Errorf("%s: %w", session.Failure.GetMessage(), errRuntimeUnrecoverable) } // run bounds a joined attempt. Admission bounds its own reads and execution. ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) @@ -155,6 +117,12 @@ func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*ap if err != nil { return nil, err } + if observed.reverted() { + // A READY Session whose recorded Actor is at rest has nothing to recover. + // This is also what a retry sees after a revert whose response was lost, + // so it returns the Session the way Create and Delete retries do. + return session, nil + } if !observed.crashed() { return nil, errRuntimeNotCrashed } diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 5029153de1..09aa6f6915 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -3,9 +3,8 @@ package session import ( "context" "crypto/sha256" + "errors" "fmt" - "slices" - "sync/atomic" "testing" "time" @@ -121,7 +120,7 @@ func TestCrashedSessionRecoveryRetriesWithoutLosingState(t *testing.T) { failed, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, failed.State) - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, failed.Operation) + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, failed.Operation) require.Equal(t, database.SessionRuntimeCrashed, failed.Failure.Reason) interrupted, interruptedVersion, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) @@ -131,9 +130,10 @@ func TestCrashedSessionRecoveryRetriesWithoutLosingState(t *testing.T) { require.Equal(t, a2a.TaskStateFailed, published.Status.State) require.Len(t, published.History, 1) require.ErrorIs(t, store.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), uuid.NewString()), database.ErrConflict) - // A new workflow instance discovers the admitted operation from PostgreSQL. + // A retried RevertSession on another workflow instance joins the admitted + // operation from PostgreSQL. restarted := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) - _, err = restarted.recoverSessionPage(t.Context(), "") + _, err = restarted.Revert(t.Context(), failed) require.NoError(t, err) ready, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) @@ -147,8 +147,9 @@ func TestCrashedSessionRecoveryRetriesWithoutLosingState(t *testing.T) { _, recoveredVersion, err := store.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) require.Equal(t, interruptedVersion, recoveredVersion, "retries do not append another terminal event") - _, err = restarted.Revert(t.Context(), ready) - require.ErrorIs(t, err, database.ErrFailedPrecondition) + retried, err := restarted.Revert(t.Context(), ready) + require.NoError(t, err, "a retry after a completed revert returns the Session") + require.True(t, proto.Equal(ready, retried)) require.Equal(t, test.wantReverts, calls, "completed recovery cannot discard another execution") }) } @@ -270,41 +271,6 @@ func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { } } -func TestSessionRecoveryCandidatePolicy(t *testing.T) { - store, actors, active, _, _ := crashedTurnFixture(t) - workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second) - for _, state := range []a2a.TaskState{"", a2a.TaskStateInputRequired, a2a.TaskStateAuthRequired, a2a.TaskStateWorking} { - session, _, err := store.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: "alice", Agent: active.Agent}, uuid.NewString()) - require.NoError(t, err) - session, err = workflow.Create(t.Context(), session) - require.NoError(t, err) - if state == "" { - continue - } - message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("input")) - message.ContextID = session.ContextId - task := a2a.NewSubmittedTask(message, message) - task.Status.State = state - hash := sha256.Sum256([]byte(task.ID)) - version, err := store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") - require.NoError(t, err) - if state != a2a.TaskStateWorking { - require.NoError(t, store.SettleSessionTask(t.Context(), session.Id, string(task.ID), version)) - } - } - page, err := store.ListSessionRecoveryCandidates(t.Context(), "", 100) - require.NoError(t, err) - require.Empty(t, page, "manual recovery does not inspect idle or active runtimes") - actor, err := actors.GetActor(t.Context(), "team-a", substrate.ActorName(active.Id)) - require.NoError(t, err) - _, err = store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, active.Id, actor.GetMetadata().GetUid()), database.SessionTaskSnapshot{}) - require.NoError(t, err) - page, err = store.ListSessionRecoveryCandidates(t.Context(), "", 100) - require.NoError(t, err) - require.Len(t, page, 1, "explicitly admitted recovery remains eligible for retry") - require.Equal(t, active.Id, page[0].Id) -} - func TestCrashedSessionRecoversAnOlderResumedTask(t *testing.T) { store, actors, session, older, version := crashedTurnFixture(t) older.Status.State = a2a.TaskStateInputRequired @@ -526,124 +492,35 @@ func TestExplicitRevertRetainsWaitCapturedByLaterTurn(t *testing.T) { require.Equal(t, a2a.TaskStateCompleted, retained.Status.State) } -func TestSessionRecoveryPagesPastUnavailableActors(t *testing.T) { - store, base, first, _, _ := crashedTurnFixture(t) - actors := &recoveryTestActors{lifecycleTestActors: base, revert: base.RevertActor, readErr: status.Error(codes.Unavailable, "control plane unavailable")} - workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) - ids := []string{first.Id} - for range 100 { - session, _, err := store.CreateSession(t.Context(), &apiv1.Session{ - Id: uuid.NewString(), Creator: first.Creator, Agent: first.Agent, - }, uuid.NewString()) - require.NoError(t, err) - session, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Create(t.Context(), session) - require.NoError(t, err) - message := a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("accepted turn")) - message.ContextID = session.ContextId - task := a2a.NewSubmittedTask(message, message) - task.Status.State = a2a.TaskStateWorking - hash := sha256.Sum256([]byte(task.ID)) - _, err = store.CreateRuntimeTask(t.Context(), session.Id, hash[:], task, "") - require.NoError(t, err) - base.actors[actorKey("team-a", substrate.ActorName(session.Id))].Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED - ids = append(ids, session.Id) - } - for _, id := range ids { - actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(id)) - require.NoError(t, err) - _, err = store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, id, actor.GetMetadata().GetUid()), database.SessionTaskSnapshot{}) - require.NoError(t, err) - } - slices.Sort(ids) - after, err := workflow.recoverSessionPage(t.Context(), "") - require.NoError(t, err) - require.Equal(t, ids[99], after, "failed Actor reads still advance the bounded page") - actors.readErr = nil - after, err = workflow.recoverSessionPage(t.Context(), after) - require.NoError(t, err) - require.Empty(t, after, "the short second page resets the cursor") - last, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(ids[100])) - require.NoError(t, err) - require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, last.GetStatus().GetState(), "the second page recovers the 101st candidate") - firstActor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(ids[0])) - require.NoError(t, err) - require.Equal(t, ateapipb.ActorState_ACTOR_STATE_CRASHED, firstActor.GetStatus().GetState(), "unavailable first-page Actors still need recovery") - after, err = workflow.recoverSessionPage(t.Context(), after) - require.NoError(t, err) - require.Equal(t, ids[99], after, "the wrapped scan revisits previously unavailable Actors") - after, err = workflow.recoverSessionPage(t.Context(), after) - require.NoError(t, err) - require.Empty(t, after, "an empty page resets the cursor") - for _, id := range ids { - actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(id)) - require.NoError(t, err) - require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, actor.GetStatus().GetState()) - } -} - -// blockedSuspendActors holds every idle suspension in runtime I/O until the -// test ends, the way a slow Substrate upload would. -type blockedSuspendActors struct { - *lifecycleTestActors - entered atomic.Int32 -} - -func (a *blockedSuspendActors) SuspendActor(ctx context.Context, _, _ string) (*ateapipb.Actor, error) { - a.entered.Add(1) - <-ctx.Done() - return nil, ctx.Err() -} - -// Start retries an admitted recovery on its own ticker. Every quiescence worker -// is blocked in an idle suspension for the whole test, so a scan that shared -// those workers would never run. -func TestStartRetriesAdmittedRecoveryWhileIdleWorkBlocks(t *testing.T) { - store, base, crashed, _, _ := crashedTurnFixture(t) - unavailable := &recoveryTestActors{lifecycleTestActors: base, revert: func(context.Context, string, string) (*ateapipb.Actor, error) { - return nil, status.Error(codes.Unavailable, "control plane unavailable") - }} - _, err := NewActorWorkflow(store, unavailable, make(chan struct{}, 1), time.Hour).Revert(t.Context(), crashed) - require.Error(t, err, "the first attempt leaves an admitted recovery for Start to retry") - - const idleSessions = 4 // one per quiescence worker - for range idleSessions { - idle, _, err := store.CreateSession(t.Context(), &apiv1.Session{Id: uuid.NewString(), Creator: crashed.Creator, Agent: crashed.Agent}, uuid.NewString()) - require.NoError(t, err) - idle, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Hour).Create(t.Context(), idle) - require.NoError(t, err) - request := stageQuiescenceTask(t, store.Client, idle, a2a.TaskStateCompleted) - require.NoError(t, store.SettleSessionTask(t.Context(), idle.Id, request.TaskId, request.Version)) - } - - actors := &blockedSuspendActors{lifecycleTestActors: base} - startQuiescenceWorker(t, NewActorWorkflow(store, actors, make(chan struct{}, 1), 20*time.Millisecond)) - require.Eventually(t, func() bool { return actors.entered.Load() == idleSessions }, 5*time.Second, 10*time.Millisecond) - require.Eventually(t, func() bool { - current, err := store.GetSessionByID(t.Context(), crashed.Id) - return err == nil && current.State == apiv1.RuntimeState_RUNTIME_STATE_READY - }, 5*time.Second, 10*time.Millisecond) - actor, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(crashed.Id)) - require.NoError(t, err) - require.Equal(t, ateapipb.ActorState_ACTOR_STATE_SUSPENDED, actor.GetStatus().GetState()) - require.EqualValues(t, idleSessions, actors.entered.Load(), "idle work stayed blocked while recovery completed") -} - // A recovery whose Actor cannot be read has issued nothing, so it must not take -// a claim that would make Delete refuse the Session. -func TestDeleteSupersedesUnissuedCrashRecovery(t *testing.T) { +// a claim that would make Delete refuse the Session. A recovery whose Actor is +// gone can never finish, so it ends even after an earlier attempt claimed it. +func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { for _, test := range []struct { name string + claimed bool gone bool readErr error }{ {name: "actor read fails", readErr: status.Error(codes.Unavailable, "control plane unavailable")}, {name: "actor is gone", gone: true}, + {name: "actor is gone after an issued attempt", claimed: true, gone: true}, } { t.Run(test.name, func(t *testing.T) { store, base, session, _, _ := crashedTurnFixture(t) key := actorKey("team-a", substrate.ActorName(session.Id)) - _, err := store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, session.Id, base.actors[key].GetMetadata().GetUid()), database.SessionTaskSnapshot{}) - require.NoError(t, err) + if test.claimed { + unavailable := &recoveryTestActors{lifecycleTestActors: base, revert: func(context.Context, string, string) (*ateapipb.Actor, error) { + return nil, status.Error(codes.Unavailable, "response unavailable") + }} + _, err := NewActorWorkflow(store, unavailable, make(chan struct{}, 1), time.Minute).Revert(t.Context(), session) + require.Error(t, err) + _, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), session, false) + require.ErrorIs(t, err, database.ErrConflict, "an issued revert blocks Delete while it might still succeed") + } else { + _, err := store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, session.Id, base.actors[key].GetMetadata().GetUid()), database.SessionTaskSnapshot{}) + require.NoError(t, err) + } if test.gone { delete(base.actors, key) } @@ -653,25 +530,28 @@ func TestDeleteSupersedesUnissuedCrashRecovery(t *testing.T) { return nil, nil } workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) - admitted, err := store.GetSessionByID(t.Context(), session.Id) - require.NoError(t, err) - _, err = workflow.Revert(t.Context(), admitted) - require.Error(t, err) - _, err = workflow.recoverSessionPage(t.Context(), "") - require.NoError(t, err) + for range 2 { + admitted, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + _, err = workflow.Revert(t.Context(), admitted) + require.Error(t, err) + require.Equal(t, test.gone, errors.Is(err, errRuntimeUnrecoverable)) + } pending, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, pending.State) - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, pending.Operation, "a failed read keeps the recovery for the next scan") + if test.gone { + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, pending.Operation, "a recovery that cannot finish is abandoned") + require.Equal(t, database.SessionRuntimeUnrecoverable, pending.Failure.GetReason()) + } else { + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, pending.Operation, "a failed read keeps the recovery for a retry") + } deleted, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), pending, false) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_DELETED, deleted.State) _, err = base.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) require.Equal(t, codes.NotFound, status.Code(err)) - page, err := store.ListSessionRecoveryCandidates(t.Context(), "", 100) - require.NoError(t, err) - require.Empty(t, page) }) } } diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index acd45136a3..3bc55f1baf 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -15,9 +15,10 @@ import ( ) type workflowStore interface { - ListSessionRecoveryCandidates(context.Context, string, int) ([]*apiv1.Session, error) GetSessionGeneration(context.Context, string) (*database.SessionGeneration, error) BeginCrashedSessionRecovery(context.Context, *database.SessionGeneration, database.SessionTaskSnapshot) (*database.SessionOperation, error) + GetSessionRecoverySnapshot(context.Context, string, uuid.UUID) (string, error) + AbandonSessionRecovery(context.Context, string, uuid.UUID, string) (*apiv1.Session, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) FinishSessionQuiescence(context.Context, *database.SessionQuiescence, *database.SessionTaskSnapshot) error @@ -162,8 +163,12 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session } var transition *substrate.ActorTransition switch { - case database.CrashRecoveryAdmitted(operation.Instance): - transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding) + case kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT: + var pinned string + if pinned, err = w.store.GetSessionRecoverySnapshot(ctx, sessionID, operation.ID); err != nil { + return w.failPreparation(ctx, operation, fmt.Errorf("load admitted recovery snapshot: %w", err)) + } + transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding, pinned) case operation.ExecutorID != uuid.Nil: transition, err = substrate.PrepareActorRetry(ctx, w.actors, binding, kind, creation) default: @@ -175,6 +180,9 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session if uid := transition.ActorUID(); uid != "" && kind != apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { if _, err := w.store.GetSessionForRuntime(ctx, session.Id, uid); err != nil { + if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT && errors.Is(err, database.ErrNotFound) { + err = fmt.Errorf("runtime Actor was replaced: %w", substrate.ErrActorUnrecoverable) + } return w.failPreparation(ctx, operation, fmt.Errorf("verify runtime actor UID: %w", err)) } } @@ -218,15 +226,12 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session // the same generation instead. Completion returns current state; supersession // returns a conflict. A local preparation error cannot clear a newer operation. func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database.SessionOperation, cause error) (*apiv1.Session, error) { + if database.CrashRecoveryAdmitted(admitted.Instance) { + return w.failRecoveryPreparation(ctx, admitted, cause) + } if admitted.ExecutorID != uuid.Nil { return nil, cause // An earlier attempt may have issued work; retain its intent. } - if database.CrashRecoveryAdmitted(admitted.Instance) { - // Admission already failed the interrupted turn, so clearing the operation - // would leave a FAILED Session that nothing retries. Keep the intent for the - // next scan. No claim was taken, so Delete can still supersede it. - return nil, cause - } finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() _, err := w.store.FinishSessionOperation(finishCtx, admitted.Instance.Id, admitted.ID, uuid.Nil, "", "", "lifecycle preparation failed") @@ -240,6 +245,26 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. return nil, errors.Join(cause, err) } +// failRecoveryPreparation keeps an admitted REVERT unless its Actor can no +// longer be restored. Admission already failed the interrupted turn, so clearing +// the operation as a preparation failure would leave a FAILED Session that a +// retried RevertSession could not continue. Without a claim, Delete can still +// supersede the kept intent. An Actor that is missing, replaced, or holding +// another snapshot shows that no revert of the recorded runtime can still be in +// progress, so the store ends the operation even after a claim. Otherwise the +// claim would block Delete forever. +func (w *ActorWorkflow) failRecoveryPreparation(ctx context.Context, admitted *database.SessionOperation, cause error) (*apiv1.Session, error) { + if !errors.Is(cause, substrate.ErrActorUnrecoverable) { + return nil, cause + } + finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if _, err := w.store.AbandonSessionRecovery(finishCtx, admitted.Instance.Id, admitted.ID, cause.Error()); err != nil { + return nil, errors.Join(cause, err) + } + return nil, fmt.Errorf("%w: %w", cause, errRuntimeUnrecoverable) +} + func operationOutcome(operation *database.SessionOperation) (*apiv1.Session, error) { if operation.Instance.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { return operation.Instance, nil diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index ff243a77e3..3414461be4 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -2,6 +2,7 @@ package substrate import ( "context" + "errors" "fmt" ateapipb "go.ate.dev/substrate/api/ateapi/v1" @@ -52,11 +53,14 @@ type ActorTransition struct { operation apiv1.RuntimeOperation actor *ateapipb.Actor creation *ActorCreation - // revert restores a crashed Actor to its last durable snapshot. The - // operation stays RESUME because that is what the Session records. - revert bool } +// ErrActorUnrecoverable reports that a crash recovery's Actor can no longer be +// restored to the snapshot its admission pinned: the Actor is missing, was +// replaced, or now reports a different durable snapshot. No retry changes that, +// and no revert of the recorded Actor can still be in progress. +var ErrActorUnrecoverable = errors.New("actor cannot be restored to the admitted snapshot") + // ActorUID returns the observed runtime identity, including after creation. func (t *ActorTransition) ActorUID() string { return t.actor.GetMetadata().GetUid() @@ -123,23 +127,32 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding } // PrepareActorRevert performs the read-only checks for crash recovery before -// the store's durable claim. Reading first keeps a failed or missing Actor from +// the store's durable claim. Reading first keeps a failed Actor read from // leaving a claim behind, so Delete can still supersede the unissued recovery. +// snapshotURI is the durable snapshot admission pinned ("" for none). Admission +// already settled the interrupted turn against it, so the revert must restore +// exactly that snapshot rather than whatever Substrate holds when it runs. // A SUSPENDED Actor means an earlier revert completed and its response was lost. -func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding) (*ActorTransition, error) { +func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding, snapshotURI string) (*ActorTransition, error) { actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) + if status.Code(err) == codes.NotFound { + return nil, fmt.Errorf("actor %s/%s is missing: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) + } if err != nil { return nil, fmt.Errorf("get Actor %s/%s: %w", binding.Atespace, binding.Name, err) } if !binding.Matches(actor) { - return nil, fmt.Errorf("actor %s/%s identity or template changed", binding.Atespace, binding.Name) + return nil, fmt.Errorf("actor %s/%s identity or template changed: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) } switch actor.GetStatus().GetState() { case ateapipb.ActorState_ACTOR_STATE_CRASHED, ateapipb.ActorState_ACTOR_STATE_REVERTING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED: default: return nil, fmt.Errorf("crashed Actor %s/%s has unexpected state %s", binding.Atespace, binding.Name, actor.GetStatus().GetState()) } - return &ActorTransition{binding: binding, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, actor: actor, revert: true}, nil + if SnapshotURI(LatestDurableActorSnapshot(actor)) != snapshotURI { + return nil, fmt.Errorf("actor %s/%s durable snapshot changed after recovery was admitted: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) + } + return &ActorTransition{binding: binding, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, actor: actor}, nil } // ApplyActorTransition requires durable execution authority. Errors retain the @@ -149,9 +162,6 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return err } binding, actor := transition.binding, transition.actor - if transition.revert { - return transition.applyRevert(ctx, actors) - } var err error switch transition.operation { case apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE: @@ -205,6 +215,8 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio err = fmt.Errorf("suspended Actor requires a durable VOLUMES snapshot") } } + case apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT: + err = transition.applyRevert(ctx, actors) case apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE: if actor != nil { // An interrupted resume must not make deletion depend on a successful @@ -221,9 +233,10 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -// applyRevert must not replace the Actor or roll it past the snapshot observed -// before the claim. A changed UID or snapshot means something other than this -// recovery acted on the runtime, so the Session must not adopt the result. +// applyRevert must not replace the Actor or restore anything but the snapshot +// PrepareActorRevert matched against admission's pin. A changed UID or snapshot +// means something other than this recovery acted on the runtime, so the Session +// must not adopt the result. func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClient) error { binding, actor := t.binding, t.actor before := proto.CloneOf(LatestDurableActorSnapshot(actor)) @@ -235,12 +248,12 @@ func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClien if state := actor.GetStatus().GetState(); state == ateapipb.ActorState_ACTOR_STATE_CRASHED || state == ateapipb.ActorState_ACTOR_STATE_REVERTING { reverted, err := actors.RevertActor(ctx, binding.Atespace, binding.Name) if err != nil { - return fmt.Errorf("revert crashed Actor %s/%s: %w", binding.Atespace, binding.Name, err) + return err } actor = reverted } if t.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) != nil || !proto.Equal(before, LatestActorSnapshot(actor)) { - return fmt.Errorf("revert Actor %s/%s returned unexpected identity or state", binding.Atespace, binding.Name) + return fmt.Errorf("revert returned unexpected identity or state") } return nil } diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 6d700e51f8..086f321e5e 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -276,7 +276,9 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { state ateapipb.ActorState revertUID string revertSnapshot *ateapipb.Snapshot + pinned *string wantPrepareErr bool + unrecoverable bool wantApplyErr bool wantCalls []string }{ @@ -284,7 +286,8 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { {name: "revert in progress", state: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantCalls: []string{"revert"}}, {name: "earlier revert response lost", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantPrepareErr: true}, - {name: "actor gone", missing: true, wantPrepareErr: true}, + {name: "actor gone", missing: true, wantPrepareErr: true, unrecoverable: true}, + {name: "snapshot changed after admission", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, pinned: new("s3://snapshots/admitted"), wantPrepareErr: true, unrecoverable: true}, {name: "replaced actor", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertUID: "another-actor", wantApplyErr: true, wantCalls: []string{"revert"}}, {name: "different snapshot", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertSnapshot: durable(1, "s3://snapshots/older"), wantApplyErr: true, wantCalls: []string{"revert"}}, } { @@ -301,9 +304,14 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { } } binding := ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"} - transition, err := PrepareActorRevert(t.Context(), actors, binding) + pinned := "s3://snapshots/latest" + if test.pinned != nil { + pinned = *test.pinned + } + transition, err := PrepareActorRevert(t.Context(), actors, binding, pinned) if test.wantPrepareErr { require.Error(t, err) + require.Equal(t, test.unrecoverable, errors.Is(err, ErrActorUnrecoverable)) require.Empty(t, actors.calls, "a failed read issues nothing") return } diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 3f73d33f8d..1fd98e6c67 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -136,7 +136,7 @@ CREATE TABLE runtime_instance ( request_id TEXT NOT NULL, prepared_revision TEXT REFERENCES runtime_revision(revision) ON DELETE RESTRICT, state TEXT NOT NULL CHECK (state IN ('RUNTIME_STATE_CREATING', 'RUNTIME_STATE_READY', 'RUNTIME_STATE_SUSPENDED', 'RUNTIME_STATE_FAILED', 'RUNTIME_STATE_DELETING', 'RUNTIME_STATE_DELETED')), - operation TEXT NOT NULL DEFAULT 'RUNTIME_OPERATION_NONE' CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE')), + operation TEXT NOT NULL DEFAULT 'RUNTIME_OPERATION_NONE' CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE', 'RUNTIME_OPERATION_REVERT')), operation_id UUID, executor_id UUID, executor_expires_at TIMESTAMPTZ, @@ -164,6 +164,13 @@ CREATE TABLE session ( dispatch_id UUID, dispatch_expires_at TIMESTAMPTZ, CHECK ((dispatch_id IS NULL) = (dispatch_expires_at IS NULL)), + -- The durable snapshot a crash recovery admitted, for the REVERT operation + -- named by recovery_operation_id. It is valid only while that operation is + -- the Session's current one, so a later operation needs no cleanup. The URI + -- is empty when the crashed Actor had no durable snapshot. + recovery_operation_id UUID, + recovery_snapshot_uri TEXT, + CHECK ((recovery_operation_id IS NULL) = (recovery_snapshot_uri IS NULL)), history_id UUID NOT NULL, CONSTRAINT session_context_binding_fkey FOREIGN KEY (history_id, context_id) REFERENCES a2a_context(id, context_id) ON DELETE RESTRICT, diff --git a/go/core/test/e2e/session_revert_test.go b/go/core/test/e2e/session_revert_test.go index c6a59464c3..597dee5274 100644 --- a/go/core/test/e2e/session_revert_test.go +++ b/go/core/test/e2e/session_revert_test.go @@ -131,10 +131,11 @@ func TestSessionRevertCrashedRuntime(t *testing.T) { reverted, err := fixture.sessions.RevertSession(fixture.ctx, request) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, reverted.Session.State) - // A client whose success response was lost retries and must learn that - // the runtime is healthy, not that it cannot be recovered. - _, err = fixture.sessions.RevertSession(fixture.ctx, request) - require.Equal(t, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", revertRefusalReason(t, err), "a recovered runtime has nothing to revert") + // A client whose success response was lost retries and gets the + // recovered Session back, as Create and Delete retries do. + retried, err := fixture.sessions.RevertSession(fixture.ctx, request) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, retried.Session.State) recovered, err := findSubstrateActor(fixture.ctx, fixture.system, namespace, name) require.NoError(t, err) require.Equal(t, running.GetMetadata().GetUid(), recovered.GetMetadata().GetUid()) diff --git a/proto/kagent/api/v1/runtime.proto b/proto/kagent/api/v1/runtime.proto index 1b22ea1feb..312777d4ff 100644 --- a/proto/kagent/api/v1/runtime.proto +++ b/proto/kagent/api/v1/runtime.proto @@ -25,4 +25,7 @@ enum RuntimeOperation { RUNTIME_OPERATION_DELETE = 4; // No lifecycle operation is active. RUNTIME_OPERATION_NONE = 5; + // Restores a crashed runtime to its last durable snapshot. Only a confirmed + // crash admits it; the Session stays FAILED until it finishes. + RUNTIME_OPERATION_REVERT = 6; } diff --git a/ui/src/api/domain/agentInstances.ts b/ui/src/api/domain/agentInstances.ts index 68568298fe..6c85dde927 100644 --- a/ui/src/api/domain/agentInstances.ts +++ b/ui/src/api/domain/agentInstances.ts @@ -56,6 +56,7 @@ export type AgentInstanceOperation = | "suspend" | "resume" | "delete" + | "revert" | "unknown"; /** diff --git a/ui/src/api/grpc/operations.ts b/ui/src/api/grpc/operations.ts index 202ed7cdb4..2ae8b2d8bc 100644 --- a/ui/src/api/grpc/operations.ts +++ b/ui/src/api/grpc/operations.ts @@ -565,6 +565,7 @@ const INSTANCE_OPERATION_BY_ENUM: Record< [RuntimeOperation.SUSPEND]: "suspend", [RuntimeOperation.RESUME]: "resume", [RuntimeOperation.DELETE]: "delete", + [RuntimeOperation.REVERT]: "revert", }; /** diff --git a/ui/src/components/agent-instances/instanceLabels.ts b/ui/src/components/agent-instances/instanceLabels.ts index 200e906f03..c1bcb306e8 100644 --- a/ui/src/components/agent-instances/instanceLabels.ts +++ b/ui/src/components/agent-instances/instanceLabels.ts @@ -109,6 +109,7 @@ const OPERATIONS: Record = { suspend: { label: "Suspending", tone: "processing", inProgress: true }, resume: { label: "Resuming", tone: "processing", inProgress: true }, delete: { label: "Deleting", tone: "warning", inProgress: true }, + revert: { label: "Reverting", tone: "warning", inProgress: true }, unknown: { label: "Unrecognised operation", tone: "warning", inProgress: true }, }; diff --git a/ui/src/components/chat/lifecycleReading.ts b/ui/src/components/chat/lifecycleReading.ts index 35f835f8ae..6e38992d76 100644 --- a/ui/src/components/chat/lifecycleReading.ts +++ b/ui/src/components/chat/lifecycleReading.ts @@ -60,6 +60,15 @@ export function lifecycleReading( }; } + if (instance.operation === "revert") { + return { + active: "resuming", + summary: "Restoring this conversation", + source: + "The runtime crashed; the controller is restoring it to its last snapshot.", + }; + } + if (instance.operation === "suspend") { return { active: "suspending", diff --git a/ui/src/generated/kagent/api/v1/runtime_pb.ts b/ui/src/generated/kagent/api/v1/runtime_pb.ts index 5177311c97..901a61509d 100644 --- a/ui/src/generated/kagent/api/v1/runtime_pb.ts +++ b/ui/src/generated/kagent/api/v1/runtime_pb.ts @@ -9,7 +9,7 @@ import { enumDesc, fileDesc } from "@bufbuild/protobuf/codegenv2"; * Describes the file kagent/api/v1/runtime.proto. */ export const file_kagent_api_v1_runtime: GenFile = /*@__PURE__*/ - fileDesc("ChtrYWdlbnQvYXBpL3YxL3J1bnRpbWUucHJvdG8SDWthZ2VudC5hcGkudjEq0AEKDFJ1bnRpbWVTdGF0ZRIdChlSVU5USU1FX1NUQVRFX1VOU1BFQ0lGSUVEEAASGgoWUlVOVElNRV9TVEFURV9DUkVBVElORxABEhcKE1JVTlRJTUVfU1RBVEVfUkVBRFkQAhIbChdSVU5USU1FX1NUQVRFX1NVU1BFTkRFRBADEhgKFFJVTlRJTUVfU1RBVEVfRkFJTEVEEAQSGgoWUlVOVElNRV9TVEFURV9ERUxFVElORxAFEhkKFVJVTlRJTUVfU1RBVEVfREVMRVRFRBAGKsoBChBSdW50aW1lT3BlcmF0aW9uEiEKHVJVTlRJTUVfT1BFUkFUSU9OX1VOU1BFQ0lGSUVEEAASHAoYUlVOVElNRV9PUEVSQVRJT05fQ1JFQVRFEAESHQoZUlVOVElNRV9PUEVSQVRJT05fU1VTUEVORBACEhwKGFJVTlRJTUVfT1BFUkFUSU9OX1JFU1VNRRADEhwKGFJVTlRJTUVfT1BFUkFUSU9OX0RFTEVURRAEEhoKFlJVTlRJTUVfT1BFUkFUSU9OX05PTkUQBUI9WjtnaXRodWIuY29tL2thZ2VudC1kZXYva2FnZW50L2dvL2FwaS9nZW4va2FnZW50L2FwaS92MTthcGl2MWIGcHJvdG8z"); + fileDesc("ChtrYWdlbnQvYXBpL3YxL3J1bnRpbWUucHJvdG8SDWthZ2VudC5hcGkudjEq0AEKDFJ1bnRpbWVTdGF0ZRIdChlSVU5USU1FX1NUQVRFX1VOU1BFQ0lGSUVEEAASGgoWUlVOVElNRV9TVEFURV9DUkVBVElORxABEhcKE1JVTlRJTUVfU1RBVEVfUkVBRFkQAhIbChdSVU5USU1FX1NUQVRFX1NVU1BFTkRFRBADEhgKFFJVTlRJTUVfU1RBVEVfRkFJTEVEEAQSGgoWUlVOVElNRV9TVEFURV9ERUxFVElORxAFEhkKFVJVTlRJTUVfU1RBVEVfREVMRVRFRBAGKugBChBSdW50aW1lT3BlcmF0aW9uEiEKHVJVTlRJTUVfT1BFUkFUSU9OX1VOU1BFQ0lGSUVEEAASHAoYUlVOVElNRV9PUEVSQVRJT05fQ1JFQVRFEAESHQoZUlVOVElNRV9PUEVSQVRJT05fU1VTUEVORBACEhwKGFJVTlRJTUVfT1BFUkFUSU9OX1JFU1VNRRADEhwKGFJVTlRJTUVfT1BFUkFUSU9OX0RFTEVURRAEEhoKFlJVTlRJTUVfT1BFUkFUSU9OX05PTkUQBRIcChhSVU5USU1FX09QRVJBVElPTl9SRVZFUlQQBkI9WjtnaXRodWIuY29tL2thZ2VudC1kZXYva2FnZW50L2dvL2FwaS9nZW4va2FnZW50L2FwaS92MTthcGl2MWIGcHJvdG8z"); /** * RuntimeState is shared by Sessions and Sandboxes. Its names are also the @@ -97,6 +97,14 @@ export enum RuntimeOperation { * @generated from enum value: RUNTIME_OPERATION_NONE = 5; */ NONE = 5, + + /** + * Restores a crashed runtime to its last durable snapshot. Only a confirmed + * crash admits it; the Session stays FAILED until it finishes. + * + * @generated from enum value: RUNTIME_OPERATION_REVERT = 6; + */ + REVERT = 6, } /** diff --git a/ui/src/mocks/transport.ts b/ui/src/mocks/transport.ts index ea5c593157..a944067b96 100644 --- a/ui/src/mocks/transport.ts +++ b/ui/src/mocks/transport.ts @@ -635,6 +635,7 @@ const PB_OPERATION_BY_NAME: Record = { suspend: RuntimeOperation.SUSPEND, resume: RuntimeOperation.RESUME, delete: RuntimeOperation.DELETE, + revert: RuntimeOperation.REVERT, unknown: RuntimeOperation.NONE, }; From c4eeb27b4cb6ea7c0856ac0a57811afa6bab59d8 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 14:59:58 -0800 Subject: [PATCH 07/15] fix(migrations): add the REVERT operation in a new migration The previous commit edited 000001_initial.sql in place, which the required migration-immutability check rejects: merged migrations are immutable. Restore 000001 and add 000002_session_revert_operation.sql, which widens the runtime_instance operation CHECK to accept RUNTIME_OPERATION_REVERT and adds session.recovery_operation_id and recovery_snapshot_uri. Down refuses while a recovery is in flight rather than rewriting its operation. The database README still told contributors to fold changes into migration 1, which contradicts the required check; it now says to add a new migration. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- go/core/internal/database/README.md | 7 ++-- .../pkg/migrations/core/000001_initial.sql | 9 +----- .../core/000003_session_revert_operation.sql | 32 +++++++++++++++++++ go/core/pkg/migrations/runner_test.go | 5 ++- 4 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 go/core/pkg/migrations/core/000003_session_revert_operation.sql diff --git a/go/core/internal/database/README.md b/go/core/internal/database/README.md index a9fa246c34..abc4b18cf8 100644 --- a/go/core/internal/database/README.md +++ b/go/core/internal/database/README.md @@ -55,9 +55,10 @@ The repository [style guide](../../../../STYLE.md) also applies. - Do not use `uuid.MustParse` on runtime inputs. Pass validated values as SQL parameters; use checked parsing when Go needs a UUID for comparison. Store calls must return errors instead of panicking on malformed IDs. -- [Migration 1](../../pkg/migrations/core/000001_initial.sql) is currently the - unreleased baseline: fold approved schema changes into it and recreate local - databases. After release, use new migrations. Keep both Up and Down valid. +- Merged migrations are immutable, including + [migration 1](../../pkg/migrations/core/000001_initial.sql); the required + `migration-immutability` check rejects any change to them. Add each schema + change as a new migration. Keep both Up and Down valid. ## Atomicity and performance diff --git a/go/core/pkg/migrations/core/000001_initial.sql b/go/core/pkg/migrations/core/000001_initial.sql index 1fd98e6c67..3f73d33f8d 100644 --- a/go/core/pkg/migrations/core/000001_initial.sql +++ b/go/core/pkg/migrations/core/000001_initial.sql @@ -136,7 +136,7 @@ CREATE TABLE runtime_instance ( request_id TEXT NOT NULL, prepared_revision TEXT REFERENCES runtime_revision(revision) ON DELETE RESTRICT, state TEXT NOT NULL CHECK (state IN ('RUNTIME_STATE_CREATING', 'RUNTIME_STATE_READY', 'RUNTIME_STATE_SUSPENDED', 'RUNTIME_STATE_FAILED', 'RUNTIME_STATE_DELETING', 'RUNTIME_STATE_DELETED')), - operation TEXT NOT NULL DEFAULT 'RUNTIME_OPERATION_NONE' CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE', 'RUNTIME_OPERATION_REVERT')), + operation TEXT NOT NULL DEFAULT 'RUNTIME_OPERATION_NONE' CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE')), operation_id UUID, executor_id UUID, executor_expires_at TIMESTAMPTZ, @@ -164,13 +164,6 @@ CREATE TABLE session ( dispatch_id UUID, dispatch_expires_at TIMESTAMPTZ, CHECK ((dispatch_id IS NULL) = (dispatch_expires_at IS NULL)), - -- The durable snapshot a crash recovery admitted, for the REVERT operation - -- named by recovery_operation_id. It is valid only while that operation is - -- the Session's current one, so a later operation needs no cleanup. The URI - -- is empty when the crashed Actor had no durable snapshot. - recovery_operation_id UUID, - recovery_snapshot_uri TEXT, - CHECK ((recovery_operation_id IS NULL) = (recovery_snapshot_uri IS NULL)), history_id UUID NOT NULL, CONSTRAINT session_context_binding_fkey FOREIGN KEY (history_id, context_id) REFERENCES a2a_context(id, context_id) ON DELETE RESTRICT, diff --git a/go/core/pkg/migrations/core/000003_session_revert_operation.sql b/go/core/pkg/migrations/core/000003_session_revert_operation.sql new file mode 100644 index 0000000000..45e10b3c73 --- /dev/null +++ b/go/core/pkg/migrations/core/000003_session_revert_operation.sql @@ -0,0 +1,32 @@ +-- +goose Up + +-- Crash recovery is its own lifecycle operation, started only after Substrate +-- confirms that the recorded Actor crashed. +ALTER TABLE runtime_instance DROP CONSTRAINT runtime_instance_operation_check; +ALTER TABLE runtime_instance ADD CONSTRAINT runtime_instance_operation_check + CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', + 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE', 'RUNTIME_OPERATION_REVERT')); + +-- The durable snapshot a crash recovery admitted, for the REVERT operation +-- named by recovery_operation_id. It is valid only while that operation is the +-- Session's current one, so a later operation needs no cleanup. The URI is +-- empty when the crashed Actor had no durable snapshot. +ALTER TABLE session + ADD COLUMN recovery_operation_id UUID, + ADD COLUMN recovery_snapshot_uri TEXT, + ADD CONSTRAINT session_recovery_snapshot_check + CHECK ((recovery_operation_id IS NULL) = (recovery_snapshot_uri IS NULL)); + +-- +goose Down + +ALTER TABLE session + DROP CONSTRAINT session_recovery_snapshot_check, + DROP COLUMN recovery_snapshot_uri, + DROP COLUMN recovery_operation_id; + +-- Fails while any recovery is still in flight, rather than rewriting its +-- operation into one the older binary would execute differently. +ALTER TABLE runtime_instance DROP CONSTRAINT runtime_instance_operation_check; +ALTER TABLE runtime_instance ADD CONSTRAINT runtime_instance_operation_check + CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', + 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE')); diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 8c15d7f17a..6582bcdba6 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -320,7 +320,7 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { for _, source := range sources { expected := []int64{0, 1} if source.Name == "core" { - expected = append(expected, 2) + expected = append(expected, 2, 3) } if versions := testVersions(t, dsn, source.TrackingTable); !slices.Equal(versions, expected) { t.Fatalf("%s versions = %v, want %v", source.Name, versions, expected) @@ -337,6 +337,9 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } + if !testColumnExists(t, dsn, "session", "recovery_snapshot_uri") { + t.Fatal("session crash recovery snapshot pin is missing") + } if !testTableExists(t, dsn, "agent_definition") || testTableExists(t, dsn, "agent_template_harness_pair") { t.Fatal("initial migration must create explicit Agent definitions without legacy pairs") } From eff21c10daa7da87de4e7aed625f7c1e5b442163 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 15:10:31 -0800 Subject: [PATCH 08/15] fix(session): abandon a claimed revert when a replacement is running PrepareActorRevert checked the Actor's state before its identity, and the recorded UID only after preparation. A same-name replacement in any state other than CRASHED, REVERTING, or SUSPENDED therefore failed with a transient error. When an earlier attempt had claimed the revert, the operation was never abandoned and Delete stayed refused, which is the stuck Session review point 2 describes. The recovery pin now carries the recorded Actor UID with the snapshot URI, and PrepareActorRevert checks identity first, so any replacement is unrecoverable. Execute selects the revert through CrashRecoveryAdmitted, the one shared check. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- go/core/internal/database/session_recovery.go | 27 ++++++++++------- .../database/session_recovery_test.go | 7 +++-- .../internal/service/session/recovery_test.go | 30 +++++++++++++++---- go/core/internal/service/session/workflow.go | 12 ++++---- go/core/internal/substrate/lifecycle.go | 15 ++++++---- go/core/internal/substrate/lifecycle_test.go | 8 ++++- 6 files changed, 67 insertions(+), 32 deletions(-) diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 5e0806a23a..6bfde03fd3 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -161,19 +161,26 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se return operation, err } -// GetSessionRecoverySnapshot returns the durable snapshot URI pinned when the -// Session's REVERT operation was admitted. The URI is empty when the crashed -// Actor had no durable snapshot. Any other operation returns ErrNotFound. -func (c *Client) GetSessionRecoverySnapshot(ctx context.Context, sessionID string, operationID uuid.UUID) (string, error) { - uri, err := queryOne(ctx, c.db, ` - SELECT s.recovery_snapshot_uri FROM session s JOIN runtime_instance r USING (id) - WHERE s.id = $1 AND s.recovery_operation_id = $2 +// SessionRecoveryPin is what an admitted REVERT must restore: the recorded Actor +// and the durable snapshot admission settled waits against. SnapshotURI is empty +// when the crashed Actor had no durable snapshot. +type SessionRecoveryPin struct { + ActorUID string + SnapshotURI string +} + +// GetSessionRecoveryPin returns the pin of the Session's current REVERT +// operation. Any other operation returns ErrNotFound. +func (c *Client) GetSessionRecoveryPin(ctx context.Context, sessionID string, operationID uuid.UUID) (SessionRecoveryPin, error) { + pin, err := queryOne(ctx, c.db, ` + SELECT s.actor_uid, s.recovery_snapshot_uri AS snapshot_uri FROM session s JOIN runtime_instance r USING (id) + WHERE s.id = $1 AND s.recovery_operation_id = $2 AND s.actor_uid IS NOT NULL AND r.operation_id = $2 AND r.operation = 'RUNTIME_OPERATION_REVERT' - `, pgx.RowTo[string], sessionID, operationID) + `, pgx.RowToStructByName[SessionRecoveryPin], sessionID, operationID) if err != nil { - return "", fmt.Errorf("read Session %s recovery snapshot: %w", sessionID, notFoundOr(err)) + return SessionRecoveryPin{}, fmt.Errorf("read Session %s recovery pin: %w", sessionID, notFoundOr(err)) } - return uri, nil + return pin, nil } // AbandonSessionRecovery ends an admitted REVERT whose Actor can no longer be diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index 0c443a17d6..ba449d3034 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -52,7 +52,7 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { } before, _, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) - _, err = client.GetSessionRecoverySnapshot(t.Context(), session.Id, uuid.New()) + _, err = client.GetSessionRecoveryPin(t.Context(), session.Id, uuid.New()) require.ErrorIs(t, err, ErrNotFound) operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), snapshot) @@ -85,9 +85,10 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { require.ErrorIs(t, client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "next"), ErrConflict) _, _, err = client.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID)}, session.Creator, uuid.NewString()) require.ErrorIs(t, err, ErrConflict) - pinned, err := client.GetSessionRecoverySnapshot(t.Context(), session.Id, operation.ID) + pin, err := client.GetSessionRecoveryPin(t.Context(), session.Id, operation.ID) require.NoError(t, err) - require.Equal(t, snapshot.URI, pinned, "the operation pins the snapshot that settlement judged waits against") + require.Equal(t, SessionRecoveryPin{ActorUID: "actor-" + session.Id, SnapshotURI: snapshot.URI}, pin, + "the operation pins the Actor and the snapshot that settlement judged waits against") }) } } diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 09aa6f6915..4cb7947e06 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -497,14 +497,16 @@ func TestExplicitRevertRetainsWaitCapturedByLaterTurn(t *testing.T) { // gone can never finish, so it ends even after an earlier attempt claimed it. func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { for _, test := range []struct { - name string - claimed bool - gone bool - readErr error + name string + claimed bool + gone bool + replaced bool + readErr error }{ {name: "actor read fails", readErr: status.Error(codes.Unavailable, "control plane unavailable")}, {name: "actor is gone", gone: true}, {name: "actor is gone after an issued attempt", claimed: true, gone: true}, + {name: "running replacement after an issued attempt", claimed: true, replaced: true}, } { t.Run(test.name, func(t *testing.T) { store, base, session, _, _ := crashedTurnFixture(t) @@ -524,6 +526,11 @@ func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { if test.gone { delete(base.actors, key) } + if test.replaced { + base.actors[key].Metadata.Uid = "replacement-uid" + base.actors[key].Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING + } + unrecoverable := test.gone || test.replaced actors := &recoveryTestActors{lifecycleTestActors: base, readErr: test.readErr} actors.revert = func(context.Context, string, string) (*ateapipb.Actor, error) { t.Fatal("an unread Actor was reverted") @@ -535,19 +542,30 @@ func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { require.NoError(t, err) _, err = workflow.Revert(t.Context(), admitted) require.Error(t, err) - require.Equal(t, test.gone, errors.Is(err, errRuntimeUnrecoverable)) + require.Equal(t, unrecoverable, errors.Is(err, errRuntimeUnrecoverable)) } pending, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, pending.State) - if test.gone { + if unrecoverable { require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, pending.Operation, "a recovery that cannot finish is abandoned") require.Equal(t, database.SessionRuntimeUnrecoverable, pending.Failure.GetReason()) } else { require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, pending.Operation, "a failed read keeps the recovery for a retry") } deleted, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), pending, false) + if test.replaced { + // Delete is admitted, but never removes a same-name replacement. + require.NotErrorIs(t, err, database.ErrConflict) + deleting, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_DELETING, deleting.State, "Delete closed admission") + replacement, err := base.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) + require.NoError(t, err) + require.Equal(t, "replacement-uid", replacement.GetMetadata().GetUid()) + return + } require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_DELETED, deleted.State) _, err = base.GetActor(t.Context(), "team-a", substrate.ActorName(session.Id)) diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 3bc55f1baf..7fdf67b91d 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -17,7 +17,7 @@ import ( type workflowStore interface { GetSessionGeneration(context.Context, string) (*database.SessionGeneration, error) BeginCrashedSessionRecovery(context.Context, *database.SessionGeneration, database.SessionTaskSnapshot) (*database.SessionOperation, error) - GetSessionRecoverySnapshot(context.Context, string, uuid.UUID) (string, error) + GetSessionRecoveryPin(context.Context, string, uuid.UUID) (database.SessionRecoveryPin, error) AbandonSessionRecovery(context.Context, string, uuid.UUID, string) (*apiv1.Session, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) @@ -163,12 +163,12 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session } var transition *substrate.ActorTransition switch { - case kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT: - var pinned string - if pinned, err = w.store.GetSessionRecoverySnapshot(ctx, sessionID, operation.ID); err != nil { - return w.failPreparation(ctx, operation, fmt.Errorf("load admitted recovery snapshot: %w", err)) + case database.CrashRecoveryAdmitted(session): + var pin database.SessionRecoveryPin + if pin, err = w.store.GetSessionRecoveryPin(ctx, sessionID, operation.ID); err != nil { + return w.failPreparation(ctx, operation, fmt.Errorf("load admitted recovery pin: %w", err)) } - transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding, pinned) + transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding, pin.ActorUID, pin.SnapshotURI) case operation.ExecutorID != uuid.Nil: transition, err = substrate.PrepareActorRetry(ctx, w.actors, binding, kind, creation) default: diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index 3414461be4..75a00eabea 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -129,11 +129,14 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding // PrepareActorRevert performs the read-only checks for crash recovery before // the store's durable claim. Reading first keeps a failed Actor read from // leaving a claim behind, so Delete can still supersede the unissued recovery. -// snapshotURI is the durable snapshot admission pinned ("" for none). Admission -// already settled the interrupted turn against it, so the revert must restore -// exactly that snapshot rather than whatever Substrate holds when it runs. -// A SUSPENDED Actor means an earlier revert completed and its response was lost. -func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding, snapshotURI string) (*ActorTransition, error) { +// actorUID is the recorded runtime and snapshotURI the durable snapshot admission +// pinned ("" for none). Admission already settled the interrupted turn against +// that snapshot, so the revert must restore exactly it rather than whatever +// Substrate holds when it runs. Identity is checked before state: a replacement +// in any state is unrecoverable, never a transient refusal that would keep a +// claimed recovery, and with it Delete, blocked. A SUSPENDED Actor means an +// earlier revert completed and its response was lost. +func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding, actorUID, snapshotURI string) (*ActorTransition, error) { actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) if status.Code(err) == codes.NotFound { return nil, fmt.Errorf("actor %s/%s is missing: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) @@ -141,7 +144,7 @@ func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding Act if err != nil { return nil, fmt.Errorf("get Actor %s/%s: %w", binding.Atespace, binding.Name, err) } - if !binding.Matches(actor) { + if !binding.Matches(actor) || actor.GetMetadata().GetUid() != actorUID { return nil, fmt.Errorf("actor %s/%s identity or template changed: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) } switch actor.GetStatus().GetState() { diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 086f321e5e..f28a2c2ffd 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -277,6 +277,7 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { revertUID string revertSnapshot *ateapipb.Snapshot pinned *string + recordedUID string wantPrepareErr bool unrecoverable bool wantApplyErr bool @@ -287,6 +288,7 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { {name: "earlier revert response lost", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantPrepareErr: true}, {name: "actor gone", missing: true, wantPrepareErr: true, unrecoverable: true}, + {name: "running replacement", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, recordedUID: "recorded-uid", wantPrepareErr: true, unrecoverable: true}, {name: "snapshot changed after admission", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, pinned: new("s3://snapshots/admitted"), wantPrepareErr: true, unrecoverable: true}, {name: "replaced actor", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertUID: "another-actor", wantApplyErr: true, wantCalls: []string{"revert"}}, {name: "different snapshot", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertSnapshot: durable(1, "s3://snapshots/older"), wantApplyErr: true, wantCalls: []string{"revert"}}, @@ -308,7 +310,11 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { if test.pinned != nil { pinned = *test.pinned } - transition, err := PrepareActorRevert(t.Context(), actors, binding, pinned) + recordedUID := "actor-uid" + if test.recordedUID != "" { + recordedUID = test.recordedUID + } + transition, err := PrepareActorRevert(t.Context(), actors, binding, recordedUID, pinned) if test.wantPrepareErr { require.Error(t, err) require.Equal(t, test.unrecoverable, errors.Is(err, ErrActorUnrecoverable)) From 5074268f3bf30d46359f77a630b800d1242b2c47 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 17:15:54 -0800 Subject: [PATCH 09/15] refactor(session): pin the recovery snapshot in one column Only BeginCrashedSessionRecovery starts a REVERT, and it writes the snapshot URI in the same transaction, so the stored URI always belongs to the current REVERT. recovery_operation_id, its paired CHECK, and the operation-ID join guarded a stale read that cannot happen. PrepareActorRevert checks the recorded Actor UID itself, so the REVERT-specific remapping of execute's later UID check was unreachable. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 3 ++- go/core/internal/database/session_recovery.go | 10 +++++----- go/core/internal/service/session/workflow.go | 3 --- .../core/000003_session_revert_operation.sql | 19 ++++++------------- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 8a6c325457..47e33bcfa5 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -245,7 +245,8 @@ work absent from the retained snapshot. Waiting tasks captured by that snapshot remain available. Checkpoint creation excludes competing recovery. Admission also pins the durable snapshot URI it judged waits against on the -Session, keyed by the REVERT operation ID. Substrate's revert restores the latest +Session. Only admission starts a REVERT, so the pin always belongs to the +current one. Substrate's revert restores the latest durable snapshot it holds when the call runs, so execution first requires the Actor to still report the pinned snapshot, then requires the reverted Actor to report it too. Otherwise a snapshot uploaded after admission could bring back a diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 6bfde03fd3..95c34157fe 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -145,11 +145,12 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se } // Execution must restore the snapshot settlement below judged waits against, // so the operation pins it rather than letting Substrate choose at revert time. + // Only this transaction starts a REVERT, so the pin always belongs to the + // current one. if err := execSQL(ctx, tx, ` - UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL, - recovery_operation_id = $3, recovery_snapshot_uri = $4 + UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL, recovery_snapshot_uri = $3 WHERE id = $1 - `, row.ID, row.Data, row.OperationID, snapshot.URI); err != nil { + `, row.ID, row.Data, snapshot.URI); err != nil { return err } if err := settleCrashedSessionTask(ctx, tx, row, snapshot); err != nil { @@ -174,8 +175,7 @@ type SessionRecoveryPin struct { func (c *Client) GetSessionRecoveryPin(ctx context.Context, sessionID string, operationID uuid.UUID) (SessionRecoveryPin, error) { pin, err := queryOne(ctx, c.db, ` SELECT s.actor_uid, s.recovery_snapshot_uri AS snapshot_uri FROM session s JOIN runtime_instance r USING (id) - WHERE s.id = $1 AND s.recovery_operation_id = $2 AND s.actor_uid IS NOT NULL - AND r.operation_id = $2 AND r.operation = 'RUNTIME_OPERATION_REVERT' + WHERE s.id = $1 AND r.operation_id = $2 AND r.operation = 'RUNTIME_OPERATION_REVERT' `, pgx.RowToStructByName[SessionRecoveryPin], sessionID, operationID) if err != nil { return SessionRecoveryPin{}, fmt.Errorf("read Session %s recovery pin: %w", sessionID, notFoundOr(err)) diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 7fdf67b91d..caaa3f9804 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -180,9 +180,6 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session if uid := transition.ActorUID(); uid != "" && kind != apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { if _, err := w.store.GetSessionForRuntime(ctx, session.Id, uid); err != nil { - if kind == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT && errors.Is(err, database.ErrNotFound) { - err = fmt.Errorf("runtime Actor was replaced: %w", substrate.ErrActorUnrecoverable) - } return w.failPreparation(ctx, operation, fmt.Errorf("verify runtime actor UID: %w", err)) } } diff --git a/go/core/pkg/migrations/core/000003_session_revert_operation.sql b/go/core/pkg/migrations/core/000003_session_revert_operation.sql index 45e10b3c73..8009fc941c 100644 --- a/go/core/pkg/migrations/core/000003_session_revert_operation.sql +++ b/go/core/pkg/migrations/core/000003_session_revert_operation.sql @@ -7,22 +7,15 @@ ALTER TABLE runtime_instance ADD CONSTRAINT runtime_instance_operation_check CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE', 'RUNTIME_OPERATION_REVERT')); --- The durable snapshot a crash recovery admitted, for the REVERT operation --- named by recovery_operation_id. It is valid only while that operation is the --- Session's current one, so a later operation needs no cleanup. The URI is --- empty when the crashed Actor had no durable snapshot. -ALTER TABLE session - ADD COLUMN recovery_operation_id UUID, - ADD COLUMN recovery_snapshot_uri TEXT, - ADD CONSTRAINT session_recovery_snapshot_check - CHECK ((recovery_operation_id IS NULL) = (recovery_snapshot_uri IS NULL)); +-- The durable snapshot the Session's current crash recovery admitted. Only +-- admission starts a REVERT and it always writes this column, so it is read +-- only while that operation is current. The URI is empty when the crashed +-- Actor had no durable snapshot. +ALTER TABLE session ADD COLUMN recovery_snapshot_uri TEXT; -- +goose Down -ALTER TABLE session - DROP CONSTRAINT session_recovery_snapshot_check, - DROP COLUMN recovery_snapshot_uri, - DROP COLUMN recovery_operation_id; +ALTER TABLE session DROP COLUMN recovery_snapshot_uri; -- Fails while any recovery is still in flight, rather than rewriting its -- operation into one the older binary would execute differently. From 3ffcbb7d5bc0c04c8d905ae0686cca2439f27e89 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 17:34:49 -0800 Subject: [PATCH 10/15] refactor(session): let Delete supersede any revert and drop NOT_CRASHED Delete supersedes an admitted REVERT even after an attempt issued it. Issued work blocks Delete because it may still create something Delete would miss; a revert restores the recorded Actor in place and creates nothing, and Substrate serializes the two calls. That removes AbandonSessionRecovery, the RuntimeUnrecoverable failure reason, and the early check in Revert. A retry against a gone or replaced Actor still returns KAGENT_SESSION_RUNTIME_UNRECOVERABLE without storing anything, so the client deletes the Session. RevertSession now returns the Session whenever the runtime is not crashed, at rest or running, as Create and Delete retries return current state. KAGENT_SESSION_RUNTIME_NOT_CRASHED and its mapping are gone; UNRECOVERABLE is the only refusal. The chat reading for a pending revert moves to the UI PR. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 34 ++++---- docs/lifecycle-retries.md | 14 ++-- .../internal/database/runtime_instances.go | 6 +- .../internal/database/session_operations.go | 3 +- go/core/internal/database/session_recovery.go | 46 ---------- .../database/session_recovery_test.go | 24 ++---- .../grpcserver/session_recovery_test.go | 5 +- go/core/internal/service/session/recovery.go | 53 ++++-------- .../internal/service/session/recovery_test.go | 83 ++++++++++--------- go/core/internal/service/session/service.go | 22 +---- .../internal/service/session/service_test.go | 31 +++---- go/core/internal/service/session/workflow.go | 32 ++----- go/core/test/e2e/session_revert_test.go | 23 +---- ui/src/components/chat/lifecycleReading.ts | 9 -- 14 files changed, 121 insertions(+), 264 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 47e33bcfa5..203fd6df5e 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -227,15 +227,13 @@ adopted implicitly. crashed runtime. It accepts active and idle Sessions, but verifies Substrate's CRASHED state for the recorded Actor UID and pinned template before admitting recovery. Quiet turns, transport failures, missing Actors and same-name -replacements cannot authorize a revert. A quiet turn returns `FAILED_PRECONDITION` -with reason `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. A missing or replaced Actor -returns `FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`, -because no retry restores it. A transport failure returns `UNAVAILABLE`. -Only an active lifecycle operation returns `ABORTED`. A Session at rest outside -READY returns `KAGENT_SESSION_RUNTIME_NOT_CRASHED`, so a client does not retry -it until its deadline. A READY Session whose recorded Actor is SUSPENDED has -nothing to recover, so RevertSession returns it. That is also what a retry sees -after a revert whose response was lost. +replacements cannot authorize a revert. A runtime that is not crashed, including +a Session at rest outside READY, has nothing to recover, so RevertSession +returns the Session unchanged. That is also what a retry sees after a revert +whose response was lost. A missing or replaced Actor returns +`FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`, because no +retry restores it. A transport failure returns `UNAVAILABLE`. Only an active +lifecycle operation returns `ABORTED`. Admission records FAILED/REVERT with a RuntimeCrashed failure under the Session lock. REVERT is its own `RuntimeOperation`: only a confirmed crash starts one, and @@ -274,20 +272,18 @@ continues it by retrying RevertSession, as with Create and Delete. While it is pending, `GetSession` reports FAILED/REVERT with failure reason `RuntimeCrashed`, and interactions are refused because a lifecycle operation is in progress. -A failed Actor read before the claim leaves the operation unclaimed, so Delete -can supersede it. If an attempt claimed the revert and the Actor is then missing, -replaced, or reports a different durable snapshot, no revert of the recorded -runtime can still be in progress. The next attempt abandons the operation: the -Session stays FAILED with reason `RuntimeUnrecoverable` and no operation, so -Delete can proceed. A live claim still wins, so an attempt that is issuing the -revert finishes first. +Delete supersedes an admitted recovery even after an attempt issued the revert. +Other issued lifecycle work blocks Delete because it may still create something +Delete would miss; a revert restores the recorded Actor in place and creates +nothing. Substrate serializes the two calls, and a late revert fails against the +deleted Actor. A recovery whose Actor is missing, replaced, or reports a +different durable snapshot stays pending, and each retry returns +`KAGENT_SESSION_RUNTIME_UNRECOVERABLE` so the client deletes the Session. A successful revert returns the Session to READY with the same Actor suspended. The next authorized interaction resumes it through Substrate ingress. Runtime changes after the retained snapshot are lost; the discarded turn receives no -checkpoint at that older snapshot. Another revert while the Actor stays -suspended returns the Session. Once an interaction resumes it, a revert returns -`FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_NOT_CRASHED`. +checkpoint at that older snapshot. Another revert returns the Session. ```mermaid sequenceDiagram diff --git a/docs/lifecycle-retries.md b/docs/lifecycle-retries.md index 6ccb6694f3..6c94c77dc8 100644 --- a/docs/lifecycle-retries.md +++ b/docs/lifecycle-retries.md @@ -57,15 +57,11 @@ Checkpoint creation returns `ABORTED` until it finishes. A revert that observed the crash before another lifecycle operation began also returns `ABORTED`. So does a revert while another lifecycle operation is active. -A revert that can never succeed against the Session as it stands returns `FAILED_PRECONDITION`. -Its `google.rpc.ErrorInfo` detail, in domain `kagent.dev`, names one of two reasons: - -| Reason | Meaning | Client action | -| --- | --- | --- | -| `KAGENT_SESSION_RUNTIME_NOT_CRASHED` | The runtime is running, or the Session is at rest outside READY. | Continue the conversation. | -| `KAGENT_SESSION_RUNTIME_UNRECOVERABLE` | The recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. The Session stays `FAILED` with failure reason `RuntimeUnrecoverable`. | Stop retrying. Delete the Session, then start another or fork one from a checkpoint. | - -A `FAILED_PRECONDITION` without either reason follows the general table below. +A revert of a runtime that is not crashed returns the Session unchanged. +A revert that can never succeed returns `FAILED_PRECONDITION` with a `google.rpc.ErrorInfo` detail in domain `kagent.dev` and reason `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`. +The recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. +Stop retrying, delete the Session, then start another or fork one from a checkpoint. Delete succeeds even while the recovery is pending. +A `FAILED_PRECONDITION` without that reason follows the general table below. Read the reason rather than the message text, which is not stable. ## Errors diff --git a/go/core/internal/database/runtime_instances.go b/go/core/internal/database/runtime_instances.go index dfdc24570c..8afd19d22e 100644 --- a/go/core/internal/database/runtime_instances.go +++ b/go/core/internal/database/runtime_instances.go @@ -85,7 +85,11 @@ func runtimeOperationNeeded(row runtimeInstanceRow, requested apiv1.RuntimeOpera if operation == requested { return false, nil } - if requested != apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE || row.ExecutorID != nil { + // A revert restores the recorded Actor in place and creates nothing, so + // Delete may supersede one even after it was issued: Substrate serializes + // the two calls, and a late revert fails against the deleted Actor. + issued := row.ExecutorID != nil && operation != apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT + if requested != apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE || issued { return false, ErrConflict } operation = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE diff --git a/go/core/internal/database/session_operations.go b/go/core/internal/database/session_operations.go index ca53013880..b629034aee 100644 --- a/go/core/internal/database/session_operations.go +++ b/go/core/internal/database/session_operations.go @@ -26,7 +26,8 @@ type SessionOperation struct { // turns, pending dispatch, and unfinished runtime cleanup, while checkpoint and // lifecycle guards remain. Accepted deletion stays retryable without force. // Creation retries and already-at-target requests return current state. Delete -// may supersede unclaimed work; uncertain lifecycle work blocks it. +// may supersede unclaimed work and any crash recovery; other uncertain lifecycle +// work blocks it. // Callers authorize access. Deleted sessions return ErrNotFound, except Delete // can observe the tombstone for a caller already authorized before deletion. func (c *Client) BeginSessionOperation(ctx context.Context, sessionID string, kind apiv1.RuntimeOperation, force bool) (*SessionOperation, error) { diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 95c34157fe..d4c19a2a94 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -18,11 +18,6 @@ import ( // that the recorded Actor crashed. Transport failures cannot admit this work. const SessionRuntimeCrashed = "RuntimeCrashed" -// SessionRuntimeUnrecoverable identifies a Session whose admitted crash -// recovery was abandoned because its recorded Actor is gone, was replaced, or no -// longer holds the admitted snapshot. No retry restores it; Delete still can. -const SessionRuntimeUnrecoverable = "RuntimeUnrecoverable" - // CrashRecoveryAdmitted reports a Session whose crash recovery was admitted and // has not finished. A retry joins that operation instead of observing Substrate // again. Only BeginCrashedSessionRecovery starts a REVERT, so the operation alone @@ -183,47 +178,6 @@ func (c *Client) GetSessionRecoveryPin(ctx context.Context, sessionID string, op return pin, nil } -// AbandonSessionRecovery ends an admitted REVERT whose Actor can no longer be -// restored: it is missing, was replaced, or no longer holds the pinned snapshot. -// Unlike a preparation failure it may end claimed work, because those -// observations show that no revert of the recorded Actor can still succeed. A -// live claim still wins, so an attempt that is issuing the revert finishes first. -// The Session stays FAILED with no operation, so Delete can proceed. -func (c *Client) AbandonSessionRecovery(ctx context.Context, sessionID string, operationID uuid.UUID, cause string) (*apiv1.Session, error) { - var result *apiv1.Session - err := c.withTx(ctx, func(tx pgx.Tx) error { - row, err := lockSession(ctx, tx, sessionID) - if err != nil { - return notFoundOr(err) - } - tag, err := tx.Exec(ctx, ` - UPDATE runtime_instance SET operation = 'RUNTIME_OPERATION_NONE', executor_id = NULL, executor_expires_at = NULL - WHERE id = $1 AND kind = 'agent' AND operation_id = $2 AND operation = 'RUNTIME_OPERATION_REVERT' - AND (executor_id IS NULL OR executor_expires_at <= clock_timestamp()) - `, row.ID, operationID) - if err != nil { - return err - } - if tag.RowsAffected() != 1 { - return ErrConflict - } - row.Operation, row.ExecutorID = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE.String(), nil - result, err = toSession(row) - if err != nil { - return err - } - result.Operation = apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE - result.Failure = &apiv1.Failure{Reason: SessionRuntimeUnrecoverable, Message: "Runtime cannot be recovered: " + cause} - result.UpdatedAt = timestamppb.Now() - row.Data, err = marshalSession(result) - if err != nil { - return err - } - return execSQL(ctx, tx, `UPDATE session SET data = $2 WHERE id = $1`, row.ID, row.Data) - }) - return result, err -} - // settleCrashedSessionTask retains captured waits and fails discarded nonterminal work. // Callers hold the Session lock; recovery cannot checkpoint discarded state. func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index ba449d3034..79d76eaa85 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -313,7 +313,7 @@ func TestSessionGenerationReadsRecordedActor(t *testing.T) { require.ErrorIs(t, err, ErrNotFound) } -func TestAbandonedRecoveryLetsDeleteProceed(t *testing.T) { +func TestDeleteSupersedesClaimedRecovery(t *testing.T) { client := NewClient(setupTestDB(t), "public") sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") session, _ := waitingTaskFixture(t, client) @@ -325,26 +325,12 @@ func TestAbandonedRecoveryLetsDeleteProceed(t *testing.T) { require.NoError(t, err) require.True(t, claimed) - _, err = client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") - require.ErrorIs(t, err, ErrConflict, "a live claim may still be issuing the revert") - _, err = client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, false) - require.ErrorIs(t, err, ErrConflict, "a claimed recovery blocks Delete") - - require.NoError(t, client.ReleaseRuntimeOperation(t.Context(), session.Id, operation.ID, executor)) - abandoned, err := client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") - require.NoError(t, err) - require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, abandoned.State) - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, abandoned.Operation) - require.Equal(t, SessionRuntimeUnrecoverable, abandoned.Failure.GetReason()) - stored, err := client.GetSessionByID(t.Context(), session.Id) - require.NoError(t, err) - require.True(t, proto.Equal(abandoned, stored)) - _, err = client.AbandonSessionRecovery(t.Context(), session.Id, operation.ID, "actor is missing") - require.ErrorIs(t, err, ErrConflict) - deletion, err := client.BeginSessionOperation(t.Context(), session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, false) - require.NoError(t, err) + require.NoError(t, err, "a revert creates nothing, so even an issued one cannot block Delete") require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, deletion.Instance.Operation) + require.Equal(t, uuid.Nil, deletion.ExecutorID) + _, err = client.FinishSessionOperation(t.Context(), session.Id, operation.ID, executor, "", uid, "") + require.ErrorIs(t, err, ErrConflict, "the superseded revert cannot finish") } func TestRevertCanOnlyJoinAnAdmittedRecovery(t *testing.T) { diff --git a/go/core/internal/grpcserver/session_recovery_test.go b/go/core/internal/grpcserver/session_recovery_test.go index e334afb469..1d4d0cc8b4 100644 --- a/go/core/internal/grpcserver/session_recovery_test.go +++ b/go/core/internal/grpcserver/session_recovery_test.go @@ -143,8 +143,9 @@ func TestRevertSessionThroughGRPC(t *testing.T) { require.Equal(t, codes.PermissionDenied, status.Code(err)) _, err = client.RevertSession(visitor, request) require.Equal(t, codes.NotFound, status.Code(err)) - _, err = client.RevertSession(owner, request) - requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", "healthy runtimes cannot revert") + healthy, err := client.RevertSession(owner, request) + require.NoError(t, err, "a healthy runtime has nothing to recover") + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, healthy.Session.State) control.actors[name].Status.State = ateapipb.ActorState_ACTOR_STATE_CRASHED before := proto.CloneOf(control.actors[name]) control.actors[name].Metadata.Uid = "replacement-uid" diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go index 33a15f9fae..2c3b87de0b 100644 --- a/go/core/internal/service/session/recovery.go +++ b/go/core/internal/service/session/recovery.go @@ -14,20 +14,11 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -// Revert refuses a Session with nothing to act on for one of two reasons. They -// ask opposite things of a client, so the API reports them with distinct reasons. -// errRuntimeNotCrashed means nothing needs recovering: the runtime is running or -// the Session is at rest in another state. A READY Session whose Actor is -// suspended is not refused; Revert returns it, which also covers a retry after a -// revert whose response was lost. -// errRuntimeUnrecoverable means the recorded Actor is gone, was replaced, or no -// longer holds the admitted snapshot. No retry restores it, and the client has -// to start another Session. Transient -// Substrate and database failures wrap neither, so they stay retryable. -var ( - errRuntimeNotCrashed = fmt.Errorf("runtime is not crashed: %w", database.ErrFailedPrecondition) - errRuntimeUnrecoverable = fmt.Errorf("runtime cannot be recovered: %w", database.ErrFailedPrecondition) -) +// errRuntimeUnrecoverable is Revert's only refusal: the recorded Actor is gone, +// was replaced, or no longer holds the admitted snapshot. No retry restores it, +// and the client deletes the Session. Transient Substrate and database failures +// do not wrap it, so they stay retryable. +var errRuntimeUnrecoverable = fmt.Errorf("runtime cannot be recovered: %w", database.ErrFailedPrecondition) // lifecycleIdle reports a Session with no lifecycle operation in flight. Only an // idle Session can admit a new revert; any other operation owns the Actor. @@ -46,11 +37,6 @@ func (o crashObservation) crashed() bool { return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED } -// reverted reports a recorded Actor at rest, as a completed revert leaves it. -func (o crashObservation) reverted() bool { - return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_SUSPENDED -} - // observeCrash reads the Session's recorded Actor and reports whether Substrate // confirms it crashed. Revert admits recovery only through this check. An Actor // that is missing, replaced, or from another template is an error, never a @@ -91,40 +77,29 @@ func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session // Revert restores a confirmed crashed runtime without replaying its interrupted turn. // Retries join the same durable operation and preserve the Actor identity. Nothing // retries an admitted recovery in the background: the caller retries RevertSession -// until it succeeds, as with CreateSession and DeleteSession. +// until it succeeds, as with CreateSession and DeleteSession. A runtime that is +// not crashed has nothing to recover, so Revert returns the Session unchanged. +// That is also what a retry sees after a revert whose response was lost. func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*apiv1.Session, error) { if database.CrashRecoveryAdmitted(session) { return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, false) } - if session.State == apiv1.RuntimeState_RUNTIME_STATE_FAILED && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE && - session.Failure.GetReason() == database.SessionRuntimeUnrecoverable { - return nil, fmt.Errorf("%s: %w", session.Failure.GetMessage(), errRuntimeUnrecoverable) - } - // run bounds a joined attempt. Admission bounds its own reads and execution. - ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) - defer cancel() - // Only an active lifecycle operation is contention: a retry converges once it - // finishes. A Session at rest in another state, such as one suspended by - // deadline cleanup, stays there until an interaction resumes it, so a retry - // would spin until the caller's deadline. + // Another lifecycle operation owns the Actor; a retry converges once it finishes. if session.Operation != apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { return nil, database.ErrConflict } if !lifecycleIdle(session) { - return nil, fmt.Errorf("runtime state is %s: %w", session.State, errRuntimeNotCrashed) + return session, nil } + // run bounds a joined attempt. Admission bounds its own reads and execution. + ctx, cancel := context.WithTimeout(ctx, database.RuntimeOperationTimeout) + defer cancel() observed, err := w.observeCrash(ctx, session) if err != nil { return nil, err } - if observed.reverted() { - // A READY Session whose recorded Actor is at rest has nothing to recover. - // This is also what a retry sees after a revert whose response was lost, - // so it returns the Session the way Create and Delete retries do. - return session, nil - } if !observed.crashed() { - return nil, errRuntimeNotCrashed + return session, nil } actor := observed.actor snapshot := substrate.LatestDurableActorSnapshot(actor) diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 4cb7947e06..23d9daf9f8 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -200,10 +200,11 @@ func TestSessionRecoveryRequiresConfirmedRecordedActorCrash(t *testing.T) { template string readErr error // want is the refusal a client sees; nil means the error must stay - // retryable, so it may match neither refusal. - want error + // retryable. healthy means Revert returns the Session unchanged. + want error + healthy bool }{ - {name: "quiet running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, want: errRuntimeNotCrashed}, + {name: "quiet running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, healthy: true}, {name: "transient read failure", readErr: status.Error(codes.Unavailable, "unreachable")}, {name: "missing actor", readErr: status.Error(codes.NotFound, "actor not found"), want: errRuntimeUnrecoverable}, {name: "same-name replacement", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, uid: "another-actor", want: errRuntimeUnrecoverable}, @@ -224,12 +225,16 @@ func TestSessionRecoveryRequiresConfirmedRecordedActorCrash(t *testing.T) { t.Fatal("unconfirmed crash issued a revert") return nil, nil } - _, err := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second).Revert(t.Context(), session) - require.Error(t, err) - if test.want != nil { + returned, err := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Second).Revert(t.Context(), session) + switch { + case test.healthy: + require.NoError(t, err) + require.True(t, proto.Equal(session, returned), "a healthy runtime has nothing to recover") + case test.want != nil: require.ErrorIs(t, err, test.want) require.ErrorIs(t, err, database.ErrFailedPrecondition) - } else { + default: + require.Error(t, err) require.NotErrorIs(t, err, database.ErrFailedPrecondition, "a transient failure must stay retryable") } current, err := store.GetSessionByID(t.Context(), session.Id) @@ -244,8 +249,8 @@ func TestSessionRecoveryRequiresConfirmedRecordedActorCrash(t *testing.T) { } // Only an active lifecycle operation is contention a retry waits out. A Session -// at rest outside READY, such as one suspended by deadline cleanup, would -// return ABORTED forever if it were reported as contention. +// at rest outside READY, such as one suspended by deadline cleanup, has nothing +// to recover, so Revert returns it rather than ABORTED forever. func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { for _, test := range []struct { name string @@ -253,7 +258,7 @@ func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { operation apiv1.RuntimeOperation want error }{ - {name: "suspended by deadline cleanup", state: apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, want: errRuntimeNotCrashed}, + {name: "suspended by deadline cleanup", state: apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE}, {name: "suspension in progress", state: apiv1.RuntimeState_RUNTIME_STATE_READY, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_SUSPEND, want: database.ErrConflict}, {name: "resume in progress", state: apiv1.RuntimeState_RUNTIME_STATE_SUSPENDED, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_RESUME, want: database.ErrConflict}, {name: "creation in progress", state: apiv1.RuntimeState_RUNTIME_STATE_CREATING, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, want: database.ErrConflict}, @@ -262,11 +267,15 @@ func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { // Both checks run on the Session the caller read, before any store // or Substrate access, so neither dependency is needed. workflow := NewActorWorkflow(nil, nil, make(chan struct{}, 1), time.Second) - _, err := workflow.Revert(t.Context(), &apiv1.Session{Id: uuid.NewString(), State: test.state, Operation: test.operation}) - require.ErrorIs(t, err, test.want) - if test.want == database.ErrConflict { - require.NotErrorIs(t, err, database.ErrFailedPrecondition) + session := &apiv1.Session{Id: uuid.NewString(), State: test.state, Operation: test.operation} + returned, err := workflow.Revert(t.Context(), session) + if test.want == nil { + require.NoError(t, err) + require.Same(t, session, returned) + return } + require.ErrorIs(t, err, test.want) + require.NotErrorIs(t, err, database.ErrFailedPrecondition) }) } } @@ -492,10 +501,11 @@ func TestExplicitRevertRetainsWaitCapturedByLaterTurn(t *testing.T) { require.Equal(t, a2a.TaskStateCompleted, retained.Status.State) } -// A recovery whose Actor cannot be read has issued nothing, so it must not take -// a claim that would make Delete refuse the Session. A recovery whose Actor is -// gone can never finish, so it ends even after an earlier attempt claimed it. -func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { +// Delete supersedes an admitted recovery whether or not an attempt issued the +// revert: a revert restores the recorded Actor in place and creates nothing that +// Delete could miss. Retries keep the recovery pending, and a gone or replaced +// Actor tells the client it cannot be recovered. +func TestDeleteSupersedesCrashRecovery(t *testing.T) { for _, test := range []struct { name string claimed bool @@ -505,6 +515,7 @@ func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { }{ {name: "actor read fails", readErr: status.Error(codes.Unavailable, "control plane unavailable")}, {name: "actor is gone", gone: true}, + {name: "after an issued attempt", claimed: true}, {name: "actor is gone after an issued attempt", claimed: true, gone: true}, {name: "running replacement after an issued attempt", claimed: true, replaced: true}, } { @@ -517,8 +528,6 @@ func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { }} _, err := NewActorWorkflow(store, unavailable, make(chan struct{}, 1), time.Minute).Revert(t.Context(), session) require.Error(t, err) - _, err = NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), session, false) - require.ErrorIs(t, err, database.ErrConflict, "an issued revert blocks Delete while it might still succeed") } else { _, err := store.BeginCrashedSessionRecovery(t.Context(), revertGeneration(t, store, session.Id, base.actors[key].GetMetadata().GetUid()), database.SessionTaskSnapshot{}) require.NoError(t, err) @@ -530,30 +539,26 @@ func TestDeleteSupersedesRecoveryThatCannotFinish(t *testing.T) { base.actors[key].Metadata.Uid = "replacement-uid" base.actors[key].Status.State = ateapipb.ActorState_ACTOR_STATE_RUNNING } - unrecoverable := test.gone || test.replaced - actors := &recoveryTestActors{lifecycleTestActors: base, readErr: test.readErr} - actors.revert = func(context.Context, string, string) (*ateapipb.Actor, error) { - t.Fatal("an unread Actor was reverted") - return nil, nil - } - workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) - for range 2 { - admitted, err := store.GetSessionByID(t.Context(), session.Id) - require.NoError(t, err) - _, err = workflow.Revert(t.Context(), admitted) - require.Error(t, err) - require.Equal(t, unrecoverable, errors.Is(err, errRuntimeUnrecoverable)) + if unrecoverable := test.gone || test.replaced; unrecoverable || test.readErr != nil { + actors := &recoveryTestActors{lifecycleTestActors: base, readErr: test.readErr} + actors.revert = func(context.Context, string, string) (*ateapipb.Actor, error) { + t.Fatal("an unread or replaced Actor was reverted") + return nil, nil + } + workflow := NewActorWorkflow(store, actors, make(chan struct{}, 1), time.Minute) + for range 2 { + admitted, err := store.GetSessionByID(t.Context(), session.Id) + require.NoError(t, err) + _, err = workflow.Revert(t.Context(), admitted) + require.Error(t, err) + require.Equal(t, unrecoverable, errors.Is(err, errRuntimeUnrecoverable)) + } } pending, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, pending.State) - if unrecoverable { - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE, pending.Operation, "a recovery that cannot finish is abandoned") - require.Equal(t, database.SessionRuntimeUnrecoverable, pending.Failure.GetReason()) - } else { - require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, pending.Operation, "a failed read keeps the recovery for a retry") - } + require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, pending.Operation, "a failed attempt keeps the recovery for a retry") deleted, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), pending, false) if test.replaced { // Delete is admitted, but never removes a same-name replacement. diff --git a/go/core/internal/service/session/service.go b/go/core/internal/service/session/service.go index a2afd9c2c2..6a4570c538 100644 --- a/go/core/internal/service/session/service.go +++ b/go/core/internal/service/session/service.go @@ -290,11 +290,10 @@ func (s *Service) Revert(ctx context.Context, id string) (*apiv1.Session, error) return nil, err } session, err = s.workflow.Revert(ctx, session) - if reason := revertRefusalReason(err); reason != "" { - // Both refusals are FAILED_PRECONDITION, but one means the runtime is - // already healthy and the other that no retry can restore it. The - // ErrorInfo reason lets a client tell them apart without parsing text. - refusal, detailErr := status.New(codes.FailedPrecondition, err.Error()).WithDetails(&errdetails.ErrorInfo{Domain: "kagent.dev", Reason: reason}) + if errors.Is(err, errRuntimeUnrecoverable) { + // The ErrorInfo reason tells a client to stop retrying and delete the + // Session, without parsing text or confusing it with other preconditions. + refusal, detailErr := status.New(codes.FailedPrecondition, err.Error()).WithDetails(&errdetails.ErrorInfo{Domain: "kagent.dev", Reason: "KAGENT_SESSION_RUNTIME_UNRECOVERABLE"}) if detailErr != nil { return nil, detailErr } @@ -312,19 +311,6 @@ func (s *Service) Revert(ctx context.Context, id string) (*apiv1.Session, error) return session, nil } -// revertRefusalReason names the ErrorInfo reason for a revert that can never -// succeed against the Session as it stands, or returns "" for any other error. -func revertRefusalReason(err error) string { - switch { - case errors.Is(err, errRuntimeNotCrashed): - return "KAGENT_SESSION_RUNTIME_NOT_CRASHED" - case errors.Is(err, errRuntimeUnrecoverable): - return "KAGENT_SESSION_RUNTIME_UNRECOVERABLE" - default: - return "" - } -} - // CreateShare mints a read-only share of the caller's session. A positive ttl bounds how long its // token grants access and must not exceed the configured maximum; zero takes the maximum, // or leaves the share valid until it is revoked or the session deleted when none is set. diff --git a/go/core/internal/service/session/service_test.go b/go/core/internal/service/session/service_test.go index 9a85d17ffb..d987526bab 100644 --- a/go/core/internal/service/session/service_test.go +++ b/go/core/internal/service/session/service_test.go @@ -209,25 +209,18 @@ func TestServiceCreateRejectsInvalidOrUnauthorizedRequests(t *testing.T) { } func TestServiceRevertNamesRefusalReason(t *testing.T) { - for _, test := range []struct { - cause error - reason string - }{ - {errRuntimeNotCrashed, "KAGENT_SESSION_RUNTIME_NOT_CRASHED"}, - {fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable), "KAGENT_SESSION_RUNTIME_UNRECOVERABLE"}, - } { - service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: test.cause}) - _, err := service.Revert(serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab") - refusal, ok := status.FromError(err) - require.True(t, ok) - require.Equal(t, codes.FailedPrecondition, refusal.Code()) - require.Equal(t, test.cause.Error(), refusal.Message()) - require.Len(t, refusal.Details(), 1) - info, ok := refusal.Details()[0].(*errdetails.ErrorInfo) - require.True(t, ok) - require.Equal(t, "kagent.dev", info.GetDomain()) - require.Equal(t, test.reason, info.GetReason()) - } + cause := fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) + service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: cause}) + _, err := service.Revert(serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab") + refusal, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.FailedPrecondition, refusal.Code()) + require.Equal(t, cause.Error(), refusal.Message()) + require.Len(t, refusal.Details(), 1) + info, ok := refusal.Details()[0].(*errdetails.ErrorInfo) + require.True(t, ok) + require.Equal(t, "kagent.dev", info.GetDomain()) + require.Equal(t, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", info.GetReason()) } func TestServiceLifecycleMethodsPreserveContentionErrors(t *testing.T) { diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index caaa3f9804..56917902c2 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -18,7 +18,6 @@ type workflowStore interface { GetSessionGeneration(context.Context, string) (*database.SessionGeneration, error) BeginCrashedSessionRecovery(context.Context, *database.SessionGeneration, database.SessionTaskSnapshot) (*database.SessionOperation, error) GetSessionRecoveryPin(context.Context, string, uuid.UUID) (database.SessionRecoveryPin, error) - AbandonSessionRecovery(context.Context, string, uuid.UUID, string) (*apiv1.Session, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) FinishSessionQuiescence(context.Context, *database.SessionQuiescence, *database.SessionTaskSnapshot) error @@ -108,7 +107,8 @@ func (w *ActorWorkflow) Suspend(ctx context.Context, session *apiv1.Session) (*a } // Delete closes admission before stopping and deleting compute. It can supersede -// unissued creation, but never deletes a session while a prior call is uncertain. +// unissued work and any crash recovery, but never deletes a session while another +// issued call is uncertain. func (w *ActorWorkflow) Delete(ctx context.Context, session *apiv1.Session, force bool) (*apiv1.Session, error) { return w.run(ctx, session.GetId(), apiv1.RuntimeOperation_RUNTIME_OPERATION_DELETE, force) } @@ -224,7 +224,13 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session // returns a conflict. A local preparation error cannot clear a newer operation. func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database.SessionOperation, cause error) (*apiv1.Session, error) { if database.CrashRecoveryAdmitted(admitted.Instance) { - return w.failRecoveryPreparation(ctx, admitted, cause) + // Admission already failed the interrupted turn, so clearing the operation + // would leave a FAILED Session that a retried RevertSession could not + // continue. Delete can supersede the kept intent, even after a claim. + if errors.Is(cause, substrate.ErrActorUnrecoverable) { + return nil, fmt.Errorf("%w: %w", cause, errRuntimeUnrecoverable) + } + return nil, cause } if admitted.ExecutorID != uuid.Nil { return nil, cause // An earlier attempt may have issued work; retain its intent. @@ -242,26 +248,6 @@ func (w *ActorWorkflow) failPreparation(ctx context.Context, admitted *database. return nil, errors.Join(cause, err) } -// failRecoveryPreparation keeps an admitted REVERT unless its Actor can no -// longer be restored. Admission already failed the interrupted turn, so clearing -// the operation as a preparation failure would leave a FAILED Session that a -// retried RevertSession could not continue. Without a claim, Delete can still -// supersede the kept intent. An Actor that is missing, replaced, or holding -// another snapshot shows that no revert of the recorded runtime can still be in -// progress, so the store ends the operation even after a claim. Otherwise the -// claim would block Delete forever. -func (w *ActorWorkflow) failRecoveryPreparation(ctx context.Context, admitted *database.SessionOperation, cause error) (*apiv1.Session, error) { - if !errors.Is(cause, substrate.ErrActorUnrecoverable) { - return nil, cause - } - finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) - defer cancel() - if _, err := w.store.AbandonSessionRecovery(finishCtx, admitted.Instance.Id, admitted.ID, cause.Error()); err != nil { - return nil, errors.Join(cause, err) - } - return nil, fmt.Errorf("%w: %w", cause, errRuntimeUnrecoverable) -} - func operationOutcome(operation *database.SessionOperation) (*apiv1.Session, error) { if operation.Instance.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE { return operation.Instance, nil diff --git a/go/core/test/e2e/session_revert_test.go b/go/core/test/e2e/session_revert_test.go index 597dee5274..9206f0eb66 100644 --- a/go/core/test/e2e/session_revert_test.go +++ b/go/core/test/e2e/session_revert_test.go @@ -16,9 +16,6 @@ import ( "github.com/kagent-dev/mockllm" "github.com/stretchr/testify/require" ateapipb "go.ate.dev/substrate/api/ateapi/v1" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -91,8 +88,9 @@ func TestSessionRevertCrashedRuntime(t *testing.T) { require.Equal(t, ateapipb.ActorState_ACTOR_STATE_RUNNING, running.GetStatus().GetState()) require.NotEmpty(t, substrate.SnapshotURI(substrate.LatestDurableActorSnapshot(running))) request := &apiv1.RevertSessionRequest{SessionId: fixture.sessionID} - _, err = fixture.sessions.RevertSession(fixture.ctx, request) - require.Equal(t, "KAGENT_SESSION_RUNTIME_NOT_CRASHED", revertRefusalReason(t, err), "healthy runtimes cannot revert") + healthy, err := fixture.sessions.RevertSession(fixture.ctx, request) + require.NoError(t, err, "a healthy runtime has nothing to recover") + require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, healthy.Session.State) assignment := running.GetStatus().GetWorkerAssignment() require.Equal(t, namespace, assignment.GetWorkerNamespace()) require.NotEmpty(t, assignment.GetWorkerPod()) @@ -155,18 +153,3 @@ func TestSessionRevertCrashedRuntime(t *testing.T) { require.Contains(t, taskText(next), "4") }) } - -// revertRefusalReason returns the kagent.dev ErrorInfo reason of a -// FAILED_PRECONDITION revert refusal. -func revertRefusalReason(t *testing.T, err error) string { - t.Helper() - refusal, ok := status.FromError(err) - require.True(t, ok, "revert error %v is not a gRPC status", err) - require.Equal(t, codes.FailedPrecondition, refusal.Code(), "revert error: %v", err) - for _, detail := range refusal.Details() { - if info, ok := detail.(*errdetails.ErrorInfo); ok && info.GetDomain() == "kagent.dev" { - return info.GetReason() - } - } - return "" -} diff --git a/ui/src/components/chat/lifecycleReading.ts b/ui/src/components/chat/lifecycleReading.ts index 6e38992d76..35f835f8ae 100644 --- a/ui/src/components/chat/lifecycleReading.ts +++ b/ui/src/components/chat/lifecycleReading.ts @@ -60,15 +60,6 @@ export function lifecycleReading( }; } - if (instance.operation === "revert") { - return { - active: "resuming", - summary: "Restoring this conversation", - source: - "The runtime crashed; the controller is restoring it to its last snapshot.", - }; - } - if (instance.operation === "suspend") { return { active: "suspending", From d8af66fe0f1b2cf54bc5d3063a9ce71134766566 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 17:42:34 -0800 Subject: [PATCH 11/15] refactor(session): drop comments that restate the code Remove comments added for crash recovery that narrate the next lines, restate helper names, or repeat test assertions. Keep the ordering, concurrency, and safety notes, shortened. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- go/core/internal/database/session_recovery.go | 61 +++++++------------ .../database/session_recovery_test.go | 7 --- .../grpcserver/session_recovery_test.go | 3 +- go/core/internal/service/session/recovery.go | 32 +++------- .../internal/service/session/recovery_test.go | 16 ++--- go/core/internal/service/session/service.go | 3 +- go/core/internal/substrate/lifecycle.go | 28 ++++----- go/core/internal/substrate/lifecycle_test.go | 2 - go/core/test/e2e/session_revert_test.go | 3 +- 9 files changed, 52 insertions(+), 103 deletions(-) diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index d4c19a2a94..8770597b90 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -14,24 +14,19 @@ import ( apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" ) -// SessionRuntimeCrashed identifies a recovery admitted after Substrate confirmed -// that the recorded Actor crashed. Transport failures cannot admit this work. +// SessionRuntimeCrashed is the failure reason set once Substrate confirms the +// recorded Actor crashed. Transport failures never set it. const SessionRuntimeCrashed = "RuntimeCrashed" -// CrashRecoveryAdmitted reports a Session whose crash recovery was admitted and -// has not finished. A retry joins that operation instead of observing Substrate -// again. Only BeginCrashedSessionRecovery starts a REVERT, so the operation alone -// identifies it. +// CrashRecoveryAdmitted reports an admitted, unfinished crash recovery. Only +// BeginCrashedSessionRecovery starts a REVERT, so the operation alone identifies it. func CrashRecoveryAdmitted(session *apiv1.Session) bool { return session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT } -// SessionGeneration is a live Session's lifecycle generation and the runtime it -// recorded, read together before crash recovery observes Substrate. OperationID -// is the Session's latest lifecycle operation, or uuid.Nil when none is recorded: -// every operation replaces it, and completion keeps it. ActorUID is the recorded -// runtime Actor, or "" when none is recorded. The template identifies where that -// Actor lives and what it must run. +// SessionGeneration is read before crash recovery observes Substrate. OperationID +// is uuid.Nil when none is recorded; every operation replaces it and completion +// keeps it. ActorUID is "" when none is recorded. type SessionGeneration struct { SessionID string OperationID uuid.UUID @@ -40,10 +35,8 @@ type SessionGeneration struct { TemplateName string } -// GetSessionGeneration reads the lifecycle generation, recorded Actor UID, and -// Actor template of a live Session in one statement. Deleted or missing Sessions -// return ErrNotFound. Callers authorize access. The read takes no lock: admission -// re-checks the generation and Actor UID under the Session lock. +// GetSessionGeneration returns ErrNotFound for deleted Sessions. Callers authorize +// access. The read takes no lock; admission re-checks it under the Session lock. func (c *Client) GetSessionGeneration(ctx context.Context, id string) (*SessionGeneration, error) { type generationRow struct { OperationID *uuid.UUID @@ -69,12 +62,10 @@ func (c *Client) GetSessionGeneration(ctx context.Context, id string) (*SessionG return generation, nil } -// BeginCrashedSessionRecovery admits confirmed-crash recovery under the Session -// lock. It preserves durable results; obsolete Actors and conflicting operations -// fail. generation is the caller's GetSessionGeneration read, taken before it -// observed the crash. The Session must still record its Actor UID, and a -// lifecycle operation begun since then makes the observation stale, because a -// concurrent revert keeps the Actor UID and returns to READY. +// BeginCrashedSessionRecovery admits recovery of a confirmed crash and preserves +// durable results. generation must be read before the crash was observed: a +// concurrent revert keeps the Actor UID and returns to READY, so only a newer +// lifecycle operation shows that the observation is stale. func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *SessionGeneration, snapshot SessionTaskSnapshot) (*SessionOperation, error) { var operation *SessionOperation err := c.withTx(ctx, func(tx pgx.Tx) error { @@ -82,8 +73,6 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se if err != nil { return notFoundOr(err) } - // Both guards are read under the lock in one statement, and are evaluated in - // their original order: Actor identity first, checkpoint creation last. type guards struct { Matches bool Busy bool @@ -117,12 +106,10 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se if current != generation.OperationID { return ErrConflict } - // An issued idle suspension does not exclude recovery here. It cannot finish - // against the crashed Actor, and its worker keeps the claim when suspension - // fails, so waiting for it would block recovery permanently. Settlement below - // retires the claim in this transaction; a late finish for it changes nothing. - // The generation check keeps the crash observation current: without another - // lifecycle operation, nothing returns the crashed Actor to service. + // An issued idle suspension does not block recovery. It cannot finish against + // the crashed Actor and its worker keeps the claim when suspension fails, so + // waiting would block forever. Settlement below retires the claim; a late + // finish changes nothing. if guard.Busy { return ErrConflict } @@ -138,10 +125,9 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se if err != nil { return err } - // Execution must restore the snapshot settlement below judged waits against, - // so the operation pins it rather than letting Substrate choose at revert time. - // Only this transaction starts a REVERT, so the pin always belongs to the - // current one. + // Pin the snapshot that settlement judges waits against, so the revert + // restores exactly it rather than whatever Substrate holds later. Only this + // transaction starts a REVERT, so the pin always belongs to the current one. if err := execSQL(ctx, tx, ` UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL, recovery_snapshot_uri = $3 WHERE id = $1 @@ -157,9 +143,8 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se return operation, err } -// SessionRecoveryPin is what an admitted REVERT must restore: the recorded Actor -// and the durable snapshot admission settled waits against. SnapshotURI is empty -// when the crashed Actor had no durable snapshot. +// SessionRecoveryPin is the Actor and durable snapshot an admitted REVERT must +// restore. SnapshotURI is empty when the crashed Actor had none. type SessionRecoveryPin struct { ActorUID string SnapshotURI string @@ -189,7 +174,7 @@ func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow } // The CTE clears the history's pending quiescence marker. The main query // reads the pre-update snapshot, but pending only tests that a boundary was - // recorded, which clearing keeps true. A task without events is malformed. + // recorded, which clearing keeps true. tasks, err := queryMany(ctx, tx, ` WITH cleared AS ( UPDATE session_task_event SET quiescence_pending = FALSE diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index 79d76eaa85..f2ff86fc03 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -195,7 +195,6 @@ func TestCrashedRecoveryRetiresIssuedIdleSuspension(t *testing.T) { require.NoError(t, err, "the retired claim no longer blocks a new turn") } -// sessionGeneration reads what a caller reads before it observes a crash. func sessionGeneration(t *testing.T, client *Client, sessionID string) *SessionGeneration { t.Helper() generation, err := client.GetSessionGeneration(t.Context(), sessionID) @@ -203,7 +202,6 @@ func sessionGeneration(t *testing.T, client *Client, sessionID string) *SessionG return generation } -// observedGeneration is a current generation from a caller that observed actorUID. func observedGeneration(t *testing.T, client *Client, sessionID, actorUID string) *SessionGeneration { t.Helper() generation := sessionGeneration(t, client, sessionID) @@ -211,8 +209,6 @@ func observedGeneration(t *testing.T, client *Client, sessionID, actorUID string return generation } -// completeCrashedRecovery admits and finishes a revert as a successful concurrent -// caller would, leaving the Session READY on the same Actor UID. func completeCrashedRecovery(t *testing.T, client *Client, sessionID, actorUID string) { t.Helper() operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, sessionID, actorUID), SessionTaskSnapshot{}) @@ -286,9 +282,6 @@ func TestCrashedRecoveryNotePinsItsTimelinePosition(t *testing.T) { require.True(t, position.Equal(*failed.Status.Timestamp), "position %s, status timestamp %s", position, failed.Status.Timestamp) } -// The generation reports the recorded Actor and the operation a crash -// observation is valid for. Admitting a recovery replaces the operation and keeps -// the UID. func TestSessionGenerationReadsRecordedActor(t *testing.T) { client := NewClient(setupTestDB(t), "public") sessionFixture(t, client, t.Context(), "team-a", "revision", "assistant", "kagent") diff --git a/go/core/internal/grpcserver/session_recovery_test.go b/go/core/internal/grpcserver/session_recovery_test.go index 1d4d0cc8b4..e8fe4a5126 100644 --- a/go/core/internal/grpcserver/session_recovery_test.go +++ b/go/core/internal/grpcserver/session_recovery_test.go @@ -178,8 +178,7 @@ func TestRevertSessionThroughGRPC(t *testing.T) { require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, ready.Session.State) require.Nil(t, ready.Session.Failure) - // A client whose success response was lost retries and gets the - // recovered Session back, as Create and Delete retries do. + // Retry as a client whose success response was lost. retried, err := client.RevertSession(owner, request) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, retried.Session.State) diff --git a/go/core/internal/service/session/recovery.go b/go/core/internal/service/session/recovery.go index 2c3b87de0b..7d348775bf 100644 --- a/go/core/internal/service/session/recovery.go +++ b/go/core/internal/service/session/recovery.go @@ -14,20 +14,14 @@ import ( "github.com/kagent-dev/kagent/go/core/internal/substrate" ) -// errRuntimeUnrecoverable is Revert's only refusal: the recorded Actor is gone, -// was replaced, or no longer holds the admitted snapshot. No retry restores it, -// and the client deletes the Session. Transient Substrate and database failures -// do not wrap it, so they stay retryable. +// errRuntimeUnrecoverable tells the client to delete the Session. Transient +// Substrate and database failures must not wrap it, so they stay retryable. var errRuntimeUnrecoverable = fmt.Errorf("runtime cannot be recovered: %w", database.ErrFailedPrecondition) -// lifecycleIdle reports a Session with no lifecycle operation in flight. Only an -// idle Session can admit a new revert; any other operation owns the Actor. func lifecycleIdle(session *apiv1.Session) bool { return session.State == apiv1.RuntimeState_RUNTIME_STATE_READY && session.Operation == apiv1.RuntimeOperation_RUNTIME_OPERATION_NONE } -// crashObservation is the Substrate state of a Session's recorded Actor, taken -// after the lifecycle generation it is valid for. type crashObservation struct { generation *database.SessionGeneration actor *ateapipb.Actor @@ -37,17 +31,13 @@ func (o crashObservation) crashed() bool { return o.actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_CRASHED } -// observeCrash reads the Session's recorded Actor and reports whether Substrate -// confirms it crashed. Revert admits recovery only through this check. An Actor -// that is missing, replaced, or from another template is an error, never a -// crash: reverting it would restore the wrong runtime. Those errors wrap -// errRuntimeUnrecoverable; transient read failures do not. +// observeCrash treats a missing, replaced, or retemplated Actor as unrecoverable, +// never as a crash: reverting it would restore the wrong runtime. // // The generation is read before the Actor. A concurrent revert keeps the Actor UID -// and returns to READY, so only the generation shows that a crash observation -// predates it. The recorded UID changes only through a lifecycle operation, -// which also replaces the generation, so admission's locked generation check -// rejects an observation compared against a stale UID. +// and returns to READY, so only the generation shows that an observation predates +// it. The UID changes only through a lifecycle operation, so admission's locked +// generation check also rejects a stale UID. func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session) (crashObservation, error) { generation, err := w.store.GetSessionGeneration(ctx, session.Id) if errors.Is(err, database.ErrNotFound) { @@ -75,11 +65,9 @@ func (w *ActorWorkflow) observeCrash(ctx context.Context, session *apiv1.Session } // Revert restores a confirmed crashed runtime without replaying its interrupted turn. -// Retries join the same durable operation and preserve the Actor identity. Nothing -// retries an admitted recovery in the background: the caller retries RevertSession -// until it succeeds, as with CreateSession and DeleteSession. A runtime that is -// not crashed has nothing to recover, so Revert returns the Session unchanged. -// That is also what a retry sees after a revert whose response was lost. +// Nothing retries an admitted recovery in the background; the caller retries until +// it succeeds. A runtime that is not crashed, including after a revert whose +// response was lost, is returned unchanged. func (w *ActorWorkflow) Revert(ctx context.Context, session *apiv1.Session) (*apiv1.Session, error) { if database.CrashRecoveryAdmitted(session) { return w.run(ctx, session.Id, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, false) diff --git a/go/core/internal/service/session/recovery_test.go b/go/core/internal/service/session/recovery_test.go index 23d9daf9f8..2f0d05dcc5 100644 --- a/go/core/internal/service/session/recovery_test.go +++ b/go/core/internal/service/session/recovery_test.go @@ -63,7 +63,6 @@ func (a *recoveryTestActors) GetActor(ctx context.Context, atespace, name string return actor, err } -// revertGeneration is a current generation from a caller that observed actorUID. func revertGeneration(t *testing.T, store *lifecycleTestStore, sessionID, actorUID string) *database.SessionGeneration { t.Helper() generation, err := store.GetSessionGeneration(t.Context(), sessionID) @@ -248,9 +247,8 @@ func TestSessionRecoveryRequiresConfirmedRecordedActorCrash(t *testing.T) { } } -// Only an active lifecycle operation is contention a retry waits out. A Session -// at rest outside READY, such as one suspended by deadline cleanup, has nothing -// to recover, so Revert returns it rather than ABORTED forever. +// A Session at rest outside READY has nothing to recover, so Revert returns it +// rather than ABORTED forever. Only an active operation is contention. func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { for _, test := range []struct { name string @@ -264,8 +262,7 @@ func TestExplicitRevertSeparatesContentionFromRestingState(t *testing.T) { {name: "creation in progress", state: apiv1.RuntimeState_RUNTIME_STATE_CREATING, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE, want: database.ErrConflict}, } { t.Run(test.name, func(t *testing.T) { - // Both checks run on the Session the caller read, before any store - // or Substrate access, so neither dependency is needed. + // Neither check reaches the store or Substrate. workflow := NewActorWorkflow(nil, nil, make(chan struct{}, 1), time.Second) session := &apiv1.Session{Id: uuid.NewString(), State: test.state, Operation: test.operation} returned, err := workflow.Revert(t.Context(), session) @@ -501,10 +498,8 @@ func TestExplicitRevertRetainsWaitCapturedByLaterTurn(t *testing.T) { require.Equal(t, a2a.TaskStateCompleted, retained.Status.State) } -// Delete supersedes an admitted recovery whether or not an attempt issued the -// revert: a revert restores the recorded Actor in place and creates nothing that -// Delete could miss. Retries keep the recovery pending, and a gone or replaced -// Actor tells the client it cannot be recovered. +// A revert restores the recorded Actor in place and creates nothing Delete could +// miss, so Delete supersedes an admitted recovery even after the revert was issued. func TestDeleteSupersedesCrashRecovery(t *testing.T) { for _, test := range []struct { name string @@ -561,7 +556,6 @@ func TestDeleteSupersedesCrashRecovery(t *testing.T) { require.Equal(t, apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, pending.Operation, "a failed attempt keeps the recovery for a retry") deleted, err := NewActorWorkflow(store, base, make(chan struct{}, 1), time.Minute).Delete(t.Context(), pending, false) if test.replaced { - // Delete is admitted, but never removes a same-name replacement. require.NotErrorIs(t, err, database.ErrConflict) deleting, err := store.GetSessionByID(t.Context(), session.Id) require.NoError(t, err) diff --git a/go/core/internal/service/session/service.go b/go/core/internal/service/session/service.go index 6a4570c538..ce0e3b9be1 100644 --- a/go/core/internal/service/session/service.go +++ b/go/core/internal/service/session/service.go @@ -291,8 +291,7 @@ func (s *Service) Revert(ctx context.Context, id string) (*apiv1.Session, error) } session, err = s.workflow.Revert(ctx, session) if errors.Is(err, errRuntimeUnrecoverable) { - // The ErrorInfo reason tells a client to stop retrying and delete the - // Session, without parsing text or confusing it with other preconditions. + // The ErrorInfo reason tells a client to stop retrying and delete the Session. refusal, detailErr := status.New(codes.FailedPrecondition, err.Error()).WithDetails(&errdetails.ErrorInfo{Domain: "kagent.dev", Reason: "KAGENT_SESSION_RUNTIME_UNRECOVERABLE"}) if detailErr != nil { return nil, detailErr diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index 75a00eabea..f4c7a296b2 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -55,10 +55,9 @@ type ActorTransition struct { creation *ActorCreation } -// ErrActorUnrecoverable reports that a crash recovery's Actor can no longer be -// restored to the snapshot its admission pinned: the Actor is missing, was -// replaced, or now reports a different durable snapshot. No retry changes that, -// and no revert of the recorded Actor can still be in progress. +// ErrActorUnrecoverable means the Actor is missing, was replaced, or reports a +// different durable snapshot. No retry changes that, and no revert of the +// recorded Actor can still be in progress. var ErrActorUnrecoverable = errors.New("actor cannot be restored to the admitted snapshot") // ActorUID returns the observed runtime identity, including after creation. @@ -126,15 +125,12 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return &ActorTransition{binding: binding, operation: operation, actor: actor, creation: creation}, nil } -// PrepareActorRevert performs the read-only checks for crash recovery before -// the store's durable claim. Reading first keeps a failed Actor read from -// leaving a claim behind, so Delete can still supersede the unissued recovery. -// actorUID is the recorded runtime and snapshotURI the durable snapshot admission -// pinned ("" for none). Admission already settled the interrupted turn against -// that snapshot, so the revert must restore exactly it rather than whatever -// Substrate holds when it runs. Identity is checked before state: a replacement -// in any state is unrecoverable, never a transient refusal that would keep a -// claimed recovery, and with it Delete, blocked. A SUSPENDED Actor means an +// PrepareActorRevert runs before the store's durable claim so a failed Actor read +// leaves no claim and Delete can still supersede the recovery. snapshotURI is the +// snapshot admission pinned ("" for none); admission already settled the +// interrupted turn against it, so the revert must restore exactly it. Identity is +// checked before state so a replacement in any state is unrecoverable rather than +// a transient refusal that keeps Delete blocked. A SUSPENDED Actor means an // earlier revert completed and its response was lost. func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding, actorUID, snapshotURI string) (*ActorTransition, error) { actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) @@ -236,10 +232,8 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -// applyRevert must not replace the Actor or restore anything but the snapshot -// PrepareActorRevert matched against admission's pin. A changed UID or snapshot -// means something other than this recovery acted on the runtime, so the Session -// must not adopt the result. +// applyRevert rejects a changed UID or snapshot: something other than this +// recovery acted on the runtime, so the Session must not adopt the result. func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClient) error { binding, actor := t.binding, t.actor before := proto.CloneOf(LatestDurableActorSnapshot(actor)) diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index f28a2c2ffd..f58a70dc65 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -72,8 +72,6 @@ func (a *transitionActors) SuspendActor(context.Context, string, string) (*ateap return proto.CloneOf(a.actor), nil } -// RevertActor restores the latest durable snapshot unless the test overrides -// the identity or snapshot it returns. func (a *transitionActors) RevertActor(context.Context, string, string) (*ateapipb.Actor, error) { a.calls = append(a.calls, "revert") a.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED diff --git a/go/core/test/e2e/session_revert_test.go b/go/core/test/e2e/session_revert_test.go index 9206f0eb66..349c011612 100644 --- a/go/core/test/e2e/session_revert_test.go +++ b/go/core/test/e2e/session_revert_test.go @@ -129,8 +129,7 @@ func TestSessionRevertCrashedRuntime(t *testing.T) { reverted, err := fixture.sessions.RevertSession(fixture.ctx, request) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, reverted.Session.State) - // A client whose success response was lost retries and gets the - // recovered Session back, as Create and Delete retries do. + // Retry as a client whose success response was lost. retried, err := fixture.sessions.RevertSession(fixture.ctx, request) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_READY, retried.Session.State) From 936419362d595241530889efe3c6a782fcb336b0 Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Thu, 5 Nov 2026 17:55:06 -0800 Subject: [PATCH 12/15] refactor(session): share task finalization with deletion and drop the ErrorInfo Crash settlement now reuses deletion's two common steps: publishSavedSessionResults publishes results whose cleanup acknowledgement was lost, and retireSessionQuiescence drops idle-work markers. finishDeletedSessionTasks calls the same helpers. The crash path keeps only what differs: it fails the open work the revert discards and keeps waits the snapshot captured. This replaces the CTE that cleared markers while reading tasks, and the re-read after settling. RevertSession's unrecoverable refusal is a plain FAILED_PRECONDITION. Nothing reads the KAGENT_SESSION_RUNTIME_UNRECOVERABLE reason, and a revert of a healthy runtime now succeeds, so the status alone says the runtime cannot be recovered. Revert maps errors exactly like Delete. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 5 +- docs/lifecycle-retries.md | 4 +- go/core/internal/database/session_deletion.go | 22 +----- go/core/internal/database/session_recovery.go | 76 +++++++------------ .../internal/database/task_finalization.go | 30 ++++++++ .../grpcserver/session_recovery_test.go | 21 +---- go/core/internal/service/session/service.go | 11 --- .../internal/service/session/service_test.go | 16 +--- 8 files changed, 69 insertions(+), 116 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 203fd6df5e..f57e9528aa 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -231,8 +231,7 @@ replacements cannot authorize a revert. A runtime that is not crashed, including a Session at rest outside READY, has nothing to recover, so RevertSession returns the Session unchanged. That is also what a retry sees after a revert whose response was lost. A missing or replaced Actor returns -`FAILED_PRECONDITION` with `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`, because no -retry restores it. A transport failure returns `UNAVAILABLE`. Only an active +`FAILED_PRECONDITION`, because no retry restores it. A transport failure returns `UNAVAILABLE`. Only an active lifecycle operation returns `ABORTED`. Admission records FAILED/REVERT with a RuntimeCrashed failure under the Session @@ -278,7 +277,7 @@ Delete would miss; a revert restores the recorded Actor in place and creates nothing. Substrate serializes the two calls, and a late revert fails against the deleted Actor. A recovery whose Actor is missing, replaced, or reports a different durable snapshot stays pending, and each retry returns -`KAGENT_SESSION_RUNTIME_UNRECOVERABLE` so the client deletes the Session. +`FAILED_PRECONDITION` so the client deletes the Session. A successful revert returns the Session to READY with the same Actor suspended. The next authorized interaction resumes it through Substrate ingress. Runtime diff --git a/docs/lifecycle-retries.md b/docs/lifecycle-retries.md index 6c94c77dc8..d596290cf1 100644 --- a/docs/lifecycle-retries.md +++ b/docs/lifecycle-retries.md @@ -58,10 +58,8 @@ A revert that observed the crash before another lifecycle operation began also r So does a revert while another lifecycle operation is active. A revert of a runtime that is not crashed returns the Session unchanged. -A revert that can never succeed returns `FAILED_PRECONDITION` with a `google.rpc.ErrorInfo` detail in domain `kagent.dev` and reason `KAGENT_SESSION_RUNTIME_UNRECOVERABLE`. -The recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. +A revert that can never succeed returns `FAILED_PRECONDITION`: the recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. Stop retrying, delete the Session, then start another or fork one from a checkpoint. Delete succeeds even while the recovery is pending. -A `FAILED_PRECONDITION` without that reason follows the general table below. Read the reason rather than the message text, which is not stable. ## Errors diff --git a/go/core/internal/database/session_deletion.go b/go/core/internal/database/session_deletion.go index 2658c70d0f..7b7b22dd46 100644 --- a/go/core/internal/database/session_deletion.go +++ b/go/core/internal/database/session_deletion.go @@ -82,26 +82,9 @@ func admitForcedSessionDeletion(ctx context.Context, tx pgx.Tx, row sessionRow) // deletion or actor absence. The caller holds the session lock and commits // these writes with the session's DELETED state. func finishDeletedSessionTasks(ctx context.Context, tx pgx.Tx, row sessionRow) error { - type boundary struct { - TaskID string - Sequence int64 - } - // A saved result may be waiting for a lost runtime cleanup acknowledgement. - // Publish it first so deletion preserves the result instead of canceling it. - boundaries, err := queryMany(ctx, tx, ` - SELECT task_id, sequence FROM session_task_event - WHERE history_id = $1 AND NOT published - AND expected_version IS NOT NULL AND quiescence_pending IS NOT NULL - ORDER BY sequence - `, pgx.RowToStructByName[boundary], row.HistoryID) - if err != nil { + if err := publishSavedSessionResults(ctx, tx, row); err != nil { return err } - for _, saved := range boundaries { - if err := settleSessionTask(ctx, tx, row, saved.TaskID, saved.Sequence); err != nil { - return err - } - } // Cancel only tasks still recorded as active after publishing saved results. // Waiting and terminal tasks keep their historical state. tasks, err := queryMany(ctx, tx, ` @@ -125,8 +108,7 @@ func finishDeletedSessionTasks(ctx context.Context, tx pgx.Tx, row sessionRow) e return err } } - // Retire idle-work markers; deletion no longer needs a later suspension. - return execSQL(ctx, tx, `UPDATE session_task_event SET quiescence_pending = FALSE WHERE history_id = $1 AND quiescence_pending`, row.HistoryID) + return retireSessionQuiescence(ctx, tx, row) } // finishSessionDeletion removes idle-expired sessions and their creation receipts. diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 8770597b90..5425cc6286 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -134,7 +134,7 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se `, row.ID, row.Data, snapshot.URI); err != nil { return err } - if err := settleCrashedSessionTask(ctx, tx, row, snapshot); err != nil { + if err := settleCrashedSessionTasks(ctx, tx, row, snapshot); err != nil { return err } operation, err = toSessionOperation(row) @@ -163,28 +163,24 @@ func (c *Client) GetSessionRecoveryPin(ctx context.Context, sessionID string, op return pin, nil } -// settleCrashedSessionTask retains captured waits and fails discarded nonterminal work. -// Callers hold the Session lock; recovery cannot checkpoint discarded state. -func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { +// settleCrashedSessionTasks publishes saved results, keeps waits the snapshot +// captured, and fails the rest of the open work the revert discards. +func settleCrashedSessionTasks(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { + if err := publishSavedSessionResults(ctx, tx, session); err != nil { + return err + } type interruptedTask struct { ID string Data []byte Version *int64 - Pending bool } - // The CTE clears the history's pending quiescence marker. The main query - // reads the pre-update snapshot, but pending only tests that a boundary was - // recorded, which clearing keeps true. + // A wait is captured when an event recording the snapshot follows the task's + // latest event. tasks, err := queryMany(ctx, tx, ` - WITH cleared AS ( - UPDATE session_task_event SET quiescence_pending = FALSE - WHERE history_id = $1 AND quiescence_pending) - SELECT t.id, t.data, latest.sequence AS version, - COALESCE(NOT latest.published AND latest.quiescence_pending IS NOT NULL, FALSE) AS pending + SELECT t.id, t.data, latest.sequence AS version FROM session_task t - LEFT JOIN LATERAL (SELECT sequence, published, quiescence_pending FROM session_task_event e - WHERE e.history_id = t.history_id AND e.task_id = t.id - ORDER BY sequence DESC LIMIT 1) latest ON TRUE + LEFT JOIN LATERAL (SELECT MAX(sequence) AS sequence FROM session_task_event e + WHERE e.history_id = t.history_id AND e.task_id = t.id) latest ON TRUE WHERE t.history_id = $1 AND (t.state IN ('TASK_STATE_SUBMITTED', 'TASK_STATE_WORKING') OR (t.state IN ('TASK_STATE_INPUT_REQUIRED', 'TASK_STATE_AUTH_REQUIRED') @@ -198,41 +194,27 @@ func settleCrashedSessionTask(ctx context.Context, tx pgx.Tx, session sessionRow return fmt.Errorf("read interrupted Session tasks: %w", err) } for _, interrupted := range tasks { - taskID, data := interrupted.ID, interrupted.Data if interrupted.Version == nil { - return fmt.Errorf("interrupted Session task %s has no events", taskID) - } - if interrupted.Pending { - if err := settleSessionTask(ctx, tx, session, taskID, *interrupted.Version); err != nil { - return err - } - // Settlement rewrote the projection, so the selected data is stale. - stored, err := readSessionTask(ctx, tx, session.HistoryID, taskID) - if err != nil { - return err - } - data = stored.Data - } - task, err := unmarshalSessionTask(data) + return fmt.Errorf("interrupted Session task %s has no events", interrupted.ID) + } + task, err := unmarshalSessionTask(interrupted.Data) if err != nil { return err } - if !task.Status.State.Terminal() { - task.Status = a2a.TaskStatus{State: a2a.TaskStateFailed, Timestamp: new(time.Now().UTC()), - Message: a2a.NewMessage(a2a.MessageRoleAgent, a2a.NewTextPart("The runtime crashed before this turn completed. Send a new message to continue."))} - apia2a.SetTimelinePosition(task.Status.Message, *task.Status.Timestamp) - event := &a2a.TaskStatusUpdateEvent{TaskID: task.ID, ContextID: task.ContextID, Status: task.Status} - if err := storeSessionTaskEvent(ctx, tx, session, task, event, false); err != nil { - return err - } - version, err := taskVersion(ctx, tx, session.HistoryID, taskID) - if err != nil { - return err - } - if err := publishPushBoundary(ctx, tx, session.HistoryID, taskID, version, task); err != nil { - return err - } + task.Status = a2a.TaskStatus{State: a2a.TaskStateFailed, Timestamp: new(time.Now().UTC()), + Message: a2a.NewMessage(a2a.MessageRoleAgent, a2a.NewTextPart("The runtime crashed before this turn completed. Send a new message to continue."))} + apia2a.SetTimelinePosition(task.Status.Message, *task.Status.Timestamp) + event := &a2a.TaskStatusUpdateEvent{TaskID: task.ID, ContextID: task.ContextID, Status: task.Status} + if err := storeSessionTaskEvent(ctx, tx, session, task, event, false); err != nil { + return err + } + version, err := taskVersion(ctx, tx, session.HistoryID, interrupted.ID) + if err != nil { + return err + } + if err := publishPushBoundary(ctx, tx, session.HistoryID, interrupted.ID, version, task); err != nil { + return err } } - return nil + return retireSessionQuiescence(ctx, tx, session) } diff --git a/go/core/internal/database/task_finalization.go b/go/core/internal/database/task_finalization.go index 6e2f575df6..2f80fb7c79 100644 --- a/go/core/internal/database/task_finalization.go +++ b/go/core/internal/database/task_finalization.go @@ -27,6 +27,36 @@ func (c *Client) SettleSessionTask(ctx context.Context, sessionID, taskID string }) } +// publishSavedSessionResults publishes every result the runtime saved whose +// cleanup acknowledgement was lost, so ending the runtime preserves them. +func publishSavedSessionResults(ctx context.Context, tx pgx.Tx, session sessionRow) error { + type boundary struct { + TaskID string + Sequence int64 + } + boundaries, err := queryMany(ctx, tx, ` + SELECT task_id, sequence FROM session_task_event + WHERE history_id = $1 AND NOT published + AND expected_version IS NOT NULL AND quiescence_pending IS NOT NULL + ORDER BY sequence + `, pgx.RowToStructByName[boundary], session.HistoryID) + if err != nil { + return err + } + for _, saved := range boundaries { + if err := settleSessionTask(ctx, tx, session, saved.TaskID, saved.Sequence); err != nil { + return err + } + } + return nil +} + +// retireSessionQuiescence drops idle-work markers for a runtime that no longer +// needs a later suspension. +func retireSessionQuiescence(ctx context.Context, tx pgx.Tx, session sessionRow) error { + return execSQL(ctx, tx, `UPDATE session_task_event SET quiescence_pending = FALSE WHERE history_id = $1 AND quiescence_pending`, session.HistoryID) +} + // settleSessionTask publishes one saved boundary under the session lock. Runtime // acknowledgement, confirmed actor deletion, or a confirmed crash must establish // cleanup first. diff --git a/go/core/internal/grpcserver/session_recovery_test.go b/go/core/internal/grpcserver/session_recovery_test.go index e8fe4a5126..5971703f9b 100644 --- a/go/core/internal/grpcserver/session_recovery_test.go +++ b/go/core/internal/grpcserver/session_recovery_test.go @@ -12,7 +12,6 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" ateapipb "go.ate.dev/substrate/api/ateapi/v1" - "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -150,11 +149,11 @@ func TestRevertSessionThroughGRPC(t *testing.T) { before := proto.CloneOf(control.actors[name]) control.actors[name].Metadata.Uid = "replacement-uid" _, err = client.RevertSession(owner, request) - requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", "a same-name replacement is not this runtime") + require.Equal(t, codes.FailedPrecondition, status.Code(err), "a same-name replacement is not this runtime") control.actors[name].Metadata.Uid = before.Metadata.Uid delete(control.actors, name) _, err = client.RevertSession(owner, request) - requireRevertRefusal(t, err, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", "a missing runtime cannot authorize recovery") + require.Equal(t, codes.FailedPrecondition, status.Code(err), "a missing runtime cannot authorize recovery") control.actors[name] = proto.CloneOf(before) calls := 0 failOnce := true @@ -204,19 +203,3 @@ func TestRevertSessionThroughGRPC(t *testing.T) { }) } } - -// requireRevertRefusal pins both the status code and the ErrorInfo reason, -// because the two FAILED_PRECONDITION refusals ask opposite things of a client. -func requireRevertRefusal(t *testing.T, err error, reason, message string) { - t.Helper() - refusal, ok := status.FromError(err) - require.True(t, ok, message) - require.Equal(t, codes.FailedPrecondition, refusal.Code(), message) - var reasons []string - for _, detail := range refusal.Details() { - if info, ok := detail.(*errdetails.ErrorInfo); ok && info.GetDomain() == "kagent.dev" { - reasons = append(reasons, info.GetReason()) - } - } - require.Equal(t, []string{reason}, reasons, message) -} diff --git a/go/core/internal/service/session/service.go b/go/core/internal/service/session/service.go index ce0e3b9be1..bb2578a4de 100644 --- a/go/core/internal/service/session/service.go +++ b/go/core/internal/service/session/service.go @@ -11,9 +11,6 @@ import ( "time" "github.com/google/uuid" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" utilvalidation "k8s.io/apimachinery/pkg/util/validation" @@ -290,14 +287,6 @@ func (s *Service) Revert(ctx context.Context, id string) (*apiv1.Session, error) return nil, err } session, err = s.workflow.Revert(ctx, session) - if errors.Is(err, errRuntimeUnrecoverable) { - // The ErrorInfo reason tells a client to stop retrying and delete the Session. - refusal, detailErr := status.New(codes.FailedPrecondition, err.Error()).WithDetails(&errdetails.ErrorInfo{Domain: "kagent.dev", Reason: "KAGENT_SESSION_RUNTIME_UNRECOVERABLE"}) - if detailErr != nil { - return nil, detailErr - } - return nil, refusal.Err() - } if errors.Is(err, database.ErrFailedPrecondition) { return nil, serviceerrors.NewFailedPrecondition(err.Error(), err) } diff --git a/go/core/internal/service/session/service_test.go b/go/core/internal/service/session/service_test.go index d987526bab..8e312517f1 100644 --- a/go/core/internal/service/session/service_test.go +++ b/go/core/internal/service/session/service_test.go @@ -13,9 +13,6 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" apiv1 "github.com/kagent-dev/kagent/go/api/gen/kagent/api/v1" "github.com/kagent-dev/kagent/go/core/internal/database" @@ -208,19 +205,12 @@ func TestServiceCreateRejectsInvalidOrUnauthorizedRequests(t *testing.T) { } } -func TestServiceRevertNamesRefusalReason(t *testing.T) { +func TestServiceRevertRefusesUnrecoverableRuntime(t *testing.T) { cause := fmt.Errorf("runtime Actor was replaced: %w", errRuntimeUnrecoverable) service := NewService(&serviceTestStore{}, serviceTestAuthorizer{}, serviceTestWorkflow{err: cause}) _, err := service.Revert(serviceTestContext("alice"), "8bd650a8-9775-488f-8bc1-0d52bf7bdcab") - refusal, ok := status.FromError(err) - require.True(t, ok) - require.Equal(t, codes.FailedPrecondition, refusal.Code()) - require.Equal(t, cause.Error(), refusal.Message()) - require.Len(t, refusal.Details(), 1) - info, ok := refusal.Details()[0].(*errdetails.ErrorInfo) - require.True(t, ok) - require.Equal(t, "kagent.dev", info.GetDomain()) - require.Equal(t, "KAGENT_SESSION_RUNTIME_UNRECOVERABLE", info.GetReason()) + require.True(t, serviceerrors.IsCode(err, serviceerrors.CodeFailedPrecondition), "error = %v", err) + require.Equal(t, cause.Error(), serviceerrors.MessageOf(err)) } func TestServiceLifecycleMethodsPreserveContentionErrors(t *testing.T) { From bae48c510ceb9ab6e16deccd92ea445b008565de Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Fri, 6 Nov 2026 07:41:22 -0800 Subject: [PATCH 13/15] refactor(session): drop the recovery snapshot pin Substrate's RevertActor restores the Actor's last completed durable snapshot, and kagent suspends with DURABILITY_MODE_SYNCHRONOUS, so an upload completes before SuspendActor returns. A crashed Actor cannot be suspended, and a revert takes no snapshot. The snapshot admission settles against is therefore the one the revert restores, and the pin guarded a case that cannot happen. applyRevert still requires the reverted Actor to report the snapshot it held before. Removed: session.recovery_snapshot_uri from migration 000002, SessionRecoveryPin and GetSessionRecoveryPin, and the "durable snapshot changed after recovery was admitted" refusal. That refusal was the only remaining path to a FAILED/REVERT Session that could only be deleted. PrepareActorRevert now only reads the Actor and checks its name and template. execute's existing recorded-UID check reports a replacement as unrecoverable, and applyRevert handles the Actor's state. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 16 ++++----- docs/lifecycle-retries.md | 2 +- go/core/internal/database/session_recovery.go | 28 +-------------- .../database/session_recovery_test.go | 7 ---- go/core/internal/service/session/workflow.go | 10 +++--- go/core/internal/substrate/lifecycle.go | 35 +++++++------------ go/core/internal/substrate/lifecycle_test.go | 16 ++------- .../core/000003_session_revert_operation.sql | 8 ----- go/core/pkg/migrations/runner_test.go | 3 -- 9 files changed, 27 insertions(+), 98 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index f57e9528aa..0491fd0cb1 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -241,13 +241,11 @@ publishes a durable terminal result saved before the crash, or fails nonterminal work absent from the retained snapshot. Waiting tasks captured by that snapshot remain available. Checkpoint creation excludes competing recovery. -Admission also pins the durable snapshot URI it judged waits against on the -Session. Only admission starts a REVERT, so the pin always belongs to the -current one. Substrate's revert restores the latest -durable snapshot it holds when the call runs, so execution first requires the -Actor to still report the pinned snapshot, then requires the reverted Actor to -report it too. Otherwise a snapshot uploaded after admission could bring back a -wait that admission had already failed. +Admission settles waits against the Actor's latest completed durable snapshot, +which is the one Substrate's revert restores. kagent suspends with synchronous +durability, so a durable upload completes before suspension returns, and a +crashed Actor cannot be suspended. Execution requires the reverted Actor to +report the same snapshot it held before the revert. The caller reads the Session's generation before it observes Substrate: its lifecycle generation, recorded Actor UID, and Actor template, in one statement. @@ -275,8 +273,8 @@ Delete supersedes an admitted recovery even after an attempt issued the revert. Other issued lifecycle work blocks Delete because it may still create something Delete would miss; a revert restores the recorded Actor in place and creates nothing. Substrate serializes the two calls, and a late revert fails against the -deleted Actor. A recovery whose Actor is missing, replaced, or reports a -different durable snapshot stays pending, and each retry returns +deleted Actor. A recovery whose Actor is missing or replaced stays pending, and +each retry returns `FAILED_PRECONDITION` so the client deletes the Session. A successful revert returns the Session to READY with the same Actor suspended. diff --git a/docs/lifecycle-retries.md b/docs/lifecycle-retries.md index d596290cf1..452b09ad1b 100644 --- a/docs/lifecycle-retries.md +++ b/docs/lifecycle-retries.md @@ -58,7 +58,7 @@ A revert that observed the crash before another lifecycle operation began also r So does a revert while another lifecycle operation is active. A revert of a runtime that is not crashed returns the Session unchanged. -A revert that can never succeed returns `FAILED_PRECONDITION`: the recorded Actor is missing, a same-name Actor replaced it, or it no longer holds the snapshot recovery admitted. +A revert that can never succeed returns `FAILED_PRECONDITION`: the recorded Actor is missing, or a same-name Actor replaced it. Stop retrying, delete the Session, then start another or fork one from a checkpoint. Delete succeeds even while the recovery is pending. Read the reason rather than the message text, which is not stable. diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index 5425cc6286..e91d322cea 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -125,13 +125,7 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se if err != nil { return err } - // Pin the snapshot that settlement judges waits against, so the revert - // restores exactly it rather than whatever Substrate holds later. Only this - // transaction starts a REVERT, so the pin always belongs to the current one. - if err := execSQL(ctx, tx, ` - UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL, recovery_snapshot_uri = $3 - WHERE id = $1 - `, row.ID, row.Data, snapshot.URI); err != nil { + if err := execSQL(ctx, tx, `UPDATE session SET data = $2, dispatch_id = NULL, dispatch_expires_at = NULL WHERE id = $1`, row.ID, row.Data); err != nil { return err } if err := settleCrashedSessionTasks(ctx, tx, row, snapshot); err != nil { @@ -143,26 +137,6 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se return operation, err } -// SessionRecoveryPin is the Actor and durable snapshot an admitted REVERT must -// restore. SnapshotURI is empty when the crashed Actor had none. -type SessionRecoveryPin struct { - ActorUID string - SnapshotURI string -} - -// GetSessionRecoveryPin returns the pin of the Session's current REVERT -// operation. Any other operation returns ErrNotFound. -func (c *Client) GetSessionRecoveryPin(ctx context.Context, sessionID string, operationID uuid.UUID) (SessionRecoveryPin, error) { - pin, err := queryOne(ctx, c.db, ` - SELECT s.actor_uid, s.recovery_snapshot_uri AS snapshot_uri FROM session s JOIN runtime_instance r USING (id) - WHERE s.id = $1 AND r.operation_id = $2 AND r.operation = 'RUNTIME_OPERATION_REVERT' - `, pgx.RowToStructByName[SessionRecoveryPin], sessionID, operationID) - if err != nil { - return SessionRecoveryPin{}, fmt.Errorf("read Session %s recovery pin: %w", sessionID, notFoundOr(err)) - } - return pin, nil -} - // settleCrashedSessionTasks publishes saved results, keeps waits the snapshot // captured, and fails the rest of the open work the revert discards. func settleCrashedSessionTasks(ctx context.Context, tx pgx.Tx, session sessionRow, snapshot SessionTaskSnapshot) error { diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index f2ff86fc03..0f9c1d552d 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -52,9 +52,6 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { } before, _, err := client.GetVersionedSessionTask(t.Context(), session.Id, string(task.ID)) require.NoError(t, err) - _, err = client.GetSessionRecoveryPin(t.Context(), session.Id, uuid.New()) - require.ErrorIs(t, err, ErrNotFound) - operation, err := client.BeginCrashedSessionRecovery(t.Context(), observedGeneration(t, client, session.Id, "actor-"+session.Id), snapshot) require.NoError(t, err) require.Equal(t, apiv1.RuntimeState_RUNTIME_STATE_FAILED, operation.Instance.State) @@ -85,10 +82,6 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { require.ErrorIs(t, client.ReserveSessionDispatch(t.Context(), session.Id, uuid.New(), "next"), ErrConflict) _, _, err = client.ReserveSessionCheckpoint(t.Context(), &apiv1.Checkpoint{Id: uuid.NewString(), SessionId: session.Id, HeadTaskId: string(task.ID)}, session.Creator, uuid.NewString()) require.ErrorIs(t, err, ErrConflict) - pin, err := client.GetSessionRecoveryPin(t.Context(), session.Id, operation.ID) - require.NoError(t, err) - require.Equal(t, SessionRecoveryPin{ActorUID: "actor-" + session.Id, SnapshotURI: snapshot.URI}, pin, - "the operation pins the Actor and the snapshot that settlement judged waits against") }) } } diff --git a/go/core/internal/service/session/workflow.go b/go/core/internal/service/session/workflow.go index 56917902c2..14e3c61783 100644 --- a/go/core/internal/service/session/workflow.go +++ b/go/core/internal/service/session/workflow.go @@ -17,7 +17,6 @@ import ( type workflowStore interface { GetSessionGeneration(context.Context, string) (*database.SessionGeneration, error) BeginCrashedSessionRecovery(context.Context, *database.SessionGeneration, database.SessionTaskSnapshot) (*database.SessionOperation, error) - GetSessionRecoveryPin(context.Context, string, uuid.UUID) (database.SessionRecoveryPin, error) ClaimSessionQuiescence(context.Context) (*database.SessionQuiescence, error) HasPendingSessionQuiescence(context.Context) (bool, error) FinishSessionQuiescence(context.Context, *database.SessionQuiescence, *database.SessionTaskSnapshot) error @@ -164,11 +163,7 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session var transition *substrate.ActorTransition switch { case database.CrashRecoveryAdmitted(session): - var pin database.SessionRecoveryPin - if pin, err = w.store.GetSessionRecoveryPin(ctx, sessionID, operation.ID); err != nil { - return w.failPreparation(ctx, operation, fmt.Errorf("load admitted recovery pin: %w", err)) - } - transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding, pin.ActorUID, pin.SnapshotURI) + transition, err = substrate.PrepareActorRevert(ctx, w.actors, binding) case operation.ExecutorID != uuid.Nil: transition, err = substrate.PrepareActorRetry(ctx, w.actors, binding, kind, creation) default: @@ -180,6 +175,9 @@ func (w *ActorWorkflow) execute(ctx context.Context, operation *database.Session if uid := transition.ActorUID(); uid != "" && kind != apiv1.RuntimeOperation_RUNTIME_OPERATION_CREATE { if _, err := w.store.GetSessionForRuntime(ctx, session.Id, uid); err != nil { + if database.CrashRecoveryAdmitted(session) && errors.Is(err, database.ErrNotFound) { + err = fmt.Errorf("runtime Actor was replaced: %w", substrate.ErrActorUnrecoverable) + } return w.failPreparation(ctx, operation, fmt.Errorf("verify runtime actor UID: %w", err)) } } diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index f4c7a296b2..aa4e0dc60c 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -55,10 +55,9 @@ type ActorTransition struct { creation *ActorCreation } -// ErrActorUnrecoverable means the Actor is missing, was replaced, or reports a -// different durable snapshot. No retry changes that, and no revert of the -// recorded Actor can still be in progress. -var ErrActorUnrecoverable = errors.New("actor cannot be restored to the admitted snapshot") +// ErrActorUnrecoverable reports that a crash recovery's Actor is missing or was +// replaced. No retry changes that, and no revert of it can still be in progress. +var ErrActorUnrecoverable = errors.New("actor is missing or was replaced") // ActorUID returns the observed runtime identity, including after creation. func (t *ActorTransition) ActorUID() string { @@ -125,14 +124,10 @@ func prepareActorTransition(ctx context.Context, actors LifecycleClient, binding return &ActorTransition{binding: binding, operation: operation, actor: actor, creation: creation}, nil } -// PrepareActorRevert runs before the store's durable claim so a failed Actor read -// leaves no claim and Delete can still supersede the recovery. snapshotURI is the -// snapshot admission pinned ("" for none); admission already settled the -// interrupted turn against it, so the revert must restore exactly it. Identity is -// checked before state so a replacement in any state is unrecoverable rather than -// a transient refusal that keeps Delete blocked. A SUSPENDED Actor means an -// earlier revert completed and its response was lost. -func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding, actorUID, snapshotURI string) (*ActorTransition, error) { +// PrepareActorRevert reads the Actor before the store's durable claim, so a +// failed read leaves no claim. A missing or retemplated Actor is unrecoverable. +// The caller then checks the recorded UID; applyRevert checks the state. +func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding ActorBinding) (*ActorTransition, error) { actor, err := actors.GetActor(ctx, binding.Atespace, binding.Name) if status.Code(err) == codes.NotFound { return nil, fmt.Errorf("actor %s/%s is missing: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) @@ -140,17 +135,9 @@ func PrepareActorRevert(ctx context.Context, actors LifecycleClient, binding Act if err != nil { return nil, fmt.Errorf("get Actor %s/%s: %w", binding.Atespace, binding.Name, err) } - if !binding.Matches(actor) || actor.GetMetadata().GetUid() != actorUID { + if !binding.Matches(actor) { return nil, fmt.Errorf("actor %s/%s identity or template changed: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) } - switch actor.GetStatus().GetState() { - case ateapipb.ActorState_ACTOR_STATE_CRASHED, ateapipb.ActorState_ACTOR_STATE_REVERTING, ateapipb.ActorState_ACTOR_STATE_SUSPENDED: - default: - return nil, fmt.Errorf("crashed Actor %s/%s has unexpected state %s", binding.Atespace, binding.Name, actor.GetStatus().GetState()) - } - if SnapshotURI(LatestDurableActorSnapshot(actor)) != snapshotURI { - return nil, fmt.Errorf("actor %s/%s durable snapshot changed after recovery was admitted: %w", binding.Atespace, binding.Name, ErrActorUnrecoverable) - } return &ActorTransition{binding: binding, operation: apiv1.RuntimeOperation_RUNTIME_OPERATION_REVERT, actor: actor}, nil } @@ -232,8 +219,10 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -// applyRevert rejects a changed UID or snapshot: something other than this -// recovery acted on the runtime, so the Session must not adopt the result. +// applyRevert requires the same Actor back, SUSPENDED, holding the durable +// snapshot it held before. Substrate restores the last completed durable +// snapshot, and kagent suspends synchronously, so admission settled against that +// snapshot; anything else means something other than this recovery acted. func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClient) error { binding, actor := t.binding, t.actor before := proto.CloneOf(LatestDurableActorSnapshot(actor)) diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index f58a70dc65..4c4dcff20b 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -274,8 +274,6 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { state ateapipb.ActorState revertUID string revertSnapshot *ateapipb.Snapshot - pinned *string - recordedUID string wantPrepareErr bool unrecoverable bool wantApplyErr bool @@ -284,10 +282,8 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { {name: "crashed", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, wantCalls: []string{"revert"}}, {name: "revert in progress", state: ateapipb.ActorState_ACTOR_STATE_REVERTING, wantCalls: []string{"revert"}}, {name: "earlier revert response lost", state: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, - {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantPrepareErr: true}, + {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantApplyErr: true}, {name: "actor gone", missing: true, wantPrepareErr: true, unrecoverable: true}, - {name: "running replacement", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, recordedUID: "recorded-uid", wantPrepareErr: true, unrecoverable: true}, - {name: "snapshot changed after admission", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, pinned: new("s3://snapshots/admitted"), wantPrepareErr: true, unrecoverable: true}, {name: "replaced actor", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertUID: "another-actor", wantApplyErr: true, wantCalls: []string{"revert"}}, {name: "different snapshot", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertSnapshot: durable(1, "s3://snapshots/older"), wantApplyErr: true, wantCalls: []string{"revert"}}, } { @@ -304,15 +300,7 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { } } binding := ActorBinding{Revision: "revision", Atespace: "team", Name: "instance", TemplateAtespace: "team", TemplateName: "revision"} - pinned := "s3://snapshots/latest" - if test.pinned != nil { - pinned = *test.pinned - } - recordedUID := "actor-uid" - if test.recordedUID != "" { - recordedUID = test.recordedUID - } - transition, err := PrepareActorRevert(t.Context(), actors, binding, recordedUID, pinned) + transition, err := PrepareActorRevert(t.Context(), actors, binding) if test.wantPrepareErr { require.Error(t, err) require.Equal(t, test.unrecoverable, errors.Is(err, ErrActorUnrecoverable)) diff --git a/go/core/pkg/migrations/core/000003_session_revert_operation.sql b/go/core/pkg/migrations/core/000003_session_revert_operation.sql index 8009fc941c..2fe3fc1493 100644 --- a/go/core/pkg/migrations/core/000003_session_revert_operation.sql +++ b/go/core/pkg/migrations/core/000003_session_revert_operation.sql @@ -7,16 +7,8 @@ ALTER TABLE runtime_instance ADD CONSTRAINT runtime_instance_operation_check CHECK (operation IN ('RUNTIME_OPERATION_NONE', 'RUNTIME_OPERATION_CREATE', 'RUNTIME_OPERATION_SUSPEND', 'RUNTIME_OPERATION_RESUME', 'RUNTIME_OPERATION_DELETE', 'RUNTIME_OPERATION_REVERT')); --- The durable snapshot the Session's current crash recovery admitted. Only --- admission starts a REVERT and it always writes this column, so it is read --- only while that operation is current. The URI is empty when the crashed --- Actor had no durable snapshot. -ALTER TABLE session ADD COLUMN recovery_snapshot_uri TEXT; - -- +goose Down -ALTER TABLE session DROP COLUMN recovery_snapshot_uri; - -- Fails while any recovery is still in flight, rather than rewriting its -- operation into one the older binary would execute differently. ALTER TABLE runtime_instance DROP CONSTRAINT runtime_instance_operation_check; diff --git a/go/core/pkg/migrations/runner_test.go b/go/core/pkg/migrations/runner_test.go index 6582bcdba6..10fae8fbd3 100644 --- a/go/core/pkg/migrations/runner_test.go +++ b/go/core/pkg/migrations/runner_test.go @@ -337,9 +337,6 @@ func TestBuiltinMigrationsRoundTrip(t *testing.T) { if !testColumnExists(t, dsn, "session", "deletion_reason") { t.Fatal("session deletion reason is missing") } - if !testColumnExists(t, dsn, "session", "recovery_snapshot_uri") { - t.Fatal("session crash recovery snapshot pin is missing") - } if !testTableExists(t, dsn, "agent_definition") || testTableExists(t, dsn, "agent_template_harness_pair") { t.Fatal("initial migration must create explicit Agent definitions without legacy pairs") } From 7dcb63de619af927ce0ea2b5abceac2f3a9eb2ec Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Fri, 6 Nov 2026 07:56:08 -0800 Subject: [PATCH 14/15] refactor(substrate): stop checking the reverted snapshot applyRevert compared the Actor's durable snapshot before and after RevertActor. That comparison cannot fail. Substrate's revert keeps the latest completed durable snapshot and discards pending uploads (workflow_revert.go, ensureInProgressSnapshotDiscarded). Every suspend of a kagent Actor is synchronous: kagent's SuspendActor passes DURABILITY_MODE_SYNCHRONOUS, and so does Substrate's template reconciler. Atelet's RequestActorSuspend, the only asynchronous path, has no caller in kagent, its guest library, or Substrate at v1.0.0-rc4. So no durable upload can complete between admission and the revert. applyRevert now requires only the same Actor back, SUSPENDED. That also answers the review question on #3395. Settling at admission is deliberate: the snapshot it settles against is the one the revert restores, and the interrupted turn fails as soon as the crash is confirmed. An asynchronous suspend would break this, and settlement would then move to when the revert finishes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- docs/architecture/runtime-and-lifecycle.md | 10 +++++---- go/core/internal/substrate/lifecycle.go | 22 ++++++-------------- go/core/internal/substrate/lifecycle_test.go | 10 ++------- 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/docs/architecture/runtime-and-lifecycle.md b/docs/architecture/runtime-and-lifecycle.md index 0491fd0cb1..a76cc94697 100644 --- a/docs/architecture/runtime-and-lifecycle.md +++ b/docs/architecture/runtime-and-lifecycle.md @@ -242,10 +242,12 @@ work absent from the retained snapshot. Waiting tasks captured by that snapshot remain available. Checkpoint creation excludes competing recovery. Admission settles waits against the Actor's latest completed durable snapshot, -which is the one Substrate's revert restores. kagent suspends with synchronous -durability, so a durable upload completes before suspension returns, and a -crashed Actor cannot be suspended. Execution requires the reverted Actor to -report the same snapshot it held before the revert. +which is the one Substrate's revert restores. Every suspend kagent issues uses +synchronous durability, so its upload completes before suspension returns, and +a crashed Actor cannot be suspended. No upload can complete between admission +and the revert, so settling at admission settles against what is restored, and +the interrupted turn fails as soon as the crash is confirmed. An asynchronous +suspend would break this and require settling when the revert finishes. The caller reads the Session's generation before it observes Substrate: its lifecycle generation, recorded Actor UID, and Actor template, in one statement. diff --git a/go/core/internal/substrate/lifecycle.go b/go/core/internal/substrate/lifecycle.go index aa4e0dc60c..40c040d59a 100644 --- a/go/core/internal/substrate/lifecycle.go +++ b/go/core/internal/substrate/lifecycle.go @@ -219,29 +219,19 @@ func ApplyActorTransition(ctx context.Context, actors LifecycleClient, transitio return nil } -// applyRevert requires the same Actor back, SUSPENDED, holding the durable -// snapshot it held before. Substrate restores the last completed durable -// snapshot, and kagent suspends synchronously, so admission settled against that -// snapshot; anything else means something other than this recovery acted. +// applyRevert requires the same Actor back, SUSPENDED. Substrate restores the +// last completed durable snapshot, which is the one admission settled against: +// every suspend kagent issues is synchronous, so no upload completes in between. func (t *ActorTransition) applyRevert(ctx context.Context, actors LifecycleClient) error { - binding, actor := t.binding, t.actor - before := proto.CloneOf(LatestDurableActorSnapshot(actor)) - if before != nil { - // Revert restores onto any node, so the locality of the source copy is - // expected to change. - before.Locality = "" - } + actor := t.actor if state := actor.GetStatus().GetState(); state == ateapipb.ActorState_ACTOR_STATE_CRASHED || state == ateapipb.ActorState_ACTOR_STATE_REVERTING { - reverted, err := actors.RevertActor(ctx, binding.Atespace, binding.Name) + reverted, err := actors.RevertActor(ctx, t.binding.Atespace, t.binding.Name) if err != nil { return err } actor = reverted } - if t.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) != nil || !proto.Equal(before, LatestActorSnapshot(actor)) { - return fmt.Errorf("revert returned unexpected identity or state") - } - return nil + return t.checkResult(actor, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) } // Matches checks that the actor belongs to the expected runtime and template. diff --git a/go/core/internal/substrate/lifecycle_test.go b/go/core/internal/substrate/lifecycle_test.go index 4c4dcff20b..1cf9f33b2e 100644 --- a/go/core/internal/substrate/lifecycle_test.go +++ b/go/core/internal/substrate/lifecycle_test.go @@ -24,7 +24,6 @@ type transitionActors struct { resumeUID string suspendSnapshot *ateapipb.Snapshot revertUID string - revertSnapshot *ateapipb.Snapshot } func (a *transitionActors) GetActor(context.Context, string, string) (*ateapipb.Actor, error) { @@ -76,9 +75,6 @@ func (a *transitionActors) RevertActor(context.Context, string, string) (*ateapi a.calls = append(a.calls, "revert") a.actor.Status.State = ateapipb.ActorState_ACTOR_STATE_SUSPENDED snapshot := proto.CloneOf(LatestDurableActorSnapshot(a.actor)) - if a.revertSnapshot != nil { - snapshot = proto.CloneOf(a.revertSnapshot) - } a.actor.Status.Snapshots = nil if snapshot != nil { snapshot.Locality = "" @@ -263,7 +259,7 @@ func TestActorSuspendRequiresDurableVolumesSnapshot(t *testing.T) { } } -func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { +func TestActorRevertKeepsTheRecordedActor(t *testing.T) { durable := func(generation int32, uri string) *ateapipb.Snapshot { return &ateapipb.Snapshot{Generation: generation, Fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, Locality: "lost-node", DurableSnapshot: &ateapipb.SnapshotStorage{State: ateapipb.SnapshotStorageState_SNAPSHOT_STORAGE_STATE_COMPLETED, Object: &ateapipb.ObjectSnapshot{SnapshotUri: uri}}} @@ -273,7 +269,6 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { missing bool state ateapipb.ActorState revertUID string - revertSnapshot *ateapipb.Snapshot wantPrepareErr bool unrecoverable bool wantApplyErr bool @@ -285,10 +280,9 @@ func TestActorRevertRestoresTheObservedSnapshotInPlace(t *testing.T) { {name: "running actor", state: ateapipb.ActorState_ACTOR_STATE_RUNNING, wantApplyErr: true}, {name: "actor gone", missing: true, wantPrepareErr: true, unrecoverable: true}, {name: "replaced actor", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertUID: "another-actor", wantApplyErr: true, wantCalls: []string{"revert"}}, - {name: "different snapshot", state: ateapipb.ActorState_ACTOR_STATE_CRASHED, revertSnapshot: durable(1, "s3://snapshots/older"), wantApplyErr: true, wantCalls: []string{"revert"}}, } { t.Run(test.name, func(t *testing.T) { - actors := &transitionActors{revertUID: test.revertUID, revertSnapshot: test.revertSnapshot} + actors := &transitionActors{revertUID: test.revertUID} if !test.missing { actors.actor = &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team", Name: "instance", Uid: "actor-uid"}, From b802704ddbb9824ff29d71f33d03a39e7eb5026a Mon Sep 17 00:00:00 2001 From: Ada Lundhe Date: Fri, 6 Nov 2026 11:26:37 -0800 Subject: [PATCH 15/15] refactor(session): fence crash admission by operation ID alone The actor_uid comparison in BeginCrashedSessionRecovery was redundant. The recorded UID only changes through a lifecycle operation, and every operation replaces operation_id, so the generation check already rejects a stale UID. It also had an edge case: with actor_uid NULL, `actor_uid = $2` scanned NULL into a bool, so the caller got Unavailable instead of a retryable Conflict. The query now checks only for a checkpoint being created. Same-name replacements are still caught before admission, in observeCrash. TestSuspendActorWaitsForLatestDurableSnapshot already asserts DURABILITY_MODE_SYNCHRONOUS. It now says crash recovery depends on that. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ada Lundhe --- go/core/internal/database/session_recovery.go | 23 ++++++------------- .../database/session_recovery_test.go | 4 ---- go/core/internal/substrate/client_test.go | 2 ++ 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/go/core/internal/database/session_recovery.go b/go/core/internal/database/session_recovery.go index e91d322cea..d591f33b5b 100644 --- a/go/core/internal/database/session_recovery.go +++ b/go/core/internal/database/session_recovery.go @@ -73,21 +73,6 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se if err != nil { return notFoundOr(err) } - type guards struct { - Matches bool - Busy bool - } - guard, err := queryOne(ctx, tx, ` - SELECT actor_uid = $2 AS matches, - EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') AS busy - FROM session WHERE id = $1 - `, pgx.RowToStructByName[guards], row.ID, generation.ActorUID) - if err != nil { - return err - } - if !guard.Matches || generation.ActorUID == "" { - return ErrConflict - } session, err := toSession(row) if err != nil { return err @@ -110,7 +95,13 @@ func (c *Client) BeginCrashedSessionRecovery(ctx context.Context, generation *Se // the crashed Actor and its worker keeps the claim when suspension fails, so // waiting would block forever. Settlement below retires the claim; a late // finish changes nothing. - if guard.Busy { + busy, err := queryOne(ctx, tx, ` + SELECT EXISTS (SELECT 1 FROM session_checkpoint WHERE source_session_id = $1 AND state = 'CREATING') + `, pgx.RowTo[bool], row.ID) + if err != nil { + return err + } + if busy { return ErrConflict } row.State = apiv1.RuntimeState_RUNTIME_STATE_FAILED.String() diff --git a/go/core/internal/database/session_recovery_test.go b/go/core/internal/database/session_recovery_test.go index 0f9c1d552d..0412f02c07 100644 --- a/go/core/internal/database/session_recovery_test.go +++ b/go/core/internal/database/session_recovery_test.go @@ -89,10 +89,8 @@ func TestCrashedRecoverySettlesTasksAtomically(t *testing.T) { func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) { for _, test := range []struct { name string - actorUID string checkpoint, superseded, lifecycle bool }{ - {name: "replacement actor", actorUID: "replacement"}, {name: "checkpoint", checkpoint: true}, {name: "revert after the crash was observed", superseded: true}, {name: "lifecycle", lifecycle: true}, @@ -105,8 +103,6 @@ func TestCrashedRecoveryAdmissionRespectsFences(t *testing.T) { // Callers read the generation before they observe the crash. generation := sessionGeneration(t, client, session.Id) switch { - case test.actorUID != "": - uid = test.actorUID case test.checkpoint: task.Status.State = a2a.TaskStateCompleted require.NoError(t, saveRuntimeTask(t, client, session.Id, task, task, &SessionTaskSnapshot{ diff --git a/go/core/internal/substrate/client_test.go b/go/core/internal/substrate/client_test.go index d709a34a7b..d063e77b6c 100644 --- a/go/core/internal/substrate/client_test.go +++ b/go/core/internal/substrate/client_test.go @@ -430,6 +430,8 @@ func TestSuspendActorWaitsForLatestDurableSnapshot(t *testing.T) { fake := &suspendActorFake{actor: actor, get: func(context.Context) (*ateapipb.Actor, error) { reads++; return next, nil }} client := &Client{ControlClient: fake, cfg: Config{CallTimeout: time.Second}} got, err := client.SuspendActor(t.Context(), "team", "actor") + // Crash recovery depends on this: with no upload left in progress, a + // revert restores the snapshot admission settled against (applyRevert). require.Equal(t, ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS, fake.request.GetOptions().GetMode()) require.Equal(t, 1, reads) if test.wantError != "" {