diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index c9da0ff5f1..6aa89223df 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -191,6 +191,10 @@ type actorSnapshots struct { // the actor in progress on this node. While it is nonzero, atelet does not // detach the actor, and the entry stays in memory. holds int + // inFlight holds the snapshots of Checkpoints and PromoteSnapshots of the + // actor currently in progress on this node. newest includes them, so an + // older generation cannot start while a newer one is in flight. + inFlight []*SnapshotRef // promotes counts the PromoteSnapshots of the actor in progress on this // node. While it is nonzero, atelet does not start a background upload of // the actor. @@ -207,6 +211,28 @@ type actorSnapshots struct { unsaved bool } +// localNeeded reports whether CommitLocal should record snap as LatestLocal: +// LatestLocal does not already hold snap or a newer snapshot, and no other +// committed snapshot has a higher generation or the same generation under +// another UUID. +func (a *actorSnapshots) localNeeded(snap SnapshotRef) bool { + if a.LatestLocal != nil && a.LatestLocal.generation() >= snap.generation() { + return false + } + for _, r := range []*SnapshotRef{a.Uploading, a.LatestDurable} { + if r == nil { + continue + } + if r.generation() > snap.generation() { + return false + } + if r.generation() == snap.generation() && !sameUUID(r, &snap) { + return false + } + } + return true +} + // uploadNeeded reports whether ref still needs uploading: neither it nor a // newer snapshot is durable. func (a *actorSnapshots) uploadNeeded(ref SnapshotRef) bool { @@ -331,6 +357,52 @@ func (a *actorSnapshots) runningDetach() *SnapshotRef { return &a.op.target } +// opTarget returns the snapshot an operation in progress acts on, or nil. +func (a *actorSnapshots) opTarget() *SnapshotRef { + if a.op == nil { + return nil + } + return &a.op.target +} + +// newest returns the actor's snapshot with the highest generation, or nil +// when atelet holds no snapshot for the actor. It includes snapshots whose +// Checkpoint or PromoteSnapshot is in flight. +func (a *actorSnapshots) newest() *SnapshotRef { + var newest *SnapshotRef + consider := func(r *SnapshotRef) { + if r != nil && (newest == nil || r.generation() > newest.generation()) { + newest = r + } + } + consider(a.LatestLocal) + consider(a.Uploading) + consider(a.LatestDurable) + consider(a.opTarget()) + for _, r := range a.inFlight { + consider(r) + } + return newest +} + +// checkGeneration returns a FailedPrecondition error if snap has a stale +// generation for the actor: below the actor's newest snapshot, or equal to +// it under another snapshot_uuid. +func (a *actorSnapshots) checkGeneration(snap *ateletpb.Snapshot) error { + newest := a.newest() + if newest == nil { + return nil + } + switch { + case snap.GetGeneration() < newest.generation(): + return apierror.FailedPrecondition("snapshot %q has generation %d, below the actor's newest snapshot %q at generation %d", snap.GetSnapshotUuid(), snap.GetGeneration(), newest.uuid(), newest.generation()) + case snap.GetGeneration() == newest.generation() && snap.GetSnapshotUuid() != newest.uuid(): + return apierror.FailedPrecondition("snapshot %q has generation %d, which the actor's newest snapshot %q already holds", snap.GetSnapshotUuid(), snap.GetGeneration(), newest.uuid()) + default: + return nil + } +} + // uploadDue returns when atelet next runs a background upload of the actor: // uploadDelay after UndurableSince, or at retryAt if that is later. An upload // or promote in progress can delay it. ok is false when there is nothing diff --git a/cmd/atelet/internal/snapshottracker/background.go b/cmd/atelet/internal/snapshottracker/background.go index e0a20d9ea3..a67d52187b 100644 --- a/cmd/atelet/internal/snapshottracker/background.go +++ b/cmd/atelet/internal/snapshottracker/background.go @@ -395,12 +395,18 @@ func (t *Tracker) waitIdleLocked(ctx context.Context, actorUID string, check fun // waitForDetachLocked waits, as long as ctx allows, until no detach of the // actor is in progress, and returns the actor's entry, which may be nil. It // releases the actor's lock while it waits, and holds it again when it -// returns. Checkpoint, Run, and Restore call this first so they never race -// with a detach pruning the actor's local directory, without waiting behind -// an upload. -func (t *Tracker) waitForDetachLocked(ctx context.Context, actorUID string) (*actorSnapshots, error) { +// returns. check, if not nil, vets the entry on every pass: an error from it +// is returned at once, without waiting further. Checkpoint, Run, and Restore +// call this first so they never race with a detach pruning the actor's local +// directory, without waiting behind an upload. +func (t *Tracker) waitForDetachLocked(ctx context.Context, actorUID string, check func(*actorSnapshots) error) (*actorSnapshots, error) { for { a := t.findLocked(actorUID) + if a != nil && check != nil { + if err := check(a); err != nil { + return nil, err + } + } if a == nil || a.runningDetach() == nil { return a, nil } @@ -412,11 +418,19 @@ func (t *Tracker) waitForDetachLocked(ctx context.Context, actorUID string) (*ac // waitForDetachOrSnapshotUploadLocked waits, as long as ctx allows, until no // detach of the actor and no background upload of snap is in progress, and -// returns the actor's entry, which may be nil. PromoteSnapshot calls this -// first so it never races with a detach or uploads the same snapshot twice. +// returns the actor's entry, which may be nil. It checks snap's generation on +// every pass, so a stale generation fails at once without waiting, and one +// superseded while waiting fails as soon as the wait ends. PromoteSnapshot +// calls this first so it never races with a detach or uploads the same +// snapshot twice. func (t *Tracker) waitForDetachOrSnapshotUploadLocked(ctx context.Context, actorUID string, snap SnapshotRef) (*actorSnapshots, error) { for { a := t.findLocked(actorUID) + if a != nil { + if err := a.checkGeneration(snap.Snapshot); err != nil { + return nil, err + } + } if a == nil || a.op == nil { return a, nil } diff --git a/cmd/atelet/internal/snapshottracker/background_test.go b/cmd/atelet/internal/snapshottracker/background_test.go index 8ad15acb0a..18facfc447 100644 --- a/cmd/atelet/internal/snapshottracker/background_test.go +++ b/cmd/atelet/internal/snapshottracker/background_test.go @@ -335,9 +335,9 @@ func TestTrackerBackoff(t *testing.T) { name string call func() }{ - {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s0", 1, clk.Now())) }}, - {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s1", 1, clk.Now())) }}, - {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s2", 2, clk.Now())) }}, + {"CommitLocal", func() { tr.CommitLocal(trackedActorUID, testRef("s1", 1, clk.Now())) }}, + {"CommitDurable", func() { tr.CommitDurable(trackedActorUID, testRef("s2", 2, clk.Now())) }}, + {"CommitPromoted", func() { tr.CommitPromoted(trackedActorUID, testRef("s3", 3, clk.Now())) }}, {"Forget", func() { if err := tr.Forget(trackedActorUID); err != nil { t.Fatalf("Forget: %v", err) diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index 80eae9a13c..64a5e69603 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -170,7 +170,6 @@ func TestTrackerDetach(t *testing.T) { store := newFakeSnapshotStore() tr := newTestTracker(t, store) local := testRef("s1", 1, tr.clock.Now()) - local.Snapshot.Object = nil tr.CommitLocal(trackedActorUID, local) promoted := testRef("s1", 1, time.Time{}) @@ -258,9 +257,10 @@ func TestTrackerDetach(t *testing.T) { // A refused detach keeps the local snapshot and uploads nothing. for _, tc := range []struct { - name string - setup func(t *testing.T, tr *Tracker) - wantCode codes.Code + name string + modifyLocal func(*SnapshotRef) + setup func(t *testing.T, tr *Tracker) + wantCode codes.Code }{ { name: "refused after a start", @@ -287,7 +287,7 @@ func TestTrackerDetach(t *testing.T) { { name: "refused while a checkpoint is in progress", setup: func(t *testing.T, tr *Tracker) { - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } @@ -305,20 +305,18 @@ func TestTrackerDetach(t *testing.T) { }, { name: "refused when durable URI mismatches local", + modifyLocal: func(local *SnapshotRef) { + local.Snapshot.Object.SnapshotUri += "-other" + }, setup: func(t *testing.T, tr *Tracker) { tr.CommitDurable(trackedActorUID, testRef("s1", 1, tr.clock.Now())) - local := testRef("s1", 1, tr.clock.Now()) - local.Snapshot.Object.SnapshotUri += "-other" - tr.CommitLocal(trackedActorUID, local) }, wantCode: codes.FailedPrecondition, }, { name: "refused without an object storage location", - setup: func(t *testing.T, tr *Tracker) { - local := testRef("s1", 1, tr.clock.Now()) + modifyLocal: func(local *SnapshotRef) { local.Snapshot.Object = nil - tr.CommitLocal(trackedActorUID, local) }, wantCode: codes.FailedPrecondition, }, @@ -326,8 +324,14 @@ func TestTrackerDetach(t *testing.T) { t.Run(tc.name, func(t *testing.T) { store := newFakeSnapshotStore() tr := newTestTracker(t, store) - tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) - tc.setup(t, tr) + local := testRef("s1", 1, tr.clock.Now()) + if tc.modifyLocal != nil { + tc.modifyLocal(&local) + } + tr.CommitLocal(trackedActorUID, local) + if tc.setup != nil { + tc.setup(t, tr) + } if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != tc.wantCode { t.Errorf("detach = %v, want %v", err, tc.wantCode) } @@ -409,7 +413,7 @@ func TestTrackerDetachTimer(t *testing.T) { tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) due := detachDueOf(tr, trackedActorUID) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s3", 3, time.Time{}).Snapshot) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } @@ -485,8 +489,10 @@ func TestTrackerWaitsForDetach(t *testing.T) { waitStarted(t, store) for name, begin := range map[string]func(context.Context, string) (func(), error){ - "beginStart": tr.BeginStart, - "beginCheckpoint": tr.BeginCheckpoint, + "beginStart": tr.BeginStart, + "beginCheckpoint": func(ctx context.Context, uid string) (func(), error) { + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + }, } { result := make(chan error, 1) results[name] = result @@ -522,6 +528,75 @@ func TestTrackerWaitsForDetach(t *testing.T) { } } +// BeginCheckpoint and BeginPromote fail at once during a detach when the +// requested snapshot is already stale, and fail once the detach ends when a +// newer snapshot was committed while they waited. +func TestTrackerDetachWaitChecksGeneration(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + tr.CommitLocal(trackedActorUID, testRef("s1", 2, tr.clock.Now())) + detached := startDetach(t, tr) + waitStarted(t, store) + + stale := testRef("s0", 1, time.Time{}) + returnsPromptly(t, "BeginCheckpoint of stale generation during detach", func() { + if _, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, stale.Snapshot); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginCheckpoint = %v, want FailedPrecondition", err) + } + }) + returnsPromptly(t, "BeginPromote of stale generation during detach", func() { + if _, _, err := tr.BeginPromote(t.Context(), trackedActorUID, stale); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginPromote = %v, want FailedPrecondition", err) + } + }) + + superseded := testRef("s2", 3, time.Time{}) + results := make(map[string]chan error) + for name, begin := range map[string]func() (func(), error){ + "beginCheckpoint": func() (func(), error) { + return tr.BeginCheckpoint(t.Context(), trackedActorUID, superseded.Snapshot) + }, + "beginPromote": func() (func(), error) { + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, superseded) + return end, err + }, + } { + result := make(chan error, 1) + results[name] = result + go func() { + end, err := begin() + if err == nil { + end() + } + result <- err + }() + } + for name, result := range results { + select { + case err := <-result: + t.Fatalf("%s returned (%v) during a detach", name, err) + case <-time.After(50 * time.Millisecond): + } + } + + tr.CommitDurable(trackedActorUID, testRef("s3", 4, tr.clock.Now())) + store.release <- nil + if err := <-detached; err != nil { + t.Fatalf("detach: %v", err) + } + for name, result := range results { + select { + case err := <-result: + if apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("%s = %v, want FailedPrecondition", name, err) + } + case <-time.After(10 * time.Second): + t.Fatalf("%s did not return after the detach", name) + } + } +} + // A failed detach keeps the local snapshot, and is retried no sooner than the // retry delay after it, nor than it was due. The upload is held back the retry // delay too. @@ -618,11 +693,12 @@ func TestTrackerDetachWithoutStore(t *testing.T) { func TestTrackerHoldsFirstCheckpoint(t *testing.T) { store := newFakeSnapshotStore() tr := newTestTracker(t, store) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID) + s1 := testRef("s1", 1, tr.clock.Now()) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, s1.Snapshot) if err != nil { t.Fatalf("beginCheckpoint: %v", err) } - tr.CommitLocal(trackedActorUID, testRef("s1", 1, tr.clock.Now())) + tr.CommitLocal(trackedActorUID, s1) if _, err := tr.Detach(t.Context(), trackedActorUID); apierror.Code(err) != codes.Aborted { t.Errorf("detach during the checkpoint = %v, want Aborted", err) } diff --git a/cmd/atelet/internal/snapshottracker/lifecycle.go b/cmd/atelet/internal/snapshottracker/lifecycle.go index 52302a3636..7fca5f4179 100644 --- a/cmd/atelet/internal/snapshottracker/lifecycle.go +++ b/cmd/atelet/internal/snapshottracker/lifecycle.go @@ -16,23 +16,30 @@ package snapshottracker import ( "context" + "slices" "time" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/proto/ateletpb" + "google.golang.org/protobuf/proto" ) // CommitLocal records snap, just stored in the actor's local checkpoint // directory by a Checkpoint in either upload mode, as the actor's latest local // snapshot, schedules the actor's detach and, if snap needs an upload (see -// uploadDue), its upload, and returns the actor's state. An older snapshot's -// upload in progress runs on, but an Uploading that keepUploading no longer -// keeps is dropped. The Checkpoint then prunes the local snapshots -// LocalSnapshotsToKeep no longer names. +// uploadDue), its upload, and returns the actor's state. A commit of a +// snapshot already recorded or superseded by a newer committed snapshot leaves +// the state as it is. An older snapshot's upload in progress runs on, but an +// Uploading that keepUploading no longer keeps is dropped. The Checkpoint then +// prunes the local snapshots LocalSnapshotsToKeep no longer names. func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) + if !a.localNeeded(snap) { + t.requeueLocked(a) + return t.stateLocked(a) + } t.setLatestLocalLocked(a, snap) if a.UndurableSince.IsZero() && a.uploadNeeded(snap) { a.UndurableSince = snap.CommitTime @@ -54,21 +61,26 @@ func (t *Tracker) CommitLocal(actorUID string, snap SnapshotRef) *ateletpb.Actor // CommitDurable records snap, just uploaded by a Checkpoint under // UPLOAD_MODE_IMMEDIATE, as the actor's latest durable snapshot, and returns -// the actor's state. It drops an older LatestLocal, which snap supersedes, so -// the Checkpoint's prune deletes it even if no local copy of snap is kept. The -// Checkpoint then records its local copy with CommitLocal, if it manages to -// keep one. An older snapshot's upload in progress runs on, and removes that -// snapshot's local copy when it ends (see removeSupersededLocalCopy). +// the actor's state. A commit of a snapshot already durable or older than +// LatestDurable leaves the state as it is. It drops an older LatestLocal, +// which snap supersedes, so the Checkpoint's prune deletes it even if no local +// copy of snap is kept. The Checkpoint then records its local copy with +// CommitLocal, if it manages to keep one. An older snapshot's upload in +// progress runs on, and removes that snapshot's local copy when it ends (see +// removeSupersededLocalCopy). func (t *Tracker) CommitDurable(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) + if !a.uploadNeeded(snap) { + t.requeueLocked(a) + return t.stateLocked(a) + } a.LatestDurable = snap.clone() - a.UndurableSince = time.Time{} if a.LatestLocal != nil && a.LatestLocal.generation() < snap.generation() { a.clearLatestLocal() } - t.succeededLocked(a) + t.durableCommittedLocked(a) return t.stateLocked(a) } @@ -96,28 +108,29 @@ func (t *Tracker) LocalSnapshotsToKeep(actorUID string) []string { // CommitPromoted records snap, a local snapshot PromoteSnapshot made durable or // found durable already, as the actor's latest durable snapshot, keeping its -// local copy and the time its checkpoint committed. It leaves DetachAt as it -// is. A retried promote of a snapshot already recorded keeps what was -// recorded. It returns the actor's state. +// local copy, the time its checkpoint committed, and any UploadTime on snap. +// It leaves DetachAt as it is. A promote of a snapshot already durable, or +// older than LatestDurable, keeps what was recorded. It returns the actor's +// state. func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.ActorSnapshotState { t.lock(actorUID) defer t.unlock(actorUID) a := t.findOrAddLocked(actorUID) - if !sameUUID(a.LatestDurable, &snap) { - if sameUUID(a.LatestLocal, &snap) { - snap.CommitTime = a.LatestLocal.CommitTime - // The local copy takes the promoted Snapshot, which names where - // it is durable. - a.setLatestLocal(snap, a.DetachAt) - } - a.LatestDurable = snap.clone() + // Skip when snap is already LatestDurable (so a retried promote keeps the + // original CommitTime and UploadTime) or older than LatestDurable (a + // newer checkpoint became durable while PromoteSnapshot ran). + if !a.uploadNeeded(snap) { + t.requeueLocked(a) + return t.stateLocked(a) } - // The promote made LatestLocal durable, unless a newer checkpoint - // superseded the promoted snapshot. - if a.LatestLocal == nil || !a.uploadNeeded(*a.LatestLocal) { - a.UndurableSince = time.Time{} + // Carry over the checkpoint commit time when promoting the tracked + // local snapshot. snap.UploadTime is already set by the caller when + // it uploaded the snapshot. + if sameUUID(a.LatestLocal, &snap) { + snap.CommitTime = a.LatestLocal.CommitTime } - t.succeededLocked(a) + a.LatestDurable = snap.clone() + t.durableCommittedLocked(a) return t.stateLocked(a) } @@ -134,7 +147,7 @@ func (t *Tracker) CommitPromoted(actorUID string, snap SnapshotRef) *ateletpb.Ac func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) - a, err := t.waitForDetachLocked(ctx, actorUID) + a, err := t.waitForDetachLocked(ctx, actorUID, nil) if err != nil { return nil, err } @@ -155,18 +168,22 @@ func (t *Tracker) BeginStart(ctx context.Context, actorUID string) (end func(), t.resetBackoffLocked(a) } } - return t.holdLocked(actorUID, a, false), nil + return t.holdLocked(actorUID, a, nil, false), nil } -// BeginCheckpoint records that a Checkpoint of the actor is in progress on this +// BeginCheckpoint records that a Checkpoint of snap is in progress on this // node, once any detach in progress has finished, and returns the func to call -// when the checkpoint ends, whether or not it succeeded. atelet does not -// detach the actor in between, so a detach cannot delete the local snapshot -// the checkpoint writes. -func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string) (end func(), err error) { +// when the checkpoint ends, whether or not it succeeded. It fails with +// FailedPrecondition when snap has a stale generation for the actor: below the +// actor's newest snapshot on this node, or equal to it under another +// snapshot_uuid. atelet does not detach the actor in between, so a detach +// cannot delete the local snapshot the checkpoint writes. +func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string, snap *ateletpb.Snapshot) (end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) - a, err := t.waitForDetachLocked(ctx, actorUID) + a, err := t.waitForDetachLocked(ctx, actorUID, func(a *actorSnapshots) error { + return a.checkGeneration(snap) + }) if err != nil { return nil, err } @@ -174,14 +191,16 @@ func (t *Tracker) BeginCheckpoint(ctx context.Context, actorUID string) (end fun // zero, because the actor's Run or Restore on this node unscheduled the // detach, or this is a retry of a Checkpoint that already committed its // snapshot, whose detach stays scheduled. - return t.holdLocked(actorUID, a, false), nil + return t.holdLocked(actorUID, a, &SnapshotRef{Snapshot: proto.CloneOf(snap)}, false), nil } // BeginPromote records that a PromoteSnapshot of snap is in progress on this // node, once any detach in progress and any background upload of snap have -// finished. It reports whether snap is already the actor's latest durable -// snapshot, and returns the func to call when the promote ends. atelet does -// not start a detach or a background upload of the actor in between. +// finished. It fails with FailedPrecondition when snap has a stale generation +// for the actor or conflicts with LatestDurable. It reports whether snap is +// already the actor's latest durable snapshot, and returns the func to call +// when the promote ends. atelet does not start a detach or a background upload +// of the actor in between. func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap SnapshotRef) (alreadyDurable bool, end func(), err error) { t.lock(actorUID) defer t.unlock(actorUID) @@ -195,21 +214,25 @@ func (t *Tracker) BeginPromote(ctx context.Context, actorUID string, snap Snapsh return false, nil, err } } - return alreadyDurable, t.holdLocked(actorUID, a, true), nil + return alreadyDurable, t.holdLocked(actorUID, a, snap.clone(), true), nil } // holdLocked counts a Run, Restore, Checkpoint, or PromoteSnapshot of the // actor in progress on its entry a, which holds detaches of the actor off (and // background uploads when promote is true), and returns the func that ends it -// and queues the actor again. An actor atelet tracks nothing of yet (a nil a) -// is held too, so no detach can begin on a snapshot its first Checkpoint -// commits before it ends. -func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, promote bool) (end func()) { +// and queues the actor again. When snap is non-nil (Checkpoint and +// PromoteSnapshot), it stays in a.inFlight until end runs, so newest includes +// it. An actor atelet tracks nothing of yet (a nil a) is held too, so no +// detach can begin on a snapshot its first Checkpoint commits before it ends. +func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, snap *SnapshotRef, promote bool) (end func()) { if a == nil { a = &actorSnapshots{actorUID: actorUID} } t.addLocked(a) a.holds++ + if snap != nil { + a.inFlight = append(a.inFlight, snap) + } if promote { a.promotes++ } @@ -217,6 +240,9 @@ func (t *Tracker) holdLocked(actorUID string, a *actorSnapshots, promote bool) ( t.lock(actorUID) defer t.unlock(actorUID) a.holds-- + if snap != nil { + a.inFlight = slices.DeleteFunc(a.inFlight, func(r *SnapshotRef) bool { return r == snap }) + } if promote { a.promotes-- } @@ -260,10 +286,23 @@ func (t *Tracker) clearLatestLocalLocked(a *actorSnapshots) { t.resetBackoffLocked(a) } +// durableCommittedLocked finishes CommitDurable or CommitPromoted once +// LatestDurable is updated. When LatestLocal is now durable too (or nil), it +// clears UndurableSince and lifts any retry backoff; when a newer LatestLocal +// still needs uploading, its UndurableSince and backoff stay in place. +func (t *Tracker) durableCommittedLocked(a *actorSnapshots) { + if a.LatestLocal == nil || !a.uploadNeeded(*a.LatestLocal) { + a.UndurableSince = time.Time{} + t.resetBackoffLocked(a) + } + t.saveLocked(a) + t.requeueLocked(a) +} + // succeededLocked finishes a change that leaves the actor in a successful end -// state, such as a new local checkpoint or a durable latest snapshot: it -// starts the actor's backoff over, saves the actor's state, and queues the -// actor for its next background work. +// state, such as a background upload or detach: it starts the actor's backoff +// over, saves the actor's state, and queues the actor for its next background +// work. func (t *Tracker) succeededLocked(a *actorSnapshots) { t.resetBackoffLocked(a) t.saveLocked(a) diff --git a/cmd/atelet/internal/snapshottracker/queues_test.go b/cmd/atelet/internal/snapshottracker/queues_test.go index 35a4456fb4..9c626d62ee 100644 --- a/cmd/atelet/internal/snapshottracker/queues_test.go +++ b/cmd/atelet/internal/snapshottracker/queues_test.go @@ -183,7 +183,7 @@ func TestTrackerUploadRunsDuringCheckpointHoldWhileDetachWaits(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID) + endCheckpoint, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, time.Time{}).Snapshot) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } diff --git a/cmd/atelet/internal/snapshottracker/tracker.go b/cmd/atelet/internal/snapshottracker/tracker.go index df96737609..c60db5f7dd 100644 --- a/cmd/atelet/internal/snapshottracker/tracker.go +++ b/cmd/atelet/internal/snapshottracker/tracker.go @@ -38,8 +38,10 @@ type Store interface { } // Tracker tracks each actor's snapshots on this node: the latest one stored -// locally, and the latest one confirmed in object storage. State reports them. -// It uploads and detaches them in the background. +// locally, and the latest one confirmed in object storage. State reports them, +// BeginCheckpoint and BeginPromote reject a stale generation before Checkpoint +// or PromoteSnapshot does any work, and commits never replace a snapshot with +// an older generation. It uploads and detaches them in the background. // // Background work runs off two delaying workqueues of actor UIDs, one for // uploads and one for detaches, each drained by a fixed pool of workers of its @@ -87,11 +89,6 @@ type Store interface { // in progress; and PromoteSnapshot waits for a detach or a background upload // of the same snapshot. A worker leaves an actor with an operation in // progress to that operation's end, which queues the actor again. -// -// TODO: Commits record the snapshot they are given without comparing -// Snapshot.generation. Checkpoint and PromoteSnapshot should reject a stale -// generation before doing any work, as Snapshot.generation describes, so that -// no commit replaces a snapshot with an older one. type Tracker struct { // ctx bounds background work, and shuts the queues down when it ends. ctx context.Context diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index a778d254a7..0d7cbd9133 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -17,6 +17,7 @@ package snapshottracker import ( "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -26,9 +27,11 @@ import ( "time" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" + "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/nodepath" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/resources" + "google.golang.org/grpc/codes" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/client-go/util/workqueue" @@ -435,6 +438,16 @@ func TestTrackerCommitPromoted(t *testing.T) { if got := st.GetLatestDurable(); uuidOf(got) != "s3" || got.GetLocalCommitTime() != nil || uuidOf(st.GetLatestLocal()) != "s2" { t.Errorf("state = %v, want s3 durable without a commit time, and s2 still local", st) } + + // An older promote finishing after a newer checkpoint keeps the newer + // durable and local snapshots. + tr.CommitLocal(trackedActorUID, testRef("s4", 4, t0.Add(4*time.Hour))) + older := testRef("s2", 2, time.Time{}) + older.UploadTime = t0.Add(5 * time.Hour) + st = tr.CommitPromoted(trackedActorUID, older) + if uuidOf(st.GetLatestDurable()) != "s3" || uuidOf(st.GetLatestLocal()) != "s4" { + t.Errorf("state after older promote = %v, want s3 durable and s4 local", st) + } } // A promote of a paused actor's local snapshot leaves its detach schedule as @@ -444,9 +457,7 @@ func TestTrackerCommitPromotedKeepsDetachSchedule(t *testing.T) { for _, started := range []bool{false, true} { t.Run(fmt.Sprintf("started=%v", started), func(t *testing.T) { tr := newIdleTestTracker(t, newFakeSnapshotStore()) - // A pause snapshot names no object storage location. paused := testRef("s1", 1, tr.clock.Now()) - paused.Snapshot.Object = nil tr.CommitLocal(trackedActorUID, paused) want := tr.clock.Now().Add(testDetachDelay) if started { @@ -468,9 +479,11 @@ func TestTrackerCommitPromotedKeepsDetachSchedule(t *testing.T) { // Promoting the latest local snapshot makes it durable and clears its upload // due time, whereas promoting an older snapshot while newer local snapshots -// still need an upload keeps the upload due when the oldest of them was. +// still need an upload keeps the upload due when the oldest of them was, +// including any retry backoff. func TestTrackerCommitPromotedClearsUploadDue(t *testing.T) { - tr := newIdleTestTracker(t, newFakeSnapshotStore()) + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) t0 := tr.clock.Now() st := tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) if st.GetUploadDueTime() == nil { @@ -486,11 +499,46 @@ func TestTrackerCommitPromotedClearsUploadDue(t *testing.T) { t2 := t0.Add(2 * time.Minute) tr.CommitLocal(trackedActorUID, testRef("s2", 2, t2)) tr.CommitLocal(trackedActorUID, testRef("s3", 3, t2.Add(time.Minute))) + tr.CommitLocal(trackedActorUID, testRef("s4", 4, t2.Add(2*time.Minute))) + older := testRef("s0", 0, time.Time{}) - older.UploadTime = t2.Add(2 * time.Minute) + older.UploadTime = t2.Add(3 * time.Minute) st = tr.CommitPromoted(trackedActorUID, older) + if got := uuidOf(st.GetLatestDurable()); got != "s1" { + t.Errorf("latest_durable after promoting below durable = %q, want s1", got) + } + if got, want := timeOf(st.GetUploadDueTime()), t2.Add(testUploadDelay); !got.Equal(want) { + t.Errorf("upload_due_time after promoting below durable = %v, want %v: an upload delay after s2", got, want) + } + + between := testRef("s2", 2, time.Time{}) + between.UploadTime = t2.Add(3 * time.Minute) + st = tr.CommitPromoted(trackedActorUID, between) + if got := uuidOf(st.GetLatestDurable()); got != "s2" { + t.Errorf("latest_durable after promoting s2 = %q, want s2", got) + } if got, want := timeOf(st.GetUploadDueTime()), t2.Add(testUploadDelay); !got.Equal(want) { - t.Errorf("upload_due_time after promoting older snapshot = %v, want %v: an upload delay after s2", got, want) + t.Errorf("upload_due_time after promoting between durable and local = %v, want %v: an upload delay after s2", got, want) + } + + stepTo(tr, t2.Add(testUploadDelay)) + store.setErr(errors.New("boom")) + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the store's error") + } + wantRetry := tr.clock.Now().Add(testRetryDelay) + st = tr.CommitPromoted(trackedActorUID, older) + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after promoting below durable during backoff = %v, want %v", got, wantRetry) + } + betweenBackoff := testRef("s3", 3, time.Time{}) + betweenBackoff.UploadTime = tr.clock.Now() + st = tr.CommitPromoted(trackedActorUID, betweenBackoff) + if got := uuidOf(st.GetLatestDurable()); got != "s3" { + t.Errorf("latest_durable after promoting s3 during backoff = %q, want s3", got) + } + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after promoting between durable and local during backoff = %v, want %v", got, wantRetry) } } @@ -542,11 +590,8 @@ func TestTrackerDropsActorWithoutDirectory(t *testing.T) { } } -// Commits for one actor take turns, so a racing deferred checkpoint never -// loses an immediate checkpoint's durable snapshot. The control plane never -// sends one actor's checkpoints concurrently: the race only exercises the -// lock, so either commit may land last, even leaving a local snapshot older -// than the durable one. +// Commits for one actor take turns, and an older local commit never replaces +// a newer durable snapshot. func TestTrackerSerializesEachActor(t *testing.T) { useTempNodeDirs(t) if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { @@ -561,9 +606,13 @@ func TestTrackerSerializesEachActor(t *testing.T) { wg.Go(func() { tr.CommitLocal(trackedActorUID, local) }) wg.Go(func() { tr.CommitDurable(trackedActorUID, durable) }) wg.Wait() - if got, want := uuidOf(tr.State(trackedActorUID).GetLatestDurable()), durable.Snapshot.GetSnapshotUuid(); got != want { + st := tr.State(trackedActorUID) + if got, want := uuidOf(st.GetLatestDurable()), durable.Snapshot.GetSnapshotUuid(); got != want { t.Fatalf("round %d: latest_durable = %q, want %q", i, got, want) } + if got := st.GetLatestLocal(); got != nil { + t.Fatalf("round %d: latest_local = %v, want nil when durable has a higher generation", i, got) + } } if len(tr.actors) != 0 { t.Errorf("in-memory actors = %d, want 0 once every commit is written", len(tr.actors)) @@ -603,3 +652,220 @@ func TestTrackerSkipsUnreadableSnapshotState(t *testing.T) { t.Errorf("state with a broken file = %v, want empty", got) } } + +// BeginCheckpoint and BeginPromote accept any generation when atelet holds no +// snapshot for the actor, and otherwise reject a generation below the actor's +// newest snapshot or equal to it under another snapshot_uuid with +// FailedPrecondition. +func TestTrackerCheckGeneration(t *testing.T) { + useTempNodeDirs(t) + if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil { + t.Fatal(err) + } + tr := newTracker(t) + t0 := time.Now() + ref := func(uuid string, generation int32) SnapshotRef { + return testRef(uuid, generation, t0) + } + snap := func(uuid string, generation int32) *ateletpb.Snapshot { + return ref(uuid, generation).Snapshot + } + checkBegin := func(t *testing.T, tr *Tracker, r SnapshotRef, wantCode codes.Code) { + t.Helper() + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, r.Snapshot) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginCheckpoint(%v) code = %v, want %v", r.Snapshot, got, wantCode) + } + _, end, err = tr.BeginPromote(t.Context(), trackedActorUID, r) + if err == nil { + end() + } + if got := apierror.Code(err); got != wantCode { + t.Errorf("BeginPromote(%v) code = %v, want %v", r.Snapshot, got, wantCode) + } + } + + checkBegin(t, tr, ref("s1", 1), codes.OK) + + // Only a local snapshot is tracked (generation 2). + tr.CommitLocal(trackedActorUID, ref("s1", 2)) + for _, tc := range []struct { + name string + ref SnapshotRef + wantCode codes.Code + }{ + {name: "same snapshot at newest generation", ref: ref("s1", 2), wantCode: codes.OK}, + {name: "higher generation", ref: ref("s2", 3), wantCode: codes.OK}, + {name: "generation below newest", ref: ref("s0", 1), wantCode: codes.FailedPrecondition}, + {name: "same uuid at lower generation", ref: ref("s1", 1), wantCode: codes.FailedPrecondition}, + {name: "different uuid at newest generation", ref: ref("other", 2), wantCode: codes.FailedPrecondition}, + } { + t.Run("localNewest/"+tc.name, func(t *testing.T) { + checkBegin(t, tr, tc.ref, tc.wantCode) + checkBegin(t, newTracker(t), tc.ref, tc.wantCode) + }) + } + + // Only a durable snapshot is tracked (generation 4); CommitDurable drops + // the older local snapshot. + tr.CommitDurable(trackedActorUID, ref("s2", 4)) + for _, tc := range []struct { + name string + ref SnapshotRef + wantCode codes.Code + }{ + {name: "same snapshot at newest generation", ref: ref("s2", 4), wantCode: codes.OK}, + {name: "higher generation", ref: ref("s3", 5), wantCode: codes.OK}, + {name: "generation below newest", ref: ref("s1", 3), wantCode: codes.FailedPrecondition}, + {name: "different uuid at newest generation", ref: ref("other", 4), wantCode: codes.FailedPrecondition}, + } { + t.Run("durableNewest/"+tc.name, func(t *testing.T) { + checkBegin(t, tr, tc.ref, tc.wantCode) + }) + } + + // Both local (generation 6) and durable (generation 4) are tracked: local is newest. + tr.CommitLocal(trackedActorUID, ref("s3", 6)) + checkBegin(t, tr, ref("s2", 4), codes.FailedPrecondition) + checkBegin(t, tr, ref("s3", 6), codes.OK) + + // While a Checkpoint or PromoteSnapshot holds an uncommitted snapshot at + // generation 8, it is newest until end() runs. + endCp, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, snap("s4", 8)) + if err != nil { + t.Fatalf("BeginCheckpoint(s4@8): %v", err) + } + checkBegin(t, tr, ref("s3", 6), codes.FailedPrecondition) + checkBegin(t, tr, ref("between", 7), codes.FailedPrecondition) + checkBegin(t, tr, ref("other", 8), codes.FailedPrecondition) + checkBegin(t, tr, ref("s4", 8), codes.OK) + checkBegin(t, tr, ref("s5", 9), codes.OK) + endCp() + // Ending without committing drops the in-flight generation, so s3@6 is newest again. + checkBegin(t, tr, ref("s3", 6), codes.OK) + checkBegin(t, tr, ref("between", 7), codes.OK) + + _, endPromote, err := tr.BeginPromote(t.Context(), trackedActorUID, ref("s4", 8)) + if err != nil { + t.Fatalf("BeginPromote(s4@8): %v", err) + } + checkBegin(t, tr, ref("between", 7), codes.FailedPrecondition) + checkBegin(t, tr, ref("other", 8), codes.FailedPrecondition) + checkBegin(t, tr, ref("s4", 8), codes.OK) + endPromote() + checkBegin(t, tr, ref("between", 7), codes.OK) + + // If durable has a higher generation than local, durable is newest. + a := &actorSnapshots{ + actorUID: trackedActorUID, + actorSnapshotsRecord: actorSnapshotsRecord{ + LatestLocal: ref("local", 3).clone(), + LatestDurable: ref("durable", 5).clone(), + }, + } + if got := apierror.Code(a.checkGeneration(snap("local", 3))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older local@3) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 5))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@5) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("durable", 5))); got != codes.OK { + t.Errorf("checkGeneration(durable@5) code = %v, want OK", got) + } + + // If Uploading holds a newer snapshot, it is newest even while no op runs. + a.Uploading = ref("saved-uploading", 6).clone() + if got := apierror.Code(a.checkGeneration(snap("durable", 5))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older durable@5) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 6))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@6) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("saved-uploading", 6))); got != codes.OK { + t.Errorf("checkGeneration(saved-uploading@6) code = %v, want OK", got) + } + + // If an operation in progress acts on a newer snapshot, its target is newest. + a.op = &actorOp{target: ref("uploading", 7)} + if got := apierror.Code(a.checkGeneration(snap("saved-uploading", 6))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(older saved-uploading@6) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("other", 7))); got != codes.FailedPrecondition { + t.Errorf("checkGeneration(other@7) code = %v, want FailedPrecondition", got) + } + if got := apierror.Code(a.checkGeneration(snap("uploading", 7))); got != codes.OK { + t.Errorf("checkGeneration(uploading@7) code = %v, want OK", got) + } +} + +// CommitLocal and CommitDurable skip older generations, and an older +// CommitDurable leaves a newer LatestLocal's upload schedule and backoff intact. +func TestTrackerCommitsSkipOlderGenerations(t *testing.T) { + store := newFakeSnapshotStore() + tr := newIdleTestTracker(t, store) + t0 := tr.clock.Now() + + st := tr.CommitLocal(trackedActorUID, testRef("s3", 3, t0)) + for _, stale := range []SnapshotRef{ + testRef("s2", 2, t0.Add(time.Minute)), + testRef("other", 3, t0.Add(time.Minute)), + testRef("s3", 3, t0.Add(time.Minute)), + } { + got := tr.CommitLocal(trackedActorUID, stale) + if !proto.Equal(got, st) { + t.Errorf("CommitLocal(%v) = %v, want unchanged %v", stale.Snapshot, got, st) + } + if keep := tr.LocalSnapshotsToKeep(trackedActorUID); !slices.Equal(keep, []string{"s3"}) { + t.Errorf("LocalSnapshotsToKeep after CommitLocal(%v) = %v, want [s3]", stale.Snapshot, keep) + } + } + + s4 := testRef("s4", 4, t0.Add(2*time.Minute)) + s4.UploadTime = t0.Add(2 * time.Minute) + st = tr.CommitDurable(trackedActorUID, s4) + for _, stale := range []SnapshotRef{ + testRef("s2", 2, t0.Add(3*time.Minute)), + testRef("other", 4, t0.Add(3*time.Minute)), + testRef("s4", 4, t0.Add(3*time.Minute)), + } { + if got := tr.CommitDurable(trackedActorUID, stale); !proto.Equal(got, st) { + t.Errorf("CommitDurable(%v) = %v, want unchanged %v", stale.Snapshot, got, st) + } + } + for _, staleLocal := range []SnapshotRef{ + testRef("s3", 3, t0.Add(3*time.Minute)), + testRef("other", 4, t0.Add(3*time.Minute)), + } { + if got := tr.CommitLocal(trackedActorUID, staleLocal); !proto.Equal(got, st) { + t.Errorf("CommitLocal(%v) below durable = %v, want unchanged %v", staleLocal.Snapshot, got, st) + } + } + + // CommitLocal(s6) sets UndurableSince; CommitDurable(s5) between LatestDurable + // (s4) and LatestLocal (s6) advances LatestDurable without clearing + // LatestLocal, UndurableSince, or retry backoff. + t6 := t0.Add(4 * time.Minute) + tr.CommitLocal(trackedActorUID, testRef("s6", 6, t6)) + stepTo(tr, t6.Add(testUploadDelay)) + store.setErr(errors.New("boom")) + if _, err := tr.Detach(t.Context(), trackedActorUID); err == nil { + t.Fatal("detach succeeded, want the store's error") + } + wantRetry := tr.clock.Now().Add(testRetryDelay) + + s5 := testRef("s5", 5, t6.Add(time.Minute)) + s5.UploadTime = tr.clock.Now() + st = tr.CommitDurable(trackedActorUID, s5) + if got := uuidOf(st.GetLatestDurable()); got != "s5" { + t.Errorf("latest_durable after CommitDurable(s5) = %q, want s5", got) + } + if got := uuidOf(st.GetLatestLocal()); got != "s6" { + t.Errorf("latest_local after CommitDurable(s5) = %q, want s6", got) + } + if got := timeOf(st.GetUploadDueTime()); !got.Equal(wantRetry) { + t.Errorf("upload_due_time after CommitDurable(s5) during backoff = %v, want %v", got, wantRetry) + } +} diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go index 1dd78c014f..995a8431dc 100644 --- a/cmd/atelet/internal/snapshottracker/upload_test.go +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -534,7 +534,9 @@ func TestTrackerStartAndCheckpointDoNotWaitForUpload(t *testing.T) { begin func(context.Context, string) (func(), error) }{ {"BeginStart", tr.BeginStart}, - {"BeginCheckpoint", tr.BeginCheckpoint}, + {"BeginCheckpoint", func(ctx context.Context, uid string) (func(), error) { + return tr.BeginCheckpoint(ctx, uid, testRef("s2", 2, time.Time{}).Snapshot) + }}, } { returnsPromptly(t, tc.name+" during a background upload", func() { end, err := tc.begin(t.Context(), trackedActorUID) @@ -559,7 +561,7 @@ func TestTrackerUploadRunsWhileActorHeld(t *testing.T) { t0 := tr.clock.Now() tr.CommitLocal(trackedActorUID, testRef("s1", 1, t0)) - end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID) + end, err := tr.BeginCheckpoint(t.Context(), trackedActorUID, testRef("s2", 2, t0).Snapshot) if err != nil { t.Fatalf("BeginCheckpoint: %v", err) } @@ -649,6 +651,36 @@ func TestTrackerBeginPromote(t *testing.T) { } }) + t.Run("rejects snapshot superseded during upload", func(t *testing.T) { + store := newFakeSnapshotStore() + store.setBlocking(true) + tr := newTestTracker(t, store) + s1 := testRef("s1", 1, tr.clock.Now()) + tr.CommitLocal(trackedActorUID, s1) + done := startUpload(t, tr, store) + + errCh := make(chan error, 1) + go func() { + _, end, err := tr.BeginPromote(t.Context(), trackedActorUID, s1) + if err == nil { + end() + } + errCh <- err + }() + select { + case err := <-errCh: + t.Fatalf("BeginPromote returned (%v) during an upload of the same snapshot", err) + case <-time.After(50 * time.Millisecond): + } + + tr.CommitLocal(trackedActorUID, testRef("s2", 2, tr.clock.Now())) + store.release <- nil + waitClosed(t, done, "the upload") + if err := <-errCh; apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("BeginPromote = %v, want FailedPrecondition", err) + } + }) + t.Run("does not wait for upload of older snapshot", func(t *testing.T) { store := newFakeSnapshotStore() store.setBlocking(true) diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 363cbcaf8a..95b50d4852 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -83,6 +83,9 @@ type fakeWorker struct { // fidelities records the fidelity each checkpoint or restore arrived // with, by RPC name. fidelities map[string]ateworkerpb.SnapshotFidelity + // onCheckpoint, if non-nil, runs inside CheckpointWorkload after snapshot + // files are written. + onCheckpoint func() } func (f *fakeWorker) recordFidelity(rpc string, fidelity ateworkerpb.SnapshotFidelity) { @@ -119,6 +122,9 @@ func (f *fakeWorker) CheckpointWorkload(_ context.Context, req *ateworkerpb.Chec } names = append(names, name) } + if f.onCheckpoint != nil { + f.onCheckpoint() + } return &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: names}, nil } @@ -1984,3 +1990,195 @@ func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { t.Errorf("pause-snap-2 after the upload of pause-snap-1: stat = %v, want it kept", err) } } + +// TestCheckpointRejectsStaleGeneration checks that Checkpoint fails with +// FailedPrecondition when given a generation below the actor's newest snapshot +// or equal to it under another snapshot_uuid, before calling worker, pruning +// local checkpoints, or changing tracked state. +func TestCheckpointRejectsStaleGeneration(t *testing.T) { + a := runPluginActor(t) + snap2 := a.snapshot("pause-snap-2", 2) + resp, err := a.checkpoint(t, snap2, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint: %v", err) + } + wantState := resp.GetState() + a.restoreLocal(t, "pause-snap-2", 2) + delete(a.worker.actorDirs, "CheckpointWorkload") + + for _, stale := range []*ateletpb.Snapshot{ + a.snapshot("stale-snap-1", 1), + a.snapshot("other-snap-2", 2), + } { + _, err := a.checkpoint(t, stale, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if code := apierror.Code(err); code != codes.FailedPrecondition { + t.Errorf("Checkpoint(%v) error = %v (code %v), want FailedPrecondition", stale, err, code) + } + if got := a.worker.actorDirs["CheckpointWorkload"]; got != nil { + t.Errorf("Checkpoint(%v) called worker.CheckpointWorkload (%v), want rejected before worker", stale, got) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); err != nil { + t.Errorf("local snapshot pause-snap-2 after rejected Checkpoint(%v): stat = %v, want preserved", stale, err) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, wantState) { + t.Errorf("tracked state after rejected Checkpoint(%v) = %v, want %v", stale, got, wantState) + } + } +} + +// TestPromoteSnapshotRejectsStaleGeneration checks that PromoteSnapshot fails +// with FailedPrecondition when given a generation below the actor's newest +// snapshot or equal to it under another snapshot_uuid, before uploading to +// object storage, pruning local checkpoints, or changing tracked state. +func TestPromoteSnapshotRejectsStaleGeneration(t *testing.T) { + a := runPluginActor(t) + snap2 := a.snapshot("pause-snap-2", 2) + resp, err := a.checkpoint(t, snap2, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + if err != nil { + t.Fatalf("Checkpoint: %v", err) + } + wantState := resp.GetState() + + for _, stale := range []*ateletpb.Snapshot{ + a.snapshot("stale-snap-1", 1), + a.snapshot("other-snap-2", 2), + } { + writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(pluginActorUID, stale.GetSnapshotUuid()), sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: a.assets.GetPauseImage(), + SnapshotFiles: []string{"checkpoint.img"}, + Fidelity: "MEMORY", + }, map[string]string{"checkpoint.img": "stale-memory"}) + + _, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: stale, + }) + if code := apierror.Code(err); code != codes.FailedPrecondition { + t.Errorf("PromoteSnapshot(%v) error = %v (code %v), want FailedPrecondition", stale, err, code) + } + if got := a.store.keys(); len(got) != 0 { + t.Errorf("objects after rejected PromoteSnapshot(%v) = %v, want nothing uploaded", stale, got) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); err != nil { + t.Errorf("local snapshot pause-snap-2 after rejected PromoteSnapshot(%v): stat = %v, want preserved", stale, err) + } + if got := a.s.snapshots.State(pluginActorUID); !proto.Equal(got, wantState) { + t.Errorf("tracked state after rejected PromoteSnapshot(%v) = %v, want %v", stale, got, wantState) + } + } +} + +// TestCheckpointInFlightRejectsOlderGeneration checks that while Checkpoint is +// running on the worker for generation 3, an out-of-order Checkpoint or +// PromoteSnapshot for generation 2 fails with FailedPrecondition. +func TestCheckpointInFlightRejectsOlderGeneration(t *testing.T) { + a := runPluginActor(t) + started := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + a.worker.onCheckpoint = func() { + once.Do(func() { + close(started) + <-release + }) + } + + cpErr := make(chan error, 1) + go func() { + _, err := a.checkpoint(t, a.snapshot("pause-snap-3", 3), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + cpErr <- err + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Checkpoint(pause-snap-3) to reach worker") + } + + stale := a.snapshot("pause-snap-2", 2) + if _, err := a.checkpoint(t, stale, ateletpb.UploadMode_UPLOAD_MODE_DEFERRED); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("Checkpoint(%v) while generation 3 is in flight = %v, want FailedPrecondition", stale, err) + } + if _, err := a.s.PromoteSnapshot(t.Context(), &ateletpb.PromoteSnapshotRequest{ + Atespace: "ate-demo", + ActorName: "counter", + ActorUid: pluginActorUID, + ActorTemplateAtespace: "default", + ActorTemplateName: "counter", + Snapshot: stale, + }); apierror.Code(err) != codes.FailedPrecondition { + t.Errorf("PromoteSnapshot(%v) while generation 3 is in flight = %v, want FailedPrecondition", stale, err) + } + + close(release) + if err := <-cpErr; err != nil { + t.Fatalf("Checkpoint(pause-snap-3): %v", err) + } + if got := a.s.snapshots.State(pluginActorUID).GetLatestLocal().GetSnapshot().GetSnapshotUuid(); got != "pause-snap-3" { + t.Errorf("latest_local = %q, want pause-snap-3", got) + } +} + +// TestCheckpointSkipsStaleCommitAndKeepsNewerLocal checks that when a stalled +// Checkpoint(gen=2) reaches CommitLocal and PruneLocalCheckpoints after a newer +// snapshot (gen=3) has already committed locally, gen=3 stays tracked and its +// local directory stays on disk while gen=2's local copy is pruned. +func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { + a := runPluginActor(t) + started := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + a.worker.onCheckpoint = func() { + once.Do(func() { + close(started) + <-release + }) + } + + type cpResult struct { + resp *ateletpb.CheckpointResponse + err error + } + resCh := make(chan cpResult, 1) + go func() { + resp, err := a.checkpoint(t, a.snapshot("pause-snap-2", 2), ateletpb.UploadMode_UPLOAD_MODE_DEFERRED) + resCh <- cpResult{resp, err} + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for Checkpoint(pause-snap-2) to reach worker") + } + + snap3 := a.snapshot("pause-snap-3", 3) + snap3Dir := ateletpath.LocalSnapshotDir(pluginActorUID, snap3.GetSnapshotUuid()) + writeLocalSnapshot(t, snap3Dir, sandboxAssetsRecord{ + SandboxClass: "gvisor", + PauseImage: a.assets.GetPauseImage(), + SnapshotFiles: []string{"checkpoint.img"}, + Fidelity: "MEMORY", + }, map[string]string{"checkpoint.img": "newer-memory"}) + wantState := a.s.snapshots.CommitLocal(pluginActorUID, snapshottracker.SnapshotRef{ + Snapshot: snap3, + CommitTime: time.Now(), + }) + + close(release) + res := <-resCh + if res.err != nil { + t.Fatalf("Checkpoint(pause-snap-2): %v", res.err) + } + if !proto.Equal(res.resp.GetState(), wantState) { + t.Errorf("Checkpoint(pause-snap-2) state = %v, want %v (keeping pause-snap-3)", res.resp.GetState(), wantState) + } + if _, err := os.Stat(snap3Dir); err != nil { + t.Errorf("newer local snapshot pause-snap-3: stat = %v, want kept", err) + } + if _, err := os.Stat(ateletpath.LocalSnapshotDir(pluginActorUID, "pause-snap-2")); !os.IsNotExist(err) { + t.Errorf("stale local snapshot pause-snap-2: stat = %v, want pruned", err) + } +} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index a3c12c4bea..7ae8c41c6f 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -879,7 +879,7 @@ func (s *Atelet) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest // A detach in progress finishes first, and none starts until the // checkpoint returns, so it cannot delete the snapshot being written. - endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID) + endCheckpoint, err := s.snapshots.BeginCheckpoint(ctx, actorUID, req.GetSnapshot()) if err != nil { return nil, err } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 0f2d82b221..23e74b729c 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -3153,8 +3153,6 @@ type Snapshot struct { // for the actor, or for its newest under another snapshot_uuid: the // caller's view is stale. // - // TODO: Temporarily, atelet does not compare it with the snapshots it holds. - // // +k8s:required // +k8s:minimum=1 Generation int32 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 860bf6d588..4e95f6ed67 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -901,8 +901,6 @@ message Snapshot { // for the actor, or for its newest under another snapshot_uuid: the // caller's view is stale. // - // TODO: Temporarily, atelet does not compare it with the snapshots it holds. - // // +k8s:required // +k8s:minimum=1 int32 generation = 2;