From a75c5e44ca67134fa06f6edcbc3e9e99441a0e78 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 9 Oct 2026 19:33:10 -0700 Subject: [PATCH 1/9] Folded Nexus operation progress onto the next Workflow Task. Each operation keeps its highest counter until a Workflow Task scheduled event carries it, and newer progress folds into a scheduled task that has not started, so a burst costs one task and writes no event of its own. A passive cluster forwards progress to the active one. --- chasm/lib/nexusoperation/operation.go | 69 ++++++- chasm/lib/nexusoperation/progress_test.go | 127 ++++++++++++ .../gen/workflowpb/v1/state.go-helpers.pb.go | 37 ++++ .../workflow/gen/workflowpb/v1/state.pb.go | 67 ++++++- chasm/lib/workflow/nexus_commands_test.go | 4 +- chasm/lib/workflow/nexus_events.go | 8 +- chasm/lib/workflow/nexus_methods.go | 3 +- chasm/lib/workflow/nexus_progress.go | 147 ++++++++++++++ chasm/lib/workflow/nexus_progress_test.go | 102 ++++++++++ chasm/lib/workflow/proto/v1/state.proto | 10 + chasm/lib/workflow/workflow.go | 4 + common/nexus/nexusrpc/completion.go | 11 +- common/nexus/nexusrpc/completion_test.go | 35 ++++ .../frontend/nexus_completion_http_handler.go | 10 +- .../history/workflow/mutable_state_impl.go | 3 +- service/history/workflow/nexus_progress.go | 89 +++++++++ .../workflow/workflow_task_state_machine.go | 16 ++ tests/nexus_progress_test.go | 184 ++++++++++++++++- tests/xdc/nexus_progress_forwarding_test.go | 187 ++++++++++++++++++ 19 files changed, 1086 insertions(+), 27 deletions(-) create mode 100644 chasm/lib/nexusoperation/progress_test.go create mode 100644 chasm/lib/workflow/nexus_progress.go create mode 100644 chasm/lib/workflow/nexus_progress_test.go create mode 100644 service/history/workflow/nexus_progress.go create mode 100644 tests/xdc/nexus_progress_forwarding_test.go diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index 03bb28ee9f7..f2c1da02a64 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -108,6 +108,22 @@ type Operation struct { // Callbacks holds completion callbacks to be invoked when this reaches a terminal state. // Keyed by completionCallbackID(requestID, index). Callbacks chasm.Map[string, *callback.Callback] + + // PendingProgress is the latest progress no Workflow Task has carried yet. + PendingProgress chasm.Field[*nexuspb.NexusOperationProgress] + // DeliveredProgress is the latest progress a Workflow Task carried or folded. A delivery whose + // counter is not above it is stale. + DeliveredProgress chasm.Field[*nexuspb.NexusOperationProgress] +} + +// ProgressStore is implemented by a parent that delivers the operation's progress to its caller. +// A parent without it, such as none for a standalone operation, drops progress. +type ProgressStore interface { + // OnNexusOperationProgress records that the operation holds pending progress. + OnNexusOperationProgress(ctx chasm.MutableContext, operation *Operation) error + // NexusOperationProgressFolds reports whether the operation's latest progress rides a Workflow + // Task that has not started, which newer progress can fold into. + NexusOperationProgressFolds(ctx chasm.Context, operation *Operation) bool } // NewOperation creates a new Operation component with the given persisted state. @@ -298,10 +314,9 @@ func (o *Operation) HandleNexusCompletion( return serviceerror.NewNotFound("operation not found") } - // Progress never stands in for the start, and nothing folds it onto a Workflow Task yet, so it - // is accepted and dropped here. - if completion.GetProgress() != nil { - return nil + // Progress never stands in for the start, so it is handled before completion-before-start. + if progress := completion.GetProgress(); progress != nil { + return o.onProgress(ctx, progress) } links := completion.GetLinks() @@ -329,6 +344,52 @@ func (o *Operation) HandleNexusCompletion( } } +// onProgress keeps the progress with the highest counter for the caller. Progress before the +// operation started is dropped, since it cannot stand in for the start response, and so is progress +// after it closed, since its completion supersedes it. +func (o *Operation) onProgress(ctx chasm.MutableContext, progress *nexuspb.NexusOperationProgress) error { + if o.Status != nexusoperationpb.OPERATION_STATUS_STARTED { + return nil + } + parent, ok := o.Store.TryGet(ctx) + if !ok { + return nil + } + store, ok := parent.(ProgressStore) + if !ok { + return nil + } + for _, seen := range []chasm.Field[*nexuspb.NexusOperationProgress]{o.PendingProgress, o.DeliveredProgress} { + if previous, ok := seen.TryGet(ctx); ok && previous.GetCounter() >= progress.GetCounter() { + return nil + } + } + if store.NexusOperationProgressFolds(ctx, o) { + o.DeliveredProgress = chasm.NewDataField(ctx, progress) + return nil + } + o.PendingProgress = chasm.NewDataField(ctx, progress) + return store.OnNexusOperationProgress(ctx, o) +} + +// TakePendingProgress returns the pending progress and records it as delivered. +func (o *Operation) TakePendingProgress(ctx chasm.MutableContext) (*nexuspb.NexusOperationProgress, bool) { + progress, ok := o.PendingProgress.TryGet(ctx) + if !ok { + return nil, false + } + o.DeliveredProgress = chasm.NewDataField(ctx, progress) + o.PendingProgress = chasm.NewEmptyField[*nexuspb.NexusOperationProgress]() + return progress, true +} + +// DropPendingProgress forgets the pending progress without delivering it. +func (o *Operation) DropPendingProgress(ctx chasm.MutableContext) { + if _, ok := o.PendingProgress.TryGet(ctx); ok { + o.PendingProgress = chasm.NewEmptyField[*nexuspb.NexusOperationProgress]() + } +} + // loadStartArgs is a ReadComponent callback that loads the start arguments from the operation. func (o *Operation) loadStartArgs( ctx chasm.Context, diff --git a/chasm/lib/nexusoperation/progress_test.go b/chasm/lib/nexusoperation/progress_test.go new file mode 100644 index 00000000000..c4dcb40109d --- /dev/null +++ b/chasm/lib/nexusoperation/progress_test.go @@ -0,0 +1,127 @@ +package nexusoperation + +import ( + "testing" + + "github.com/stretchr/testify/require" + nexuspb "go.temporal.io/api/nexus/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" +) + +// progressStoreComponent is a parent that takes progress, and can say that a scheduled task the +// operation's progress rides has not started. +type progressStoreComponent struct { + mockStoreComponent + + folds bool + pending int +} + +func (s *progressStoreComponent) OnNexusOperationProgress(chasm.MutableContext, *Operation) error { + s.pending++ + return nil +} + +func (s *progressStoreComponent) NexusOperationProgressFolds(chasm.Context, *Operation) bool { + return s.folds +} + +func TestHandleNexusProgress(t *testing.T) { + progress := func(counter int64) *persistencespb.ChasmNexusCompletion { + return &persistencespb.ChasmNexusCompletion{ + Outcome: &persistencespb.ChasmNexusCompletion_Progress{ + Progress: &nexuspb.NexusOperationProgress{Counter: counter}, + }, + } + } + newOp := func(status nexusoperationpb.OperationStatus, store OperationStore) *Operation { + op := newTestOperation() + op.Status = status + if store != nil { + op.Store = chasm.NewMockParentPtr(store) + } + return op + } + pendingCounter := func(ctx chasm.Context, op *Operation) int64 { + p, _ := op.PendingProgress.TryGet(ctx) + return p.GetCounter() + } + deliveredCounter := func(ctx chasm.Context, op *Operation) int64 { + p, _ := op.DeliveredProgress.TryGet(ctx) + return p.GetCounter() + } + + for _, tc := range []struct { + name string + status nexusoperationpb.OperationStatus + store OperationStore + counters []int64 + wantPending int64 + }{ + {name: "HighestCounterWins", status: nexusoperationpb.OPERATION_STATUS_STARTED, store: &progressStoreComponent{}, counters: []int64{1, 3, 2}, wantPending: 3}, + {name: "DroppedBeforeStart", status: nexusoperationpb.OPERATION_STATUS_SCHEDULED, store: &progressStoreComponent{}, counters: []int64{1}}, + {name: "DroppedAfterClose", status: nexusoperationpb.OPERATION_STATUS_SUCCEEDED, store: &progressStoreComponent{}, counters: []int64{1}}, + {name: "DroppedWithoutCaller", status: nexusoperationpb.OPERATION_STATUS_STARTED, counters: []int64{1}}, + {name: "DroppedByAParentThatTakesNoProgress", status: nexusoperationpb.OPERATION_STATUS_STARTED, store: &mockStoreComponent{}, counters: []int64{1}}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + op := newOp(tc.status, tc.store) + for _, counter := range tc.counters { + require.NoError(t, op.HandleNexusCompletion(ctx, progress(counter))) + } + require.Equal(t, tc.wantPending, pendingCounter(ctx, op)) + require.Equal(t, tc.status, op.Status, "progress never changes the operation status") + }) + } + + t.Run("StaleAfterDelivery", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + store := &progressStoreComponent{} + op := newOp(nexusoperationpb.OPERATION_STATUS_STARTED, store) + require.NoError(t, op.HandleNexusCompletion(ctx, progress(5))) + taken, ok := op.TakePendingProgress(ctx) + require.True(t, ok) + require.Equal(t, int64(5), taken.GetCounter()) + require.Zero(t, pendingCounter(ctx, op)) + + require.NoError(t, op.HandleNexusCompletion(ctx, progress(4))) + require.Zero(t, pendingCounter(ctx, op), "a counter at or below the delivered one is stale") + require.NoError(t, op.HandleNexusCompletion(ctx, progress(6))) + require.Equal(t, int64(6), pendingCounter(ctx, op)) + require.Equal(t, 2, store.pending) + }) + + t.Run("FoldsIntoAnUnstartedTask", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + store := &progressStoreComponent{} + op := newOp(nexusoperationpb.OPERATION_STATUS_STARTED, store) + require.NoError(t, op.HandleNexusCompletion(ctx, progress(1))) + _, ok := op.TakePendingProgress(ctx) + require.True(t, ok) + + // The task carrying counter 1 has not started, so newer progress rides it. + store.folds = true + require.NoError(t, op.HandleNexusCompletion(ctx, progress(2))) + require.Zero(t, pendingCounter(ctx, op), "folded progress asks for no task of its own") + require.Equal(t, int64(2), deliveredCounter(ctx, op)) + require.Equal(t, 1, store.pending) + + // Once that task starts, newer progress waits for the next one. + store.folds = false + require.NoError(t, op.HandleNexusCompletion(ctx, progress(3))) + require.Equal(t, int64(3), pendingCounter(ctx, op)) + require.Equal(t, 2, store.pending) + }) + + t.Run("DropPendingProgress", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + op := newOp(nexusoperationpb.OPERATION_STATUS_STARTED, &progressStoreComponent{}) + require.NoError(t, op.HandleNexusCompletion(ctx, progress(1))) + op.DropPendingProgress(ctx) + require.Zero(t, pendingCounter(ctx, op)) + require.Zero(t, deliveredCounter(ctx, op), "dropped progress is not delivered") + }) +} diff --git a/chasm/lib/workflow/gen/workflowpb/v1/state.go-helpers.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/state.go-helpers.pb.go index 50980db658e..824ce98f0c5 100644 --- a/chasm/lib/workflow/gen/workflowpb/v1/state.go-helpers.pb.go +++ b/chasm/lib/workflow/gen/workflowpb/v1/state.go-helpers.pb.go @@ -115,3 +115,40 @@ func (this *IncomingSignalData) Equal(that interface{}) bool { return proto.Equal(this, that1) } + +// Marshal an object of type NexusProgressState to the protobuf v3 wire format +func (val *NexusProgressState) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type NexusProgressState from the protobuf v3 wire format +func (val *NexusProgressState) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *NexusProgressState) Size() int { + return proto.Size(val) +} + +// Equal returns whether two NexusProgressState values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *NexusProgressState) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *NexusProgressState + switch t := that.(type) { + case *NexusProgressState: + that1 = t + case NexusProgressState: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go index 304853e6afd..078d9fc1bcb 100644 --- a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go +++ b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go @@ -171,6 +171,63 @@ func (x *IncomingSignalData) GetEventId() int64 { return 0 } +// The Workflow's Nexus operation progress, by the scheduled event ID of each operation. Kept on the +// Workflow so a transaction learns whether any progress waits without loading every operation. +type NexusProgressState struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Operations whose progress waits for the next Workflow Task scheduled event. + Pending []int64 `protobuf:"varint,1,rep,packed,name=pending,proto3" json:"pending,omitempty"` + // Operations whose latest progress rides a scheduled Workflow Task that has not started yet. + // Newer progress for one of them folds into that task, since the read it triggers sees it. + Scheduled []int64 `protobuf:"varint,2,rep,packed,name=scheduled,proto3" json:"scheduled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NexusProgressState) Reset() { + *x = NexusProgressState{} + mi := &file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NexusProgressState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NexusProgressState) ProtoMessage() {} + +func (x *NexusProgressState) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NexusProgressState.ProtoReflect.Descriptor instead. +func (*NexusProgressState) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescGZIP(), []int{3} +} + +func (x *NexusProgressState) GetPending() []int64 { + if x != nil { + return x.Pending + } + return nil +} + +func (x *NexusProgressState) GetScheduled() []int64 { + if x != nil { + return x.Scheduled + } + return nil +} + var File_temporal_server_chasm_lib_workflow_proto_v1_state_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc = "" + @@ -182,7 +239,10 @@ const file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc = "" "\x1bNexusCancellationParentData\x12,\n" + "\x12requested_event_id\x18\x01 \x01(\x03R\x10requestedEventId\"/\n" + "\x12IncomingSignalData\x12\x19\n" + - "\bevent_id\x18\x01 \x01(\x03R\aeventIdBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" + "\bevent_id\x18\x01 \x01(\x03R\aeventId\"L\n" + + "\x12NexusProgressState\x12\x18\n" + + "\apending\x18\x01 \x03(\x03R\apending\x12\x1c\n" + + "\tscheduled\x18\x02 \x03(\x03R\tscheduledBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" var ( file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescOnce sync.Once @@ -196,11 +256,12 @@ func file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescGZIP() return file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescData } -var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_goTypes = []any{ (*NexusOperationParentData)(nil), // 0: temporal.server.chasm.lib.workflow.proto.v1.NexusOperationParentData (*NexusCancellationParentData)(nil), // 1: temporal.server.chasm.lib.workflow.proto.v1.NexusCancellationParentData (*IncomingSignalData)(nil), // 2: temporal.server.chasm.lib.workflow.proto.v1.IncomingSignalData + (*NexusProgressState)(nil), // 3: temporal.server.chasm.lib.workflow.proto.v1.NexusProgressState } var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -221,7 +282,7 @@ func file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc), len(file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/chasm/lib/workflow/nexus_commands_test.go b/chasm/lib/workflow/nexus_commands_test.go index d2bf45dc980..80c6158f829 100644 --- a/chasm/lib/workflow/nexus_commands_test.go +++ b/chasm/lib/workflow/nexus_commands_test.go @@ -738,7 +738,7 @@ func TestHandleCancelCommand(t *testing.T) { event := tcx.history.Events[0] // TODO: Complete the operation using CHASM equivalent of CompletedEventDefinition. - tcx.wf.removeNexusOperation(event.EventId) + tcx.wf.removeNexusOperation(tcx.chasmCtx, event.EventId) // The operation is no longer in the CHASM tree, so the handler defers to the HSM command handler // via ErrCommandTargetNotFound. @@ -771,7 +771,7 @@ func TestHandleCancelCommand(t *testing.T) { event := tcx.history.Events[0] // TODO: Complete the operation using CHASM equivalent of CompletedEventDefinition. - tcx.wf.removeNexusOperation(event.EventId) + tcx.wf.removeNexusOperation(tcx.chasmCtx, event.EventId) // Try to cancel - should succeed because there's a buffered completion. err = tcx.cancelHandler(tcx.chasmCtx, tcx.wf, commandValidator{maxPayloadSize: 1}, &commandpb.Command{ diff --git a/chasm/lib/workflow/nexus_events.go b/chasm/lib/workflow/nexus_events.go index 11612767d0a..dafe282cab8 100644 --- a/chasm/lib/workflow/nexus_events.go +++ b/chasm/lib/workflow/nexus_events.go @@ -230,7 +230,7 @@ func (d CompletedEventDefinition) Apply(ctx chasm.MutableContext, wf *Workflow, }); err != nil { return err } - wf.removeNexusOperation(attrs.GetScheduledEventId()) + wf.removeNexusOperation(ctx, attrs.GetScheduledEventId()) return nil } @@ -269,7 +269,7 @@ func (d FailedEventDefinition) Apply(ctx chasm.MutableContext, wf *Workflow, eve }); err != nil { return err } - wf.removeNexusOperation(attrs.GetScheduledEventId()) + wf.removeNexusOperation(ctx, attrs.GetScheduledEventId()) return nil } @@ -308,7 +308,7 @@ func (d CanceledEventDefinition) Apply(ctx chasm.MutableContext, wf *Workflow, e }); err != nil { return err } - wf.removeNexusOperation(attrs.GetScheduledEventId()) + wf.removeNexusOperation(ctx, attrs.GetScheduledEventId()) return nil } @@ -344,7 +344,7 @@ func (d TimedOutEventDefinition) Apply(ctx chasm.MutableContext, wf *Workflow, e }); err != nil { return err } - wf.removeNexusOperation(attrs.GetScheduledEventId()) + wf.removeNexusOperation(ctx, attrs.GetScheduledEventId()) return nil } diff --git a/chasm/lib/workflow/nexus_methods.go b/chasm/lib/workflow/nexus_methods.go index e20ca619d21..ad1374c175e 100644 --- a/chasm/lib/workflow/nexus_methods.go +++ b/chasm/lib/workflow/nexus_methods.go @@ -34,8 +34,9 @@ func (w *Workflow) addNexusOperation( } // removeNexusOperation removes a Nexus operation from the workflow. -func (w *Workflow) removeNexusOperation(key int64) { +func (w *Workflow) removeNexusOperation(ctx chasm.MutableContext, key int64) { delete(w.Operations, key) + w.forgetNexusProgress(ctx, key) } // pendingNexusOperationCount returns the number of pending Nexus operations in the workflow. diff --git a/chasm/lib/workflow/nexus_progress.go b/chasm/lib/workflow/nexus_progress.go new file mode 100644 index 00000000000..53657a281e6 --- /dev/null +++ b/chasm/lib/workflow/nexus_progress.go @@ -0,0 +1,147 @@ +package workflow + +import ( + "slices" + + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/nexusoperation" + chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" + "go.temporal.io/server/common" +) + +var _ nexusoperation.ProgressStore = (*Workflow)(nil) + +// nexusProgressKey is the scheduled event ID that keys an operation in the Workflow. +func nexusProgressKey(op *nexusoperation.Operation) (int64, bool) { + parentData := &chasmworkflowpb.NexusOperationParentData{} + if err := op.GetParentData().UnmarshalTo(parentData); err != nil { + return 0, false + } + return parentData.GetScheduledEventId(), true +} + +// nexusProgressIndex answers the index, or an empty one when no operation ever had progress. +func (w *Workflow) nexusProgressIndex(ctx chasm.Context) *chasmworkflowpb.NexusProgressState { + if index, ok := w.NexusProgress.TryGet(ctx); ok { + return index + } + return &chasmworkflowpb.NexusProgressState{} +} + +// setNexusProgressIndex stores the index, or drops it when it is empty, so a Workflow whose +// operations report no progress carries nothing. +func (w *Workflow) setNexusProgressIndex(ctx chasm.MutableContext, index *chasmworkflowpb.NexusProgressState) { + if len(index.GetPending()) == 0 && len(index.GetScheduled()) == 0 { + if _, ok := w.NexusProgress.TryGet(ctx); ok { + w.NexusProgress = chasm.NewEmptyField[*chasmworkflowpb.NexusProgressState]() + } + return + } + w.NexusProgress = chasm.NewDataField(ctx, index) +} + +// OnNexusOperationProgress records that an operation holds progress for the next Workflow Task +// scheduled event. +func (w *Workflow) OnNexusOperationProgress(ctx chasm.MutableContext, op *nexusoperation.Operation) error { + key, ok := nexusProgressKey(op) + if !ok { + op.DropPendingProgress(ctx) + return nil + } + index := common.CloneProto(w.nexusProgressIndex(ctx)) + if !slices.Contains(index.Pending, key) { + index.Pending = append(index.Pending, key) + slices.Sort(index.Pending) + } + w.setNexusProgressIndex(ctx, index) + return nil +} + +// NexusOperationProgressFolds reports whether an operation's latest progress rides a scheduled +// Workflow Task that has not started. That task's read will see what newer progress reports. +func (w *Workflow) NexusOperationProgressFolds(ctx chasm.Context, op *nexusoperation.Operation) bool { + key, ok := nexusProgressKey(op) + return ok && slices.Contains(w.nexusProgressIndex(ctx).GetScheduled(), key) +} + +// HasPendingNexusProgress reports whether any operation holds progress for the next Workflow Task +// scheduled event. It reads one field, so it is cheap on every transaction close. +func (w *Workflow) HasPendingNexusProgress(ctx chasm.Context) bool { + return len(w.nexusProgressIndex(ctx).GetPending()) > 0 +} + +// HasScheduledNexusProgress reports whether a scheduled Workflow Task carries progress. +func (w *Workflow) HasScheduledNexusProgress(ctx chasm.Context) bool { + return len(w.nexusProgressIndex(ctx).GetScheduled()) > 0 +} + +// TakeNexusProgress returns the pending progress of every operation in scheduled event ID order, +// each naming its operation, for a Workflow Task scheduled event that is being written. The event +// acknowledges it, and until the task starts newer progress for those operations folds into it. +func (w *Workflow) TakeNexusProgress(ctx chasm.MutableContext) []*nexuspb.NexusOperationProgress { + index := common.CloneProto(w.nexusProgressIndex(ctx)) + var taken []*nexuspb.NexusOperationProgress + for _, key := range index.GetPending() { + field, ok := w.Operations[key] + if !ok { + continue + } + progress, ok := field.Get(ctx).TakePendingProgress(ctx) + if !ok { + continue + } + named := common.CloneProto(progress) + named.Operation = &nexuspb.NexusOperationProgress_ScheduledEventId{ScheduledEventId: key} + taken = append(taken, named) + if !slices.Contains(index.Scheduled, key) { + index.Scheduled = append(index.Scheduled, key) + } + } + slices.Sort(index.Scheduled) + index.Pending = nil + w.setNexusProgressIndex(ctx, index) + return taken +} + +// ClearScheduledNexusProgress forgets which operations the scheduled Workflow Task carries. Called +// once that task starts, fails or times out: from then on newer progress needs a task of its own. +func (w *Workflow) ClearScheduledNexusProgress(ctx chasm.MutableContext) { + index := w.nexusProgressIndex(ctx) + if len(index.GetScheduled()) == 0 { + return + } + index = common.CloneProto(index) + index.Scheduled = nil + w.setNexusProgressIndex(ctx, index) +} + +// DropPendingNexusProgress forgets every operation's pending progress. It guards against a Workflow +// Task that was scheduled for progress but could not carry it, which would otherwise schedule +// another task on every transaction. +func (w *Workflow) DropPendingNexusProgress(ctx chasm.MutableContext) { + index := w.nexusProgressIndex(ctx) + if len(index.GetPending()) == 0 { + return + } + for _, key := range index.GetPending() { + if field, ok := w.Operations[key]; ok { + field.Get(ctx).DropPendingProgress(ctx) + } + } + index = common.CloneProto(index) + index.Pending = nil + w.setNexusProgressIndex(ctx, index) +} + +// forgetNexusProgress removes a closed operation from the index. +func (w *Workflow) forgetNexusProgress(ctx chasm.MutableContext, key int64) { + index := w.nexusProgressIndex(ctx) + if !slices.Contains(index.GetPending(), key) && !slices.Contains(index.GetScheduled(), key) { + return + } + index = common.CloneProto(index) + index.Pending = slices.DeleteFunc(index.Pending, func(k int64) bool { return k == key }) + index.Scheduled = slices.DeleteFunc(index.Scheduled, func(k int64) bool { return k == key }) + w.setNexusProgressIndex(ctx, index) +} diff --git a/chasm/lib/workflow/nexus_progress_test.go b/chasm/lib/workflow/nexus_progress_test.go new file mode 100644 index 00000000000..984638e45aa --- /dev/null +++ b/chasm/lib/workflow/nexus_progress_test.go @@ -0,0 +1,102 @@ +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/require" + nexuspb "go.temporal.io/api/nexus/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/nexusoperation" + nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" + "google.golang.org/protobuf/types/known/anypb" +) + +func TestNexusProgressIndex(t *testing.T) { + newWorkflowWithOperations := func(t *testing.T, ctx chasm.MutableContext, keys ...int64) *Workflow { + wf := &Workflow{} + for _, key := range keys { + parentData, err := anypb.New(&chasmworkflowpb.NexusOperationParentData{ScheduledEventId: key}) + require.NoError(t, err) + op := nexusoperation.NewOperation(&nexusoperationpb.OperationState{ + Status: nexusoperationpb.OPERATION_STATUS_STARTED, + ParentData: parentData, + }) + op.Store = chasm.NewMockParentPtr[nexusoperation.OperationStore](wf) + wf.addNexusOperation(ctx, key, op) + } + return wf + } + deliver := func(t *testing.T, ctx chasm.MutableContext, wf *Workflow, key, counter int64) { + err := wf.Operations[key].Get(ctx).HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + Outcome: &persistencespb.ChasmNexusCompletion_Progress{ + Progress: &nexuspb.NexusOperationProgress{Counter: counter}, + }, + }) + require.NoError(t, err) + } + counters := func(taken []*nexuspb.NexusOperationProgress) map[int64]int64 { + out := map[int64]int64{} + for _, p := range taken { + out[p.GetScheduledEventId()] = p.GetCounter() + } + return out + } + + t.Run("TakesPendingProgressPerOperationInScheduledOrder", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + wf := newWorkflowWithOperations(t, ctx, 7, 5) + require.False(t, wf.HasPendingNexusProgress(ctx)) + deliver(t, ctx, wf, 7, 1) + deliver(t, ctx, wf, 5, 2) + deliver(t, ctx, wf, 7, 3) + require.True(t, wf.HasPendingNexusProgress(ctx)) + + taken := wf.TakeNexusProgress(ctx) + require.Len(t, taken, 2) + require.Equal(t, int64(5), taken[0].GetScheduledEventId()) + require.Equal(t, map[int64]int64{5: 2, 7: 3}, counters(taken)) + require.False(t, wf.HasPendingNexusProgress(ctx)) + require.True(t, wf.HasScheduledNexusProgress(ctx)) + }) + + t.Run("FoldsIntoTheUnstartedTaskUntilItStarts", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + wf := newWorkflowWithOperations(t, ctx, 5, 9) + deliver(t, ctx, wf, 5, 1) + require.Len(t, wf.TakeNexusProgress(ctx), 1) + + deliver(t, ctx, wf, 5, 2) + require.False(t, wf.HasPendingNexusProgress(ctx), "operation 5 rides the unstarted task") + deliver(t, ctx, wf, 9, 1) + require.True(t, wf.HasPendingNexusProgress(ctx), "operation 9 is not on that task") + + wf.ClearScheduledNexusProgress(ctx) + require.False(t, wf.HasScheduledNexusProgress(ctx)) + deliver(t, ctx, wf, 5, 3) + require.Equal(t, map[int64]int64{5: 3, 9: 1}, counters(wf.TakeNexusProgress(ctx))) + }) + + t.Run("ForgetsAClosedOperation", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + wf := newWorkflowWithOperations(t, ctx, 5, 9) + deliver(t, ctx, wf, 5, 1) + deliver(t, ctx, wf, 9, 1) + wf.removeNexusOperation(ctx, 5) + require.Equal(t, map[int64]int64{9: 1}, counters(wf.TakeNexusProgress(ctx))) + wf.removeNexusOperation(ctx, 9) + require.False(t, wf.HasScheduledNexusProgress(ctx)) + _, ok := wf.NexusProgress.TryGet(ctx) + require.False(t, ok, "an empty index is not stored") + }) + + t.Run("DropsPendingProgressAsALivelockGuard", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + wf := newWorkflowWithOperations(t, ctx, 5) + deliver(t, ctx, wf, 5, 1) + wf.DropPendingNexusProgress(ctx) + require.False(t, wf.HasPendingNexusProgress(ctx)) + require.Empty(t, wf.TakeNexusProgress(ctx)) + }) +} diff --git a/chasm/lib/workflow/proto/v1/state.proto b/chasm/lib/workflow/proto/v1/state.proto index 0ff01d116b5..4021ca65e17 100644 --- a/chasm/lib/workflow/proto/v1/state.proto +++ b/chasm/lib/workflow/proto/v1/state.proto @@ -25,3 +25,13 @@ message NexusCancellationParentData { message IncomingSignalData { int64 event_id = 1; } + +// The Workflow's Nexus operation progress, by the scheduled event ID of each operation. Kept on the +// Workflow so a transaction learns whether any progress waits without loading every operation. +message NexusProgressState { + // Operations whose progress waits for the next Workflow Task scheduled event. + repeated int64 pending = 1; + // Operations whose latest progress rides a scheduled Workflow Task that has not started yet. + // Newer progress for one of them folds into that task, since the read it triggers sees it. + repeated int64 scheduled = 2; +} diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index 5a2575b704b..fd616a6c67d 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -45,6 +45,10 @@ type Workflow struct { // Updates indexed by update ID, used to store the update components. Updates chasm.Map[string, *WorkflowUpdate] + + // NexusProgress indexes the Nexus operations whose progress waits for, or rides, a Workflow + // Task, so a transaction can tell without loading every operation. + NexusProgress chasm.Field[*chasmworkflowpb.NexusProgressState] } func NewWorkflow( diff --git a/common/nexus/nexusrpc/completion.go b/common/nexus/nexusrpc/completion.go index 6db219efe9a..cc215999b49 100644 --- a/common/nexus/nexusrpc/completion.go +++ b/common/nexus/nexusrpc/completion.go @@ -104,8 +104,11 @@ type CompleteOperationOptions struct { // Error to send with the completion. If set, the completion is unsuccessful. Error *nexus.OperationError // Result to deliver with the completion. Uses the client's serializer to serialize the result into the request body. - // Only used for successful completions. + // Only used for successful completions and progress. Result any + // Progress makes the request a progress delivery rather than a completion: the state is running and Result holds + // the progress body. The operation stays open. + Progress bool } // nolint:revive // This method is long but it's more readable to keep the logic in one place since it's all related to @@ -154,7 +157,11 @@ func (c CompleteOperationOptions) applyToHTTPRequest(cc *CompletionHTTPClient, r addContentHeaderToHTTPHeader(reader.Header, request.Header) } request.Body = reader.ReadCloser - request.Header.Set(headerOperationState, string(nexus.OperationStateSucceeded)) + state := nexus.OperationStateSucceeded + if c.Progress { + state = nexus.OperationStateRunning + } + request.Header.Set(headerOperationState, string(state)) } if c.Header != nil { diff --git a/common/nexus/nexusrpc/completion_test.go b/common/nexus/nexusrpc/completion_test.go index 90addfc5660..9aead3292ab 100644 --- a/common/nexus/nexusrpc/completion_test.go +++ b/common/nexus/nexusrpc/completion_test.go @@ -3,6 +3,7 @@ package nexusrpc_test import ( "context" "errors" + "io" "net/url" "testing" "time" @@ -258,3 +259,37 @@ func TestBadRequestCompletion(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerErr.Type) } + +type progressHandler struct { + got chan *nexusrpc.CompletionRequest + body chan []byte +} + +func (h *progressHandler) CompleteOperation(ctx context.Context, completion *nexusrpc.CompletionRequest) error { + body, err := io.ReadAll(completion.Result.Reader) + if err != nil { + return err + } + h.got <- completion + h.body <- body + return nil +} + +func TestProgressDelivery(t *testing.T) { + h := &progressHandler{got: make(chan *nexusrpc.CompletionRequest, 1), body: make(chan []byte, 1)} + ctx, callbackURL, teardown := setupForCompletion(t, h, nil, nil) + defer teardown() + + c := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{}) + err := c.CompleteOperation(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ + Result: &nexus.Content{Header: nexus.Header{"type": "application/json"}, Data: []byte(`{"counter": 2}`)}, + Progress: true, + OperationToken: "test-operation-token", + }) + require.NoError(t, err) + completion := <-h.got + require.Equal(t, nexus.OperationStateRunning, completion.State) + require.Equal(t, "test-operation-token", completion.OperationToken) + require.True(t, completion.CloseTime.IsZero(), "progress leaves the operation open") + require.JSONEq(t, `{"counter": 2}`, string(<-h.body)) +} diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index e34cc059f32..78334e73c54 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -509,9 +509,13 @@ func (h *nexusCompletionHandler) forwardCompleteOperation(ctx context.Context, r Links: r.Links, } case nexus.OperationStateRunning: - // A 400 would turn progress off for the callback. Progress is not forwarded, and the next - // delivery after a failover reaches the active cluster. - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "operation progress is not forwarded") + completion = nexusrpc.CompleteOperationOptions{ + Result: r.Result.Reader, + Progress: true, + OperationToken: r.OperationToken, + StartTime: r.StartTime, + Links: r.Links, + } default: return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation state: %q", r.State) } diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 7dc0513c32f..7be22a2bc70 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7963,7 +7963,8 @@ func (ms *MutableStateImpl) closeTransactionHandleWorkflowTaskScheduling( } } - return nil + // Nexus operation progress writes no event, so it schedules the task that carries it. + return ms.scheduleWorkflowTaskForNexusProgress() } func (ms *MutableStateImpl) closeTransactionHandleSpeculativeWorkflowTask( diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go new file mode 100644 index 00000000000..d71b0eab4d9 --- /dev/null +++ b/service/history/workflow/nexus_progress.go @@ -0,0 +1,89 @@ +package workflow + +import ( + "context" + + historypb "go.temporal.io/api/history/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/common/log/tag" +) + +// chasmWorkflowView resolves the CHASM Workflow component through a read-only context, or answers +// false for a Workflow without one. It is asked on every transaction close, so it never takes the +// component for writing. +func (ms *MutableStateImpl) chasmWorkflowView() (*chasmworkflow.Workflow, chasm.Context, bool) { + node, ok := ms.chasmTree.(*chasm.Node) + if !ok { + return nil, nil, false + } + chasmCtx := chasm.NewContext(context.Background(), node) + rootComponent, err := node.ComponentByPath(chasmCtx, nil) + if err != nil { + return nil, nil, false + } + wf, ok := rootComponent.(*chasmworkflow.Workflow) + return wf, chasmCtx, ok +} + +// hasPendingNexusProgress reports whether a Nexus operation holds progress for the next Workflow +// Task scheduled event. Progress writes no event, so nothing else schedules the task that carries +// it. +func (ms *MutableStateImpl) hasPendingNexusProgress() bool { + wf, chasmCtx, ok := ms.chasmWorkflowView() + return ok && wf.HasPendingNexusProgress(chasmCtx) +} + +// attachNexusProgress puts the pending Nexus operation progress on a WorkflowTaskScheduled event +// for a task no worker has seen yet. The event is the acknowledgment, so the progress is taken. +func (ms *MutableStateImpl) attachNexusProgress(event *historypb.HistoryEvent) { + if !ms.hasPendingNexusProgress() { + return + } + wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + ms.logger.Warn("cannot attach Nexus operation progress", tag.Error(err)) + return + } + attrs := event.GetWorkflowTaskScheduledEventAttributes() + attrs.NexusOperationProgress = append(attrs.NexusOperationProgress, wf.TakeNexusProgress(chasmCtx)...) +} + +// clearScheduledNexusProgress forgets which operations the scheduled Workflow Task carries, once +// that task starts, fails or times out, so newer progress schedules a task of its own. Checked +// through a read-only view first, since almost no Workflow has any. +func (ms *MutableStateImpl) clearScheduledNexusProgress() { + wf, chasmCtx, ok := ms.chasmWorkflowView() + if !ok || !wf.HasScheduledNexusProgress(chasmCtx) { + return + } + wf, mutableCtx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + ms.logger.Warn("cannot clear scheduled Nexus operation progress", tag.Error(err)) + return + } + wf.ClearScheduledNexusProgress(mutableCtx) +} + +// scheduleWorkflowTaskForNexusProgress schedules a Workflow Task to carry pending Nexus operation +// progress when no task is pending to carry it. +func (ms *MutableStateImpl) scheduleWorkflowTaskForNexusProgress() error { + if ms.HasPendingWorkflowTask() || ms.IsWorkflowExecutionStatusPaused() || !ms.hasPendingNexusProgress() { + return nil + } + if _, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL); err != nil { + return err + } + // A task that was scheduled for progress but did not carry it would leave the progress + // pending, and every later transaction would schedule another task for it. + if ms.hasPendingNexusProgress() { + ms.logger.Warn("dropped Nexus operation progress that a scheduled Workflow Task did not carry") + wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + return err + } + wf.DropPendingNexusProgress(chasmCtx) + } + return nil +} diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index ec85a6afa9e..6f59202d026 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -267,6 +267,7 @@ func (m *workflowTaskStateMachine) ApplyWorkflowTaskTimedOutEvent(timeoutType en func (m *workflowTaskStateMachine) AddWorkflowTaskScheduleToStartTimeoutEvent( workflowTask *historyi.WorkflowTaskInfo, ) (*historypb.HistoryEvent, error) { + m.ms.clearScheduledNexusProgress() opTag := tag.WorkflowActionWorkflowTaskTimedOut if m.ms.executionInfo.WorkflowTaskScheduledEventId != workflowTask.ScheduledEventID || m.ms.executionInfo.WorkflowTaskStartedEventId > 0 { m.ms.logger.Warn(mutableStateInvalidHistoryActionMsg, opTag, @@ -367,6 +368,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( attempt, scheduleTime, ) + m.ms.attachNexusProgress(scheduledEvent) scheduledEventID = scheduledEvent.GetEventId() } else { // WorkflowTaskScheduledEvent will be created later. @@ -584,6 +586,8 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent( workflowTask.Attempt, startTime, ) + // The worker has not seen this task yet, so its scheduled event can carry what is pending. + m.ms.attachNexusProgress(scheduledEvent) scheduledEventID = scheduledEvent.GetEventId() } @@ -626,6 +630,10 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent( m.emitWorkflowTaskAttemptStats(workflowTask.Attempt) + // The task is reading from here on, so newer Nexus operation progress may not reach it and + // goes to the next task instead. + m.ms.clearScheduledNexusProgress() + // TODO merge active & passive task generation if err = m.ms.taskGenerator.GenerateStartWorkflowTaskTasks( scheduledEventID, @@ -672,6 +680,7 @@ func (m *workflowTaskStateMachine) processBuildIdRedirectInfo( workflowTask.Attempt, workflowTask.ScheduledTime, ) + m.ms.attachNexusProgress(scheduledEvent) newWorkflowTask = m.getWorkflowTaskInfo() newWorkflowTask.ScheduledEventID = scheduledEvent.GetEventId() // Using 1 as the attempt in MS. it's needed so that the new WFT is not considered transient. @@ -899,6 +908,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskFailedEvent( newRunID string, forkEventVersion int64, ) (*historypb.HistoryEvent, error) { + m.ms.clearScheduledNexusProgress() // IMPORTANT: returned event can be nil under some circumstances. Specifically, if WT is transient. @@ -974,6 +984,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskFailedEvent( func (m *workflowTaskStateMachine) AddWorkflowTaskTimedOutEvent( workflowTask *historyi.WorkflowTaskInfo, ) (*historypb.HistoryEvent, error) { + m.ms.clearScheduledNexusProgress() if workflowTask.Type == enumsspb.WORKFLOW_TASK_TYPE_SPECULATIVE { m.ms.RemoveSpeculativeWorkflowTaskTimeoutTask() @@ -1554,6 +1565,11 @@ func (m *workflowTaskStateMachine) convertSpeculativeWorkflowTaskToNormal() erro if scheduledEvent.EventId != wt.ScheduledEventID { return serviceerror.NewInternalf("it could be a bug, scheduled event Id: %d for normal workflow task doesn't match the one from speculative workflow task: %d", scheduledEvent.EventId, wt.ScheduledEventID) } + // A task a worker already started ran without this event, so only one it has not seen yet may + // carry progress. + if wt.StartedEventID == common.EmptyEventID { + m.ms.attachNexusProgress(scheduledEvent) + } if wtAlreadyStarted := wt.StartedEventID != common.EmptyEventID; wtAlreadyStarted { // If WT was already started then started event is written to the history and diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index 9ee01cad39a..626bc344fb9 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -2,15 +2,17 @@ package tests import ( "context" + "fmt" "net/http" "strings" "time" "github.com/nexus-rpc/sdk-go/nexus" - "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" "go.temporal.io/sdk/workflow" @@ -40,8 +42,8 @@ func postNexusProgress(ctx context.Context, callbackURL, callbackToken, body str // TestNexusOperationProgressIntake drives progress deliveries through the completion endpoint. // Every refusal is a 400, which tells the handler to stop sending progress; an accepted delivery -// changes nothing in the caller's History; progress before the start response is dropped; and a -// closed operation answers 404. +// rides the next Workflow Task scheduled event; progress before the start response is dropped; and +// a closed operation answers 404. func (s *NexusWorkflowTestSuite) TestNexusOperationProgressIntake(chasmEnabled bool) { for _, enabled := range []bool{false, true} { env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, enabled)) @@ -126,10 +128,16 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressIntake(chasmEnabled b s.Equal(http.StatusBadRequest, progress(body), body) } s.Equal(http.StatusOK, progress(`{"position": "cursor-2", "counter": "2", "metadata": {"topic": "t"}}`)) - // Nothing folds progress yet, so an accepted delivery leaves History as it was. - require.Never(s.T(), func() bool { - return len(env.GetHistory(env.Namespace().String(), wfExec)) != len(started) - }, 300*time.Millisecond, 50*time.Millisecond) + // An accepted delivery rides the next Workflow Task's scheduled event. + scheduledEventID := s.RequireHistoryEvent(started, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() + s.Await(func(s *NexusWorkflowTestSuite) { + carried := carriedNexusProgress(env.GetHistory(env.Namespace().String(), wfExec)[len(started):]) + s.Len(carried, 1) + s.Equal(scheduledEventID, carried[0].GetScheduledEventId()) + s.Equal(int64(2), carried[0].GetCounter()) + s.Equal([]byte("cursor-2"), carried[0].GetPosition()) + s.Equal([]byte(`"t"`), carried[0].GetMetadata()["topic"].GetData()) + }, 10*time.Second, 50*time.Millisecond) } s.NoError(s.sendNexusCompletionRequest(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ @@ -145,3 +153,165 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressIntake(chasmEnabled b w.Stop() } } + +// carriedNexusProgress lists the Nexus operation progress the scheduled events in hist carry. +func carriedNexusProgress(hist []*historypb.HistoryEvent) []*nexuspb.NexusOperationProgress { + var carried []*nexuspb.NexusOperationProgress + for _, event := range hist { + carried = append(carried, event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) + } + return carried +} + +// TestNexusOperationProgress checks how progress reaches a CHASM caller Workflow: on Workflow Task +// scheduled events with no event of its own, folded into a scheduled task that has not started, +// dropped after completion, and replayed. The HSM caller refuses progress. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { + env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + + var callbackToken, callbackURL string + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + callbackToken = options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) + + callerWF := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(endpointName, "service") + var result string + if err := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}).Get(ctx, &result); err != nil { + return "", err + } + workflow.GetSignalChannel(ctx, "finish").Receive(ctx, nil) + return result, nil + } + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, callerWF) + s.NoError(err) + newWorker := func() worker.Worker { + w := worker.New(env.SdkClient(), taskQueue, worker.Options{}) + w.RegisterWorkflow(callerWF) + s.NoError(w.Start()) + return w + } + w := newWorker() + + wfExec := &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()} + history := func() []*historypb.HistoryEvent { + return env.GetHistory(env.Namespace().String(), wfExec) + } + // Idle means the Workflow is blocked on the operation or the Signal with no task in flight. + waitIdle := func(check func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent)) []*historypb.HistoryEvent { + var hist []*historypb.HistoryEvent + s.Await(func(s *NexusWorkflowTestSuite) { + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID()) + s.NoError(err) + s.Nil(desc.GetPendingWorkflowTask()) + hist = history() + s.Equal(enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, hist[len(hist)-1].GetEventType()) + check(s, hist) + }, 10*time.Second, 50*time.Millisecond) + return hist + } + progress := func(counter int) int { + status, err := postNexusProgress(ctx, callbackURL, callbackToken, fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter)) + s.NoError(err) + return status + } + onlyWorkflowTaskEvents := func(hist []*historypb.HistoryEvent) { + for _, event := range hist { + s.Contains([]enumspb.EventType{ + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, + enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, + }, event.GetEventType(), "progress must add no event of its own") + } + } + + beforeBurst := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { + s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + }) + if !chasmEnabled { + s.Equal(http.StatusBadRequest, progress(1), "an HSM caller refuses progress, so the handler stops sending it") + w.Stop() + return + } + scheduledEventID := s.scheduledEventID(beforeBurst) + + // With no worker polling, the task the first delivery schedules stays unstarted, and later + // deliveries fold into it instead of asking for tasks of their own. + w.Stop() + s.Equal(http.StatusOK, progress(1)) + s.Await(func(s *NexusWorkflowTestSuite) { + s.Len(carriedNexusProgress(history()[len(beforeBurst):]), 1) + }, 10*time.Second, 50*time.Millisecond) + s.Equal(http.StatusOK, progress(2)) + s.Equal(http.StatusOK, progress(3)) + s.Equal(http.StatusOK, progress(2), "a stale counter is accepted and dropped") + w = newWorker() + afterFold := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { + s.Greater(len(hist), len(beforeBurst)) + }) + folded := afterFold[len(beforeBurst):] + onlyWorkflowTaskEvents(folded) + carried := carriedNexusProgress(folded) + s.Len(carried, 1, "one task carries the whole burst") + s.Equal(scheduledEventID, carried[0].GetScheduledEventId()) + s.Equal(int64(1), carried[0].GetCounter(), "the task that was already scheduled carries what it had") + var scheduledTasks int + for _, event := range folded { + if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + scheduledTasks++ + } + } + s.Equal(1, scheduledTasks) + + // After that task started, newer progress needs a task of its own. + s.Equal(http.StatusOK, progress(4)) + afterNext := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { + s.Len(carriedNexusProgress(hist[len(afterFold):]), 1) + }) + onlyWorkflowTaskEvents(afterNext[len(afterFold):]) + s.Equal(int64(4), carriedNexusProgress(afterNext[len(afterFold):])[0].GetCounter()) + s.Equal(http.StatusOK, progress(3), "a counter below a folded one is stale") + + s.NoError(s.sendNexusCompletionRequest(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ + Result: testcore.MustToPayload(s.T(), "result"), + Header: nexus.Header{commonnexus.CallbackTokenHeader: callbackToken}, + })) + afterCompletion := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { + s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED) + }) + s.Equal(http.StatusNotFound, progress(5), "the operation is gone with its completion") + s.Len(history(), len(afterCompletion), "progress after completion must not touch the Workflow") + + s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "finish", nil)) + var result string + s.NoError(run.Get(ctx, &result)) + s.Equal("result", result) + w.Stop() + + replayer := worker.NewWorkflowReplayer() + replayer.RegisterWorkflow(callerWF) + s.NoError(replayer.ReplayWorkflowHistory(nil, &historypb.History{Events: history()})) + + resp, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: wfExec, + }) + s.NoError(err) + s.Len(carriedNexusProgress(resp.GetHistory().GetEvents()), 2, "the progress is part of History") +} + +func (s *NexusWorkflowTestSuite) scheduledEventID(hist []*historypb.HistoryEvent) int64 { + return s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() +} diff --git a/tests/xdc/nexus_progress_forwarding_test.go b/tests/xdc/nexus_progress_forwarding_test.go new file mode 100644 index 00000000000..78062e9f28d --- /dev/null +++ b/tests/xdc/nexus_progress_forwarding_test.go @@ -0,0 +1,187 @@ +package xdc + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/suite" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/api/operatorservice/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + cnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/tests/testcore" +) + +// NexusProgressForwardingSuite runs CHASM caller Workflows, the only ones that take Nexus +// operation progress, across two clusters. +type NexusProgressForwardingSuite struct { + xdcBaseSuite +} + +func TestNexusProgressForwardingSuite(t *testing.T) { + t.Parallel() + s := &NexusProgressForwardingSuite{} + s.enableTransitionHistory = true + suite.Run(t, s) +} + +func (s *NexusProgressForwardingSuite) SetupSuite() { + s.dynamicConfigOverrides = map[dynamicconfig.Key]any{ + dynamicconfig.FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS.Key(): 1000, + dynamicconfig.RefreshNexusEndpointsMinWait.Key(): 1 * time.Millisecond, + dynamicconfig.EnableChasm.Key(): true, + dynamicconfig.EnableCHASMCallbacks.Key(): true, + chasmnexus.Enabled.Key(): true, + chasmnexus.EnableChasmWorkflowOperations.Key(): true, + chasmnexus.ChasmWorkflowOperationsRolloutPercent.Key(): 100, + chasmnexus.EnableProgress.Key(): true, + } + s.setupSuite() +} + +func (s *NexusProgressForwardingSuite) SetupTest() { + s.setupTest() +} + +func (s *NexusProgressForwardingSuite) TearDownSuite() { + s.tearDownSuite() +} + +// TestProgressForwardedFromStandbyToActive posts progress to the standby cluster, which forwards it +// to the active one instead of refusing it, so the caller's next Workflow Task carries it. +func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive() { + // The callback URL points at the standby cluster, so every delivery lands there first. + standbyCallbackURL := "http://" + s.clusters[1].Host().FrontendHTTPAddress() + "/namespaces/{{.NamespaceName}}/nexus/callback" + s.clusters[0].OverrideDynamicConfig(s.T(), chasmnexus.CallbackURLTemplate, standbyCallbackURL) + s.clusters[1].OverrideDynamicConfig(s.T(), chasmnexus.CallbackURLTemplate, standbyCallbackURL) + + ctx := testcore.NewContext() + ns := s.createGlobalNamespace() + taskQueue := fmt.Sprintf("%v-%v", "test-task-queue", uuid.New()) + endpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + + var callbackToken, callbackURL string + h := nexustest.Handler{ + OnStartOperation: func(ctx context.Context, service, operation string, input *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + callbackToken = options.CallbackHeader.Get(cnexus.CallbackTokenHeader) + callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + listenAddr := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddr, h) + createEndpointReq := &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: endpointName, + Target: &nexuspb.EndpointTarget{ + Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{Url: "http://" + listenAddr}, + }, + }, + }, + } + for _, cluster := range s.clusters { + _, err := cluster.OperatorClient().CreateNexusEndpoint(ctx, createEndpointReq) + s.NoError(err) + } + + activeSDKClient, err := client.Dial(client.Options{ + HostPort: s.clusters[0].Host().FrontendGRPCAddress(), + Namespace: ns, + Logger: log.NewSdkLogger(s.logger), + }) + s.NoError(err) + run, err := activeSDKClient.ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + s.NoError(err) + + active := s.clusters[0].FrontendClient() + poll := func() *workflowservice.PollWorkflowTaskQueueResponse { + resp, err := active.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + Identity: "test", + }) + s.NoError(err) + return resp + } + respond := func(resp *workflowservice.PollWorkflowTaskQueueResponse, commands ...*commandpb.Command) { + _, err := active.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Identity: "test", + TaskToken: resp.TaskToken, + Commands: commands, + }) + s.NoError(err) + } + + respond(poll(), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpointName, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(s.T(), "input"), + }, + }, + }) + startedTask := poll() + scheduledIdx := slices.IndexFunc(startedTask.History.Events, func(e *historypb.HistoryEvent) bool { + return e.GetNexusOperationScheduledEventAttributes() != nil + }) + s.Positive(scheduledIdx) + s.Positive(slices.IndexFunc(startedTask.History.Events, func(e *historypb.HistoryEvent) bool { + return e.GetNexusOperationStartedEventAttributes() != nil + })) + respond(startedTask) + + // The standby has to know the operation before it can forward progress for it. + await.RequireTrue(s.T(), func() bool { + resp, err := s.clusters[1].FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()}, + }) + return err == nil && len(resp.PendingNexusOperations) > 0 + }, 5*time.Second, 200*time.Millisecond) + + c := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{}) + err = c.CompleteOperation(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ + Result: &nexus.Content{Header: nexus.Header{"type": "application/json"}, Data: []byte(`{"position": "p1", "counter": 1}`)}, + Progress: true, + Header: nexus.Header{cnexus.CallbackTokenHeader: callbackToken}, + }) + s.NoError(err, "the standby forwards progress rather than refusing it") + + progressTask := poll() + var carried []*nexuspb.NexusOperationProgress + for _, event := range progressTask.History.Events { + carried = append(carried, event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) + } + s.Len(carried, 1) + s.Equal(startedTask.History.Events[scheduledIdx].GetEventId(), carried[0].GetScheduledEventId()) + s.Equal(int64(1), carried[0].GetCounter()) + s.Equal([]byte("p1"), carried[0].GetPosition()) + respond(progressTask, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }) + s.NoError(run.Get(ctx, nil)) +} From f5576578d2c40c05ab02c0340fdb0fa47bb125c6 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 9 Oct 2026 20:26:21 -0700 Subject: [PATCH 2/9] Kept Nexus progress off tasks forced to heartbeat a local activity. SDKs replay a heartbeat chain differently from live, so progress on one of its tasks could reach the Workflow in another activation. It now waits for the next normal task. --- .../api/respondworkflowtaskcompleted/api.go | 7 + service/history/interfaces/mutable_state.go | 3 + .../history/interfaces/mutable_state_mock.go | 12 ++ .../history/workflow/mutable_state_impl.go | 10 +- service/history/workflow/nexus_progress.go | 2 +- tests/nexus_progress_test.go | 128 ++++++++++++++++++ 6 files changed, 160 insertions(+), 2 deletions(-) diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index b941fc82f56..f1b79ca46ed 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -626,6 +626,13 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( bypassTaskGeneration = false } + if request.GetForceCreateNewWorkflowTask() { + // A worker forces a new task to heartbeat a long local activity. SDKs replay such a task + // as part of a heartbeat chain, so progress on it could reach the Workflow in a different + // activation on replay than live. It waits for the next task instead. + ms.HoldNexusProgress() + } + var newWTErr error // If we checked WT heartbeat timeout before and WT wasn't timed out, // then OriginalScheduledTime needs to be carried over to the new WT. diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 7b7c9857783..cfa79a46fbb 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -80,6 +80,9 @@ type ( AddFirstWorkflowTaskScheduled(parentClock *clockspb.VectorClock, event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error) AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *timestamppb.Timestamp, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) + // HoldNexusProgress keeps pending Nexus operation progress off any Workflow Task scheduled in + // this transaction, so it waits for a later task. + HoldNexusProgress() AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion, int64) (*historypb.HistoryEvent, *WorkflowTaskInfo, error) AddWorkflowTaskTimedOutEvent(workflowTask *WorkflowTaskInfo) (*historypb.HistoryEvent, error) AddExternalWorkflowExecutionCancelRequested(int64, namespace.Name, namespace.ID, string, string) (*historypb.HistoryEvent, error) diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index 43b8ffc0f00..4ce3acf355e 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -3125,6 +3125,18 @@ func (mr *MockMutableStateMockRecorder) HasStartedWorkflowTask() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasStartedWorkflowTask", reflect.TypeOf((*MockMutableState)(nil).HasStartedWorkflowTask)) } +// HoldNexusProgress mocks base method. +func (m *MockMutableState) HoldNexusProgress() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "HoldNexusProgress") +} + +// HoldNexusProgress indicates an expected call of HoldNexusProgress. +func (mr *MockMutableStateMockRecorder) HoldNexusProgress() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HoldNexusProgress", reflect.TypeOf((*MockMutableState)(nil).HoldNexusProgress)) +} + // InitTransitionHistory mocks base method. func (m *MockMutableState) InitTransitionHistory() { m.ctrl.T.Helper() diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 7be22a2bc70..d8a07ac1c69 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -210,7 +210,10 @@ type ( // (via AttachChasmRequestID). It gates the lazy request-ID sweep at transaction close so only // transactions that added an ID pay for the scan. chasmRequestIDsAdded bool - updateInfoUpdated map[string]struct{} + // holdNexusProgress keeps Nexus operation progress off the Workflow Task scheduled in the + // current transaction (see HoldNexusProgress). + holdNexusProgress bool + updateInfoUpdated map[string]struct{} // following xxxUserDataUpdated fields are for tracking if activity/timer user data updated. // This help to determine if we need to update transition history: For // user data change, we need to update transition history. No update for @@ -3642,6 +3645,10 @@ func (ms *MutableStateImpl) AddWorkflowTaskScheduledEventAsHeartbeat( return ms.workflowTaskManager.AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration, originalScheduledTimestamp, workflowTaskType) } +func (ms *MutableStateImpl) HoldNexusProgress() { + ms.holdNexusProgress = true +} + func (ms *MutableStateImpl) ApplyTransientWorkflowTaskScheduled() (*historyi.WorkflowTaskInfo, error) { return ms.workflowTaskManager.ApplyTransientWorkflowTaskScheduled() } @@ -8584,6 +8591,7 @@ func (ms *MutableStateImpl) cleanupTransaction() error { ms.executionStateUpdated = false ms.workflowTaskUpdated = false ms.chasmRequestIDsAdded = false + ms.holdNexusProgress = false ms.isResetStateUpdated = false ms.timeSkippingInfoUpdated = false ms.updateInfoUpdated = make(map[string]struct{}) diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go index d71b0eab4d9..57fa16a9a8e 100644 --- a/service/history/workflow/nexus_progress.go +++ b/service/history/workflow/nexus_progress.go @@ -38,7 +38,7 @@ func (ms *MutableStateImpl) hasPendingNexusProgress() bool { // attachNexusProgress puts the pending Nexus operation progress on a WorkflowTaskScheduled event // for a task no worker has seen yet. The event is the acknowledgment, so the progress is taken. func (ms *MutableStateImpl) attachNexusProgress(event *historypb.HistoryEvent) { - if !ms.hasPendingNexusProgress() { + if ms.holdNexusProgress || !ms.hasPendingNexusProgress() { return } wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index f3b350de4d9..7cf6cda84d3 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync" "time" "github.com/nexus-rpc/sdk-go/nexus" @@ -312,6 +313,133 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { s.Len(carriedNexusProgress(resp.GetHistory().GetEvents()), 2, "the progress is part of History") } +// TestNexusOperationProgressSkipsHeartbeatTasks checks that progress arriving during a long local +// activity stays off the tasks a worker forces to heartbeat it, and rides the next normal task. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(chasmEnabled bool) { + if !chasmEnabled { + // An HSM caller refuses progress; TestNexusOperationProgress covers it. + return + } + env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + + var callbackToken, callbackURL string + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + callbackToken = options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) + + laStarted := make(chan struct{}) + var laStartedOnce sync.Once + releaseLA := make(chan struct{}) + longLocalActivity := func(ctx context.Context) error { + laStartedOnce.Do(func() { close(laStarted) }) + select { + case <-releaseLA: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + callerWF := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(endpointName, "service") + fut := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}) + if err := fut.GetNexusOperationExecution().Get(ctx, nil); err != nil { + return "", err + } + laCtx := workflow.WithLocalActivityOptions(ctx, workflow.LocalActivityOptions{StartToCloseTimeout: time.Minute}) + if err := workflow.ExecuteLocalActivity(laCtx, longLocalActivity).Get(ctx, nil); err != nil { + return "", err + } + var result string + err := fut.Get(ctx, &result) + return result, err + } + w := worker.New(env.SdkClient(), taskQueue, worker.Options{}) + w.RegisterWorkflow(callerWF) + w.RegisterActivity(longLocalActivity) + s.NoError(w.Start()) + defer w.Stop() + + // A short task timeout makes the worker heartbeat the local activity often. + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + TaskQueue: taskQueue, + WorkflowTaskTimeout: time.Second, + }, callerWF) + s.NoError(err) + wfExec := &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()} + history := func() []*historypb.HistoryEvent { + return env.GetHistory(env.Namespace().String(), wfExec) + } + select { + case <-laStarted: + case <-ctx.Done(): + s.FailNow("the local activity did not start") + } + + beforeProgress := history() + status, err := postNexusProgress(ctx, callbackURL, callbackToken, `{"position": "p1", "counter": 1}`) + s.NoError(err) + s.Equal(http.StatusOK, status) + countScheduled := func(hist []*historypb.HistoryEvent) int { + var n int + for _, event := range hist { + if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + n++ + } + } + return n + } + var duringLA []*historypb.HistoryEvent + s.Await(func(s *NexusWorkflowTestSuite) { + duringLA = history()[len(beforeProgress):] + s.GreaterOrEqual(countScheduled(duringLA), 2, "the worker keeps heartbeating the local activity") + }, 10*time.Second, 50*time.Millisecond) + s.Empty(carriedNexusProgress(duringLA), "no heartbeat task carries progress") + + close(releaseLA) + var afterLA []*historypb.HistoryEvent + s.Await(func(s *NexusWorkflowTestSuite) { + afterLA = history()[len(beforeProgress):] + s.Len(carriedNexusProgress(afterLA), 1) + }, 10*time.Second, 50*time.Millisecond) + markerIdx := -1 + carrierIdx := -1 + for i, event := range afterLA { + if event.GetEventType() == enumspb.EVENT_TYPE_MARKER_RECORDED { + markerIdx = i + } + if len(event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()) > 0 { + carrierIdx = i + } + } + s.Positive(markerIdx, "the local activity's result is recorded") + s.Greater(carrierIdx, markerIdx, "the first normal task after the heartbeats carries the progress") + s.Equal(int64(1), carriedNexusProgress(afterLA)[0].GetCounter()) + + s.NoError(s.sendNexusCompletionRequest(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ + Result: testcore.MustToPayload(s.T(), "result"), + Header: nexus.Header{commonnexus.CallbackTokenHeader: callbackToken}, + })) + var result string + s.NoError(run.Get(ctx, &result)) + s.Equal("result", result) + + replayer := worker.NewWorkflowReplayer() + replayer.RegisterWorkflow(callerWF) + s.NoError(replayer.ReplayWorkflowHistory(nil, &historypb.History{Events: history()})) +} + func (s *NexusWorkflowTestSuite) scheduledEventID(hist []*historypb.HistoryEvent) int64 { return s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() } From c352e4feb56a30d4a1b4da10d43ecda083beb552 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 9 Oct 2026 20:53:37 -0700 Subject: [PATCH 3/9] Held Nexus progress off the retries of a heartbeat task too. A retried heartbeat task is replayed within the same heartbeat chain, so the hold is persisted on the Workflow task info and released only when a task that is not a retry is scheduled. --- api/persistence/v1/executions.pb.go | 23 +++- .../api/persistence/v1/executions.proto | 3 + service/history/interfaces/mutable_state.go | 4 +- .../history/workflow/mutable_state_impl.go | 4 +- service/history/workflow/nexus_progress.go | 2 +- .../workflow/workflow_task_state_machine.go | 6 + tests/nexus_progress_test.go | 114 ++++++++++++++++++ 7 files changed, 145 insertions(+), 11 deletions(-) diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go index 5206f098e99..651c473b875 100644 --- a/api/persistence/v1/executions.pb.go +++ b/api/persistence/v1/executions.pb.go @@ -185,10 +185,13 @@ type WorkflowExecutionInfo struct { // Used by the TemporalReportedProblems search attribute to track continuous failure count. // (-- api-linter: core::0140::prepositions=disabled // aip.dev/not-precedent: "since" is needed here. --) - WorkflowTaskAttemptsSinceLastSuccess int32 `protobuf:"varint,111,opt,name=workflow_task_attempts_since_last_success,json=workflowTaskAttemptsSinceLastSuccess,proto3" json:"workflow_task_attempts_since_last_success,omitempty"` - CancelRequested bool `protobuf:"varint,29,opt,name=cancel_requested,json=cancelRequested,proto3" json:"cancel_requested,omitempty"` - CancelRequestId string `protobuf:"bytes,32,opt,name=cancel_request_id,json=cancelRequestId,proto3" json:"cancel_request_id,omitempty"` - StickyTaskQueue string `protobuf:"bytes,33,opt,name=sticky_task_queue,json=stickyTaskQueue,proto3" json:"sticky_task_queue,omitempty"` + WorkflowTaskAttemptsSinceLastSuccess int32 `protobuf:"varint,111,opt,name=workflow_task_attempts_since_last_success,json=workflowTaskAttemptsSinceLastSuccess,proto3" json:"workflow_task_attempts_since_last_success,omitempty"` + // Set while the pending Workflow Task, or a retry of it, was forced by a worker to heartbeat a + // local activity. Nexus operation progress waits for a task without it. + WorkflowTaskHoldsNexusProgress bool `protobuf:"varint,118,opt,name=workflow_task_holds_nexus_progress,json=workflowTaskHoldsNexusProgress,proto3" json:"workflow_task_holds_nexus_progress,omitempty"` + CancelRequested bool `protobuf:"varint,29,opt,name=cancel_requested,json=cancelRequested,proto3" json:"cancel_requested,omitempty"` + CancelRequestId string `protobuf:"bytes,32,opt,name=cancel_request_id,json=cancelRequestId,proto3" json:"cancel_request_id,omitempty"` + StickyTaskQueue string `protobuf:"bytes,33,opt,name=sticky_task_queue,json=stickyTaskQueue,proto3" json:"sticky_task_queue,omitempty"` // (-- api-linter: core::0140::prepositions=disabled // aip.dev/not-precedent: "to" is used to indicate interval. --) StickyScheduleToStartTimeout *durationpb.Duration `protobuf:"bytes,34,opt,name=sticky_schedule_to_start_timeout,json=stickyScheduleToStartTimeout,proto3" json:"sticky_schedule_to_start_timeout,omitempty"` @@ -644,6 +647,13 @@ func (x *WorkflowExecutionInfo) GetWorkflowTaskAttemptsSinceLastSuccess() int32 return 0 } +func (x *WorkflowExecutionInfo) GetWorkflowTaskHoldsNexusProgress() bool { + if x != nil { + return x.WorkflowTaskHoldsNexusProgress + } + return false +} + func (x *WorkflowExecutionInfo) GetCancelRequested() bool { if x != nil { return x.CancelRequested @@ -4985,7 +4995,7 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\x05R\x03key\x12D\n" + "\x05value\x18\x02 \x01(\v2..temporal.server.api.persistence.v1.QueueStateR\x05value:\x028\x01J\x04\b\x04\x10\x05J\x04\b\x05\x10\x06J\x04\b\b\x10\tJ\x04\b\t\x10\n" + "J\x04\b\n" + - "\x10\vJ\x04\b\v\x10\fJ\x04\b\f\x10\rJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11\"\xb9D\n" + + "\x10\vJ\x04\b\v\x10\fJ\x04\b\f\x10\rJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11\"\x85E\n" + "\x15WorkflowExecutionInfo\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12\x1f\n" + "\vworkflow_id\x18\x02 \x01(\tR\n" + @@ -5024,7 +5034,8 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "\x16workflow_task_build_id\x18X \x01(\tR\x13workflowTaskBuildId\x12S\n" + "'workflow_task_build_id_redirect_counter\x18Y \x01(\x03R\"workflowTaskBuildIdRedirectCounter\x12.\n" + "\x13workflow_task_stamp\x18m \x01(\x05R\x11workflowTaskStamp\x12W\n" + - ")workflow_task_attempts_since_last_success\x18o \x01(\x05R$workflowTaskAttemptsSinceLastSuccess\x12)\n" + + ")workflow_task_attempts_since_last_success\x18o \x01(\x05R$workflowTaskAttemptsSinceLastSuccess\x12J\n" + + "\"workflow_task_holds_nexus_progress\x18v \x01(\bR\x1eworkflowTaskHoldsNexusProgress\x12)\n" + "\x10cancel_requested\x18\x1d \x01(\bR\x0fcancelRequested\x12*\n" + "\x11cancel_request_id\x18 \x01(\tR\x0fcancelRequestId\x12*\n" + "\x11sticky_task_queue\x18! \x01(\tR\x0fstickyTaskQueue\x12a\n" + diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto index 643a28dbfcc..bfae191f2b2 100644 --- a/proto/internal/temporal/server/api/persistence/v1/executions.proto +++ b/proto/internal/temporal/server/api/persistence/v1/executions.proto @@ -113,6 +113,9 @@ message WorkflowExecutionInfo { // (-- api-linter: core::0140::prepositions=disabled // aip.dev/not-precedent: "since" is needed here. --) int32 workflow_task_attempts_since_last_success = 111; + // Set while the pending Workflow Task, or a retry of it, was forced by a worker to heartbeat a + // local activity. Nexus operation progress waits for a task without it. + bool workflow_task_holds_nexus_progress = 118; bool cancel_requested = 29; string cancel_request_id = 32; diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index cfa79a46fbb..8dd7b195321 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -80,8 +80,8 @@ type ( AddFirstWorkflowTaskScheduled(parentClock *clockspb.VectorClock, event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error) AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *timestamppb.Timestamp, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) - // HoldNexusProgress keeps pending Nexus operation progress off any Workflow Task scheduled in - // this transaction, so it waits for a later task. + // HoldNexusProgress keeps pending Nexus operation progress off the Workflow Task scheduled in + // this transaction and off its retries, so it waits for a later task. HoldNexusProgress() AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion, int64) (*historypb.HistoryEvent, *WorkflowTaskInfo, error) AddWorkflowTaskTimedOutEvent(workflowTask *WorkflowTaskInfo) (*historypb.HistoryEvent, error) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index d8a07ac1c69..1e1041c8fbe 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -210,8 +210,8 @@ type ( // (via AttachChasmRequestID). It gates the lazy request-ID sweep at transaction close so only // transactions that added an ID pay for the scan. chasmRequestIDsAdded bool - // holdNexusProgress keeps Nexus operation progress off the Workflow Task scheduled in the - // current transaction (see HoldNexusProgress). + // holdNexusProgress marks the Workflow Task scheduled in the current transaction as one that + // holds Nexus operation progress (see HoldNexusProgress). holdNexusProgress bool updateInfoUpdated map[string]struct{} // following xxxUserDataUpdated fields are for tracking if activity/timer user data updated. diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go index 57fa16a9a8e..dccf502a8f0 100644 --- a/service/history/workflow/nexus_progress.go +++ b/service/history/workflow/nexus_progress.go @@ -38,7 +38,7 @@ func (ms *MutableStateImpl) hasPendingNexusProgress() bool { // attachNexusProgress puts the pending Nexus operation progress on a WorkflowTaskScheduled event // for a task no worker has seen yet. The event is the acknowledgment, so the progress is taken. func (ms *MutableStateImpl) attachNexusProgress(event *historypb.HistoryEvent) { - if ms.holdNexusProgress || !ms.hasPendingNexusProgress() { + if ms.executionInfo.GetWorkflowTaskHoldsNexusProgress() || !ms.hasPendingNexusProgress() { return } wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index 6f59202d026..f33da96a41b 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -318,6 +318,12 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( return nil, m.ms.createInternalServerError(opTag) } + // A heartbeat task holds Nexus progress, and so do its retries, which SDKs replay within the + // same heartbeat chain. Any other task releases it. + if m.ms.holdNexusProgress || !m.ms.IsTransientWorkflowTask() { + m.ms.executionInfo.WorkflowTaskHoldsNexusProgress = m.ms.holdNexusProgress + } + // Create real WorkflowTaskScheduledEvent when workflow task: // - is not transient (is not continuously failing) // and diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index 7cf6cda84d3..3a519493aa6 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -9,10 +9,12 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" + commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" nexuspb "go.temporal.io/api/nexus/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -440,6 +442,118 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c s.NoError(replayer.ReplayWorkflowHistory(nil, &historypb.History{Events: history()})) } +// TestNexusOperationProgressSkipsHeartbeatRetries checks that the retry of a failed heartbeat task +// holds progress like the task it retries, and that the next normal task carries it. A Signal +// arrives before the retry starts, so the retry gets a scheduled event of its own. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries(chasmEnabled bool) { + if !chasmEnabled { + // An HSM caller refuses progress; TestNexusOperationProgress covers it. + return + } + env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + ns := env.Namespace().String() + + var callbackToken, callbackURL string + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + callbackToken = options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) + + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + s.NoError(err) + frontend := env.FrontendClient() + poll := func() *workflowservice.PollWorkflowTaskQueueResponse { + resp, err := frontend.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + Identity: "test", + }) + s.NoError(err) + s.NotEmpty(resp.GetTaskToken()) + return resp + } + respond := func(task *workflowservice.PollWorkflowTaskQueueResponse, forceCreate bool, commands ...*commandpb.Command) { + _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + ForceCreateNewWorkflowTask: forceCreate, + }) + s.NoError(err) + } + // lastScheduled is the scheduled event of the task a poll answered. + lastScheduled := func(task *workflowservice.PollWorkflowTaskQueueResponse) *historypb.HistoryEvent { + events := task.GetHistory().GetEvents() + for i := len(events) - 1; i >= 0; i-- { + if events[i].GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + return events[i] + } + } + s.FailNow("the task has no scheduled event") + return nil + } + + respond(poll(), false, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpointName, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(s.T(), "input"), + }, + }, + }) + startedTask := poll() + s.RequireHistoryEvent(startedTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + + // The worker forces a heartbeat task, takes it, gets progress meanwhile, and fails it. + respond(startedTask, true) + heartbeatTask := poll() + s.Empty(lastScheduled(heartbeatTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()) + status, err := postNexusProgress(ctx, callbackURL, callbackToken, `{"position": "p1", "counter": 1}`) + s.NoError(err) + s.Equal(http.StatusOK, status) + _, err = frontend.RespondWorkflowTaskFailed(ctx, &workflowservice.RespondWorkflowTaskFailedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: heartbeatTask.GetTaskToken(), + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, + }) + s.NoError(err) + s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "wake", nil)) + + retryTask := poll() + s.RequireHistoryEvent(retryTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED) + s.Empty(lastScheduled(retryTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), + "the retry of a heartbeat task carries no progress") + respond(retryTask, false) + + normalTask := poll() + carried := lastScheduled(normalTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + s.Len(carried, 1, "the next normal task carries the progress") + s.Equal(int64(1), carried[0].GetCounter()) + respond(normalTask, false, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }) + s.NoError(run.Get(ctx, nil)) +} + func (s *NexusWorkflowTestSuite) scheduledEventID(hist []*historypb.HistoryEvent) int64 { return s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() } From 82e9f2ab0bed2061306ec752bae6d9c875d8b3af Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 9 Oct 2026 21:02:14 -0700 Subject: [PATCH 4/9] Sent folded Nexus progress on one follow-up Workflow Task. Progress that folds into a scheduled task is newer than what that task's event carries, so the Workflow never saw the highest counter. It now waits for the next task once that one starts. --- chasm/lib/nexusoperation/operation.go | 12 ++---- chasm/lib/nexusoperation/progress_test.go | 24 +++-------- .../workflow/gen/workflowpb/v1/state.pb.go | 18 ++++++-- chasm/lib/workflow/nexus_progress.go | 42 ++++++++++++------- chasm/lib/workflow/nexus_progress_test.go | 16 +++++-- chasm/lib/workflow/proto/v1/state.proto | 4 ++ tests/nexus_progress_test.go | 16 ++++--- 7 files changed, 76 insertions(+), 56 deletions(-) diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index f2c1da02a64..cdb8423090c 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -111,7 +111,7 @@ type Operation struct { // PendingProgress is the latest progress no Workflow Task has carried yet. PendingProgress chasm.Field[*nexuspb.NexusOperationProgress] - // DeliveredProgress is the latest progress a Workflow Task carried or folded. A delivery whose + // DeliveredProgress is the latest progress a Workflow Task carried. A delivery whose // counter is not above it is stale. DeliveredProgress chasm.Field[*nexuspb.NexusOperationProgress] } @@ -119,11 +119,9 @@ type Operation struct { // ProgressStore is implemented by a parent that delivers the operation's progress to its caller. // A parent without it, such as none for a standalone operation, drops progress. type ProgressStore interface { - // OnNexusOperationProgress records that the operation holds pending progress. + // OnNexusOperationProgress records that the operation holds pending progress. The parent + // decides which Workflow Task takes it. OnNexusOperationProgress(ctx chasm.MutableContext, operation *Operation) error - // NexusOperationProgressFolds reports whether the operation's latest progress rides a Workflow - // Task that has not started, which newer progress can fold into. - NexusOperationProgressFolds(ctx chasm.Context, operation *Operation) bool } // NewOperation creates a new Operation component with the given persisted state. @@ -364,10 +362,6 @@ func (o *Operation) onProgress(ctx chasm.MutableContext, progress *nexuspb.Nexus return nil } } - if store.NexusOperationProgressFolds(ctx, o) { - o.DeliveredProgress = chasm.NewDataField(ctx, progress) - return nil - } o.PendingProgress = chasm.NewDataField(ctx, progress) return store.OnNexusOperationProgress(ctx, o) } diff --git a/chasm/lib/nexusoperation/progress_test.go b/chasm/lib/nexusoperation/progress_test.go index c4dcb40109d..5d3c3f0a831 100644 --- a/chasm/lib/nexusoperation/progress_test.go +++ b/chasm/lib/nexusoperation/progress_test.go @@ -10,12 +10,10 @@ import ( nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" ) -// progressStoreComponent is a parent that takes progress, and can say that a scheduled task the -// operation's progress rides has not started. +// progressStoreComponent is a parent that takes progress. type progressStoreComponent struct { mockStoreComponent - folds bool pending int } @@ -24,10 +22,6 @@ func (s *progressStoreComponent) OnNexusOperationProgress(chasm.MutableContext, return nil } -func (s *progressStoreComponent) NexusOperationProgressFolds(chasm.Context, *Operation) bool { - return s.folds -} - func TestHandleNexusProgress(t *testing.T) { progress := func(counter int64) *persistencespb.ChasmNexusCompletion { return &persistencespb.ChasmNexusCompletion{ @@ -94,7 +88,7 @@ func TestHandleNexusProgress(t *testing.T) { require.Equal(t, 2, store.pending) }) - t.Run("FoldsIntoAnUnstartedTask", func(t *testing.T) { + t.Run("NewerProgressWaitsUntilATaskCarriesIt", func(t *testing.T) { ctx := &chasm.MockMutableContext{} store := &progressStoreComponent{} op := newOp(nexusoperationpb.OPERATION_STATUS_STARTED, store) @@ -102,17 +96,11 @@ func TestHandleNexusProgress(t *testing.T) { _, ok := op.TakePendingProgress(ctx) require.True(t, ok) - // The task carrying counter 1 has not started, so newer progress rides it. - store.folds = true + // Whether it folds into a task that has not started is the parent's call; the operation + // keeps it until a task carries it. require.NoError(t, op.HandleNexusCompletion(ctx, progress(2))) - require.Zero(t, pendingCounter(ctx, op), "folded progress asks for no task of its own") - require.Equal(t, int64(2), deliveredCounter(ctx, op)) - require.Equal(t, 1, store.pending) - - // Once that task starts, newer progress waits for the next one. - store.folds = false - require.NoError(t, op.HandleNexusCompletion(ctx, progress(3))) - require.Equal(t, int64(3), pendingCounter(ctx, op)) + require.Equal(t, int64(2), pendingCounter(ctx, op)) + require.Equal(t, int64(1), deliveredCounter(ctx, op)) require.Equal(t, 2, store.pending) }) diff --git a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go index 078d9fc1bcb..062b2481105 100644 --- a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go +++ b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go @@ -179,7 +179,11 @@ type NexusProgressState struct { Pending []int64 `protobuf:"varint,1,rep,packed,name=pending,proto3" json:"pending,omitempty"` // Operations whose latest progress rides a scheduled Workflow Task that has not started yet. // Newer progress for one of them folds into that task, since the read it triggers sees it. - Scheduled []int64 `protobuf:"varint,2,rep,packed,name=scheduled,proto3" json:"scheduled,omitempty"` + Scheduled []int64 `protobuf:"varint,2,rep,packed,name=scheduled,proto3" json:"scheduled,omitempty"` + // Operations whose progress folded into that task but is newer than what its scheduled event + // carries. Once the task starts, fails or times out they wait like pending ones, so one follow-up + // task carries the highest counter. + Folded []int64 `protobuf:"varint,3,rep,packed,name=folded,proto3" json:"folded,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -228,6 +232,13 @@ func (x *NexusProgressState) GetScheduled() []int64 { return nil } +func (x *NexusProgressState) GetFolded() []int64 { + if x != nil { + return x.Folded + } + return nil +} + var File_temporal_server_chasm_lib_workflow_proto_v1_state_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc = "" + @@ -239,10 +250,11 @@ const file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc = "" "\x1bNexusCancellationParentData\x12,\n" + "\x12requested_event_id\x18\x01 \x01(\x03R\x10requestedEventId\"/\n" + "\x12IncomingSignalData\x12\x19\n" + - "\bevent_id\x18\x01 \x01(\x03R\aeventId\"L\n" + + "\bevent_id\x18\x01 \x01(\x03R\aeventId\"d\n" + "\x12NexusProgressState\x12\x18\n" + "\apending\x18\x01 \x03(\x03R\apending\x12\x1c\n" + - "\tscheduled\x18\x02 \x03(\x03R\tscheduledBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" + "\tscheduled\x18\x02 \x03(\x03R\tscheduled\x12\x16\n" + + "\x06folded\x18\x03 \x03(\x03R\x06foldedBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" var ( file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescOnce sync.Once diff --git a/chasm/lib/workflow/nexus_progress.go b/chasm/lib/workflow/nexus_progress.go index 53657a281e6..1883d9ca028 100644 --- a/chasm/lib/workflow/nexus_progress.go +++ b/chasm/lib/workflow/nexus_progress.go @@ -32,7 +32,7 @@ func (w *Workflow) nexusProgressIndex(ctx chasm.Context) *chasmworkflowpb.NexusP // setNexusProgressIndex stores the index, or drops it when it is empty, so a Workflow whose // operations report no progress carries nothing. func (w *Workflow) setNexusProgressIndex(ctx chasm.MutableContext, index *chasmworkflowpb.NexusProgressState) { - if len(index.GetPending()) == 0 && len(index.GetScheduled()) == 0 { + if len(index.GetPending()) == 0 && len(index.GetScheduled()) == 0 && len(index.GetFolded()) == 0 { if _, ok := w.NexusProgress.TryGet(ctx); ok { w.NexusProgress = chasm.NewEmptyField[*chasmworkflowpb.NexusProgressState]() } @@ -42,7 +42,8 @@ func (w *Workflow) setNexusProgressIndex(ctx chasm.MutableContext, index *chasmw } // OnNexusOperationProgress records that an operation holds progress for the next Workflow Task -// scheduled event. +// scheduled event. Progress for an operation that rides a scheduled Workflow Task that has not +// started folds into that task: it asks for no task of its own until that one starts. func (w *Workflow) OnNexusOperationProgress(ctx chasm.MutableContext, op *nexusoperation.Operation) error { key, ok := nexusProgressKey(op) if !ok { @@ -50,21 +51,18 @@ func (w *Workflow) OnNexusOperationProgress(ctx chasm.MutableContext, op *nexuso return nil } index := common.CloneProto(w.nexusProgressIndex(ctx)) - if !slices.Contains(index.Pending, key) { - index.Pending = append(index.Pending, key) - slices.Sort(index.Pending) + waiting := &index.Pending + if slices.Contains(index.Scheduled, key) { + waiting = &index.Folded + } + if !slices.Contains(*waiting, key) { + *waiting = append(*waiting, key) + slices.Sort(*waiting) } w.setNexusProgressIndex(ctx, index) return nil } -// NexusOperationProgressFolds reports whether an operation's latest progress rides a scheduled -// Workflow Task that has not started. That task's read will see what newer progress reports. -func (w *Workflow) NexusOperationProgressFolds(ctx chasm.Context, op *nexusoperation.Operation) bool { - key, ok := nexusProgressKey(op) - return ok && slices.Contains(w.nexusProgressIndex(ctx).GetScheduled(), key) -} - // HasPendingNexusProgress reports whether any operation holds progress for the next Workflow Task // scheduled event. It reads one field, so it is cheap on every transaction close. func (w *Workflow) HasPendingNexusProgress(ctx chasm.Context) bool { @@ -73,7 +71,8 @@ func (w *Workflow) HasPendingNexusProgress(ctx chasm.Context) bool { // HasScheduledNexusProgress reports whether a scheduled Workflow Task carries progress. func (w *Workflow) HasScheduledNexusProgress(ctx chasm.Context) bool { - return len(w.nexusProgressIndex(ctx).GetScheduled()) > 0 + index := w.nexusProgressIndex(ctx) + return len(index.GetScheduled()) > 0 || len(index.GetFolded()) > 0 } // TakeNexusProgress returns the pending progress of every operation in scheduled event ID order, @@ -105,14 +104,23 @@ func (w *Workflow) TakeNexusProgress(ctx chasm.MutableContext) []*nexuspb.NexusO } // ClearScheduledNexusProgress forgets which operations the scheduled Workflow Task carries. Called -// once that task starts, fails or times out: from then on newer progress needs a task of its own. +// once that task starts, fails or times out: from then on newer progress needs a task of its own, +// and progress that folded into it waits for the next task, since its scheduled event carries an +// older counter. func (w *Workflow) ClearScheduledNexusProgress(ctx chasm.MutableContext) { index := w.nexusProgressIndex(ctx) - if len(index.GetScheduled()) == 0 { + if len(index.GetScheduled()) == 0 && len(index.GetFolded()) == 0 { return } index = common.CloneProto(index) + for _, key := range index.Folded { + if !slices.Contains(index.Pending, key) { + index.Pending = append(index.Pending, key) + } + } + slices.Sort(index.Pending) index.Scheduled = nil + index.Folded = nil w.setNexusProgressIndex(ctx, index) } @@ -137,11 +145,13 @@ func (w *Workflow) DropPendingNexusProgress(ctx chasm.MutableContext) { // forgetNexusProgress removes a closed operation from the index. func (w *Workflow) forgetNexusProgress(ctx chasm.MutableContext, key int64) { index := w.nexusProgressIndex(ctx) - if !slices.Contains(index.GetPending(), key) && !slices.Contains(index.GetScheduled(), key) { + if !slices.Contains(index.GetPending(), key) && !slices.Contains(index.GetScheduled(), key) && + !slices.Contains(index.GetFolded(), key) { return } index = common.CloneProto(index) index.Pending = slices.DeleteFunc(index.Pending, func(k int64) bool { return k == key }) index.Scheduled = slices.DeleteFunc(index.Scheduled, func(k int64) bool { return k == key }) + index.Folded = slices.DeleteFunc(index.Folded, func(k int64) bool { return k == key }) w.setNexusProgressIndex(ctx, index) } diff --git a/chasm/lib/workflow/nexus_progress_test.go b/chasm/lib/workflow/nexus_progress_test.go index 984638e45aa..dd386039f18 100644 --- a/chasm/lib/workflow/nexus_progress_test.go +++ b/chasm/lib/workflow/nexus_progress_test.go @@ -61,21 +61,25 @@ func TestNexusProgressIndex(t *testing.T) { require.True(t, wf.HasScheduledNexusProgress(ctx)) }) - t.Run("FoldsIntoTheUnstartedTaskUntilItStarts", func(t *testing.T) { + t.Run("FoldsIntoTheUnstartedTaskThenWaitsForTheNextOne", func(t *testing.T) { ctx := &chasm.MockMutableContext{} wf := newWorkflowWithOperations(t, ctx, 5, 9) deliver(t, ctx, wf, 5, 1) require.Len(t, wf.TakeNexusProgress(ctx), 1) deliver(t, ctx, wf, 5, 2) + deliver(t, ctx, wf, 5, 3) require.False(t, wf.HasPendingNexusProgress(ctx), "operation 5 rides the unstarted task") deliver(t, ctx, wf, 9, 1) require.True(t, wf.HasPendingNexusProgress(ctx), "operation 9 is not on that task") + require.Equal(t, map[int64]int64{9: 1}, counters(wf.TakeNexusProgress(ctx)), + "folded progress is not taken while its task waits to start") + // The task started with counter 1 on its event, so counter 3 needs one more task. wf.ClearScheduledNexusProgress(ctx) require.False(t, wf.HasScheduledNexusProgress(ctx)) - deliver(t, ctx, wf, 5, 3) - require.Equal(t, map[int64]int64{5: 3, 9: 1}, counters(wf.TakeNexusProgress(ctx))) + require.True(t, wf.HasPendingNexusProgress(ctx)) + require.Equal(t, map[int64]int64{5: 3}, counters(wf.TakeNexusProgress(ctx))) }) t.Run("ForgetsAClosedOperation", func(t *testing.T) { @@ -83,8 +87,12 @@ func TestNexusProgressIndex(t *testing.T) { wf := newWorkflowWithOperations(t, ctx, 5, 9) deliver(t, ctx, wf, 5, 1) deliver(t, ctx, wf, 9, 1) + require.Len(t, wf.TakeNexusProgress(ctx), 2) + deliver(t, ctx, wf, 5, 2) + deliver(t, ctx, wf, 9, 2) wf.removeNexusOperation(ctx, 5) - require.Equal(t, map[int64]int64{9: 1}, counters(wf.TakeNexusProgress(ctx))) + wf.ClearScheduledNexusProgress(ctx) + require.Equal(t, map[int64]int64{9: 2}, counters(wf.TakeNexusProgress(ctx)), "a closed operation's folded progress is gone") wf.removeNexusOperation(ctx, 9) require.False(t, wf.HasScheduledNexusProgress(ctx)) _, ok := wf.NexusProgress.TryGet(ctx) diff --git a/chasm/lib/workflow/proto/v1/state.proto b/chasm/lib/workflow/proto/v1/state.proto index 4021ca65e17..9bad1825ccd 100644 --- a/chasm/lib/workflow/proto/v1/state.proto +++ b/chasm/lib/workflow/proto/v1/state.proto @@ -34,4 +34,8 @@ message NexusProgressState { // Operations whose latest progress rides a scheduled Workflow Task that has not started yet. // Newer progress for one of them folds into that task, since the read it triggers sees it. repeated int64 scheduled = 2; + // Operations whose progress folded into that task but is newer than what its scheduled event + // carries. Once the task starts, fails or times out they wait like pending ones, so one follow-up + // task carries the highest counter. + repeated int64 folded = 3; } diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index 3a519493aa6..57eb8a4d0c3 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -167,8 +167,9 @@ func carriedNexusProgress(hist []*historypb.HistoryEvent) []*nexuspb.NexusOperat } // TestNexusOperationProgress checks how progress reaches a CHASM caller Workflow: on Workflow Task -// scheduled events with no event of its own, folded into a scheduled task that has not started, -// dropped after completion, and replayed. The HSM caller refuses progress. +// scheduled events with no event of its own, folded into a scheduled task that has not started and +// then carried by one follow-up task so the highest counter arrives, dropped after completion, and +// replayed. The HSM caller refuses progress. func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) ctx := s.Context() @@ -262,21 +263,24 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { s.Equal(http.StatusOK, progress(2), "a stale counter is accepted and dropped") w = newWorker() afterFold := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { - s.Greater(len(hist), len(beforeBurst)) + carried := carriedNexusProgress(hist[len(beforeBurst):]) + s.NotEmpty(carried) + s.Equal(int64(3), carried[len(carried)-1].GetCounter(), "the Workflow sees the highest counter") }) folded := afterFold[len(beforeBurst):] onlyWorkflowTaskEvents(folded) carried := carriedNexusProgress(folded) - s.Len(carried, 1, "one task carries the whole burst") + s.Len(carried, 2) s.Equal(scheduledEventID, carried[0].GetScheduledEventId()) s.Equal(int64(1), carried[0].GetCounter(), "the task that was already scheduled carries what it had") + s.Equal(int64(3), carried[1].GetCounter(), "one follow-up task carries what folded into it") var scheduledTasks int for _, event := range folded { if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { scheduledTasks++ } } - s.Equal(1, scheduledTasks) + s.Equal(2, scheduledTasks, "the burst costs at most one extra task") // After that task started, newer progress needs a task of its own. s.Equal(http.StatusOK, progress(4)) @@ -312,7 +316,7 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { Execution: wfExec, }) s.NoError(err) - s.Len(carriedNexusProgress(resp.GetHistory().GetEvents()), 2, "the progress is part of History") + s.Len(carriedNexusProgress(resp.GetHistory().GetEvents()), 3, "the progress is part of History") } // TestNexusOperationProgressSkipsHeartbeatTasks checks that progress arriving during a long local From 30c5a0a75f3824833d4cf7947db25292741fe562 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 9 Oct 2026 23:11:43 -0700 Subject: [PATCH 5/9] Held Nexus progress off tasks a failover converts. A cluster rebuilt from replicated events never has the heartbeat hold marker, so a converted retry could carry progress. A test now pins that progress still reaches the Workflow around a rejected Update. --- .../workflow/workflow_task_state_machine.go | 17 +- tests/nexus_progress_test.go | 148 ++++++++++++++++++ 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index f33da96a41b..948003ca759 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -330,6 +330,11 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( // - is not speculative. createWorkflowTaskScheduledEvent := !m.ms.IsTransientWorkflowTask() && workflowTaskType != enumsspb.WORKFLOW_TASK_TYPE_SPECULATIVE + // A cluster rebuilt from replicated events never learns workflow_task_holds_nexus_progress, so + // a task a failover converts may retry a heartbeat task without the hold. Such a task carries no + // progress; the progress waits for the task after it. + holdForFailover := false + // If while scheduling a workflow task and new events has come, then this workflow task cannot be a transient/speculative. // Flush any buffered events before creating the workflow task, otherwise it will result in invalid IDs for // transient/speculative workflow task and will cause in timeout processing to not work for transient workflow tasks. @@ -354,6 +359,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( m.ms.executionInfo.WorkflowTaskAttempt = 1 workflowTaskType = enumsspb.WORKFLOW_TASK_TYPE_NORMAL createWorkflowTaskScheduledEvent = true + holdForFailover = true } } @@ -374,7 +380,9 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( attempt, scheduleTime, ) - m.ms.attachNexusProgress(scheduledEvent) + if !holdForFailover { + m.ms.attachNexusProgress(scheduledEvent) + } scheduledEventID = scheduledEvent.GetEventId() } else { // WorkflowTaskScheduledEvent will be created later. @@ -592,8 +600,11 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent( workflowTask.Attempt, startTime, ) - // The worker has not seen this task yet, so its scheduled event can carry what is pending. - m.ms.attachNexusProgress(scheduledEvent) + // The worker has not seen this task yet, so its scheduled event can carry what is pending, + // unless a failover converted it (see holdForFailover in AddWorkflowTaskScheduledEventAsHeartbeat). + if workflowTask.Version == m.ms.GetCurrentVersion() { + m.ms.attachNexusProgress(scheduledEvent) + } scheduledEventID = scheduledEvent.GetEventId() } diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index 57eb8a4d0c3..5cf3a8eea31 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -15,6 +15,7 @@ import ( historypb "go.temporal.io/api/history/v1" nexuspb "go.temporal.io/api/nexus/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" + updatepb "go.temporal.io/api/update/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -23,6 +24,8 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/testvars" + "go.temporal.io/server/common/testing/updateutils" "go.temporal.io/server/tests/testcore" ) @@ -558,6 +561,151 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries s.NoError(run.Get(ctx, nil)) } +// TestNexusOperationProgressRidesSpeculativeTasks checks that progress reaches the Workflow when it +// arrives while a speculative task for an Update is pending, and the Update is then rejected. The +// progress transaction turns the speculative task into a normal one: one that has not started +// carries the progress, and one already started is kept, so a follow-up task carries it. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks(chasmEnabled bool) { + if !chasmEnabled { + // An HSM caller refuses progress; TestNexusOperationProgress covers it. + return + } + env := s.newTestEnv(chasmEnabled, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + ns := env.Namespace().String() + tv := testvars.New(s.T()) + updates := updateutils.New(s.T()) + + var callbackToken, callbackURL string + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + callbackToken = options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) + + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + s.NoError(err) + wfExec := &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()} + frontend := env.FrontendClient() + poll := func() *workflowservice.PollWorkflowTaskQueueResponse { + resp, err := frontend.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + Identity: "test", + }) + s.NoError(err) + s.NotEmpty(resp.GetTaskToken()) + return resp + } + respond := func(task *workflowservice.PollWorkflowTaskQueueResponse, commands ...*commandpb.Command) { + _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + }) + s.NoError(err) + } + rejectUpdate := func(task *workflowservice.PollWorkflowTaskQueueResponse) { + s.Len(task.GetMessages(), 1, "the task carries the Update request") + _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Messages: updates.UpdateRejectMessages(tv, task.GetMessages()[0]), + }) + s.NoError(err) + } + sendUpdate := func(updateID string) <-chan *updatepb.Outcome { + done := make(chan *updatepb.Outcome, 1) + go func() { + resp, err := frontend.UpdateWorkflowExecution(ctx, &workflowservice.UpdateWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: wfExec, + Request: &updatepb.Request{ + Meta: &updatepb.Meta{UpdateId: updateID}, + Input: &updatepb.Input{Name: "update"}, + }, + WaitPolicy: &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}, + }) + s.NoError(err) + done <- resp.GetOutcome() + }() + return done + } + // carried is the progress on the scheduled event of the task a poll answered. + carried := func(task *workflowservice.PollWorkflowTaskQueueResponse) []*nexuspb.NexusOperationProgress { + events := task.GetHistory().GetEvents() + for i := len(events) - 1; i >= 0; i-- { + if events[i].GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + return events[i].GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + } + } + return nil + } + postProgress := func(counter int) { + status, err := postNexusProgress(ctx, callbackURL, callbackToken, fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter)) + s.NoError(err) + s.Equal(http.StatusOK, status) + } + + respond(poll(), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpointName, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(s.T(), "input"), + }, + }, + }) + startedTask := poll() + s.RequireHistoryEvent(startedTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + respond(startedTask) + + // An Update schedules a speculative task, and progress arrives before any worker takes it. + firstUpdate := sendUpdate("unstarted") + s.Await(func(s *NexusWorkflowTestSuite) { + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID()) + s.NoError(err) + s.NotNil(desc.GetPendingWorkflowTask()) + }, 10*time.Second, 50*time.Millisecond) + postProgress(1) + unstartedTask := poll() + s.Len(carried(unstartedTask), 1, "the task that was speculative carries the progress") + s.Equal(int64(1), carried(unstartedTask)[0].GetCounter()) + rejectUpdate(unstartedTask) + s.NotNil((<-firstUpdate).GetFailure(), "the Update was rejected") + + // This time the worker has already started the speculative task when progress arrives. + secondUpdate := sendUpdate("started") + startedSpeculative := poll() + s.Len(startedSpeculative.GetMessages(), 1) + postProgress(2) + rejectUpdate(startedSpeculative) + s.NotNil((<-secondUpdate).GetFailure(), "the Update was rejected") + followUp := poll() + s.Len(carried(followUp), 1, "a follow-up task carries the progress the started task could not") + s.Equal(int64(2), carried(followUp)[0].GetCounter()) + respond(followUp, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }) + s.NoError(run.Get(ctx, nil)) +} + func (s *NexusWorkflowTestSuite) scheduledEventID(hist []*historypb.HistoryEvent) int64 { return s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() } From fcf27aa5308435612bfcfb81108c9aa4c63c087d Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 10 Oct 2026 13:01:55 -0700 Subject: [PATCH 6/9] Rate-limited the Workflow Tasks that Nexus operation progress schedules. A long stream could fill the caller's History, one task per delivery. Progress that a task can't carry now waits instead of being dropped. --- chasm/lib/nexusoperation/config.go | 9 ++ .../workflow/gen/workflowpb/v1/state.pb.go | 44 ++++-- .../gen/workflowpb/v1/tasks.go-helpers.pb.go | 43 ++++++ .../workflow/gen/workflowpb/v1/tasks.pb.go | 116 +++++++++++++++ chasm/lib/workflow/library.go | 6 + chasm/lib/workflow/nexus_progress.go | 72 +++++++--- chasm/lib/workflow/nexus_progress_tasks.go | 29 ++++ chasm/lib/workflow/nexus_progress_test.go | 50 ++++++- chasm/lib/workflow/proto/v1/state.proto | 8 ++ chasm/lib/workflow/proto/v1/tasks.proto | 9 ++ service/history/configs/config.go | 2 + .../workflow/mutable_state_impl_test.go | 89 ++++++++++++ service/history/workflow/nexus_progress.go | 47 ++++-- tests/nexus_progress_test.go | 136 ++++++++++++++++++ 14 files changed, 614 insertions(+), 46 deletions(-) create mode 100644 chasm/lib/workflow/gen/workflowpb/v1/tasks.go-helpers.pb.go create mode 100644 chasm/lib/workflow/gen/workflowpb/v1/tasks.pb.go create mode 100644 chasm/lib/workflow/nexus_progress_tasks.go create mode 100644 chasm/lib/workflow/proto/v1/tasks.proto diff --git a/chasm/lib/nexusoperation/config.go b/chasm/lib/nexusoperation/config.go index abeb2b7196c..87d258a73df 100644 --- a/chasm/lib/nexusoperation/config.go +++ b/chasm/lib/nexusoperation/config.go @@ -60,6 +60,15 @@ one per operation onto the caller's next Workflow Task scheduled event. When fal delivery is refused with a 400, which tells the handler to stop sending progress.`, ) +var ProgressMinInterval = dynamicconfig.NewNamespaceDurationSetting( + "nexusoperation.progressMinInterval", + time.Second, + `Experimental. The shortest time between a Workflow Task that carried Nexus operation progress +and the next task that progress schedules for the same caller Workflow. Progress that arrives +sooner waits, and newer progress folds into it. A task scheduled for any other reason still +carries it. This bounds the History a long stream costs the caller.`, +) + var ChasmWorkflowOperationsRolloutPercent = dynamicconfig.NewNamespaceIntSetting( "nexusoperation.chasmWorkflowOperationsRolloutPercent", 0, diff --git a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go index 062b2481105..ec7312386b0 100644 --- a/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go +++ b/chasm/lib/workflow/gen/workflowpb/v1/state.pb.go @@ -13,6 +13,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -183,7 +184,13 @@ type NexusProgressState struct { // Operations whose progress folded into that task but is newer than what its scheduled event // carries. Once the task starts, fails or times out they wait like pending ones, so one follow-up // task carries the highest counter. - Folded []int64 `protobuf:"varint,3,rep,packed,name=folded,proto3" json:"folded,omitempty"` + Folded []int64 `protobuf:"varint,3,rep,packed,name=folded,proto3" json:"folded,omitempty"` + // When a Workflow Task last carried progress. Progress waits at least + // nexusoperation.progressMinInterval after it before it schedules a task of its own, so a long + // stream can't fill the caller's History. + LastCarriedTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=last_carried_time,json=lastCarriedTime,proto3" json:"last_carried_time,omitempty"` + // When the armed NexusProgressReleaseTask lets waiting progress schedule a task. + ReleaseTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=release_time,json=releaseTime,proto3" json:"release_time,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -239,22 +246,38 @@ func (x *NexusProgressState) GetFolded() []int64 { return nil } +func (x *NexusProgressState) GetLastCarriedTime() *timestamppb.Timestamp { + if x != nil { + return x.LastCarriedTime + } + return nil +} + +func (x *NexusProgressState) GetReleaseTime() *timestamppb.Timestamp { + if x != nil { + return x.ReleaseTime + } + return nil +} + var File_temporal_server_chasm_lib_workflow_proto_v1_state_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDesc = "" + "\n" + - "7temporal/server/chasm/lib/workflow/proto/v1/state.proto\x12+temporal.server.chasm.lib.workflow.proto.v1\"|\n" + + "7temporal/server/chasm/lib/workflow/proto/v1/state.proto\x12+temporal.server.chasm.lib.workflow.proto.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"|\n" + "\x18NexusOperationParentData\x12,\n" + "\x12scheduled_event_id\x18\x01 \x01(\x03R\x10scheduledEventId\x122\n" + "\x15scheduled_event_token\x18\x02 \x01(\fR\x13scheduledEventToken\"K\n" + "\x1bNexusCancellationParentData\x12,\n" + "\x12requested_event_id\x18\x01 \x01(\x03R\x10requestedEventId\"/\n" + "\x12IncomingSignalData\x12\x19\n" + - "\bevent_id\x18\x01 \x01(\x03R\aeventId\"d\n" + + "\bevent_id\x18\x01 \x01(\x03R\aeventId\"\xeb\x01\n" + "\x12NexusProgressState\x12\x18\n" + "\apending\x18\x01 \x03(\x03R\apending\x12\x1c\n" + "\tscheduled\x18\x02 \x03(\x03R\tscheduled\x12\x16\n" + - "\x06folded\x18\x03 \x03(\x03R\x06foldedBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" + "\x06folded\x18\x03 \x03(\x03R\x06folded\x12F\n" + + "\x11last_carried_time\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x0flastCarriedTime\x12=\n" + + "\frelease_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\vreleaseTimeBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" var ( file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_rawDescOnce sync.Once @@ -274,13 +297,16 @@ var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_goTypes = []any (*NexusCancellationParentData)(nil), // 1: temporal.server.chasm.lib.workflow.proto.v1.NexusCancellationParentData (*IncomingSignalData)(nil), // 2: temporal.server.chasm.lib.workflow.proto.v1.IncomingSignalData (*NexusProgressState)(nil), // 3: temporal.server.chasm.lib.workflow.proto.v1.NexusProgressState + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp } var file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 4, // 0: temporal.server.chasm.lib.workflow.proto.v1.NexusProgressState.last_carried_time:type_name -> google.protobuf.Timestamp + 4, // 1: temporal.server.chasm.lib.workflow.proto.v1.NexusProgressState.release_time:type_name -> google.protobuf.Timestamp + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_temporal_server_chasm_lib_workflow_proto_v1_state_proto_init() } diff --git a/chasm/lib/workflow/gen/workflowpb/v1/tasks.go-helpers.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/tasks.go-helpers.pb.go new file mode 100644 index 00000000000..41200317c8b --- /dev/null +++ b/chasm/lib/workflow/gen/workflowpb/v1/tasks.go-helpers.pb.go @@ -0,0 +1,43 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package workflowpb + +import ( + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type NexusProgressReleaseTask to the protobuf v3 wire format +func (val *NexusProgressReleaseTask) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type NexusProgressReleaseTask from the protobuf v3 wire format +func (val *NexusProgressReleaseTask) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *NexusProgressReleaseTask) Size() int { + return proto.Size(val) +} + +// Equal returns whether two NexusProgressReleaseTask values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *NexusProgressReleaseTask) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *NexusProgressReleaseTask + switch t := that.(type) { + case *NexusProgressReleaseTask: + that1 = t + case NexusProgressReleaseTask: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/workflow/gen/workflowpb/v1/tasks.pb.go b/chasm/lib/workflow/gen/workflowpb/v1/tasks.pb.go new file mode 100644 index 00000000000..8f9134565e5 --- /dev/null +++ b/chasm/lib/workflow/gen/workflowpb/v1/tasks.pb.go @@ -0,0 +1,116 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/workflow/proto/v1/tasks.proto + +package workflowpb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Lets Nexus operation progress that waited out nexusoperation.progressMinInterval schedule a +// Workflow Task. +type NexusProgressReleaseTask struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NexusProgressReleaseTask) Reset() { + *x = NexusProgressReleaseTask{} + mi := &file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NexusProgressReleaseTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NexusProgressReleaseTask) ProtoMessage() {} + +func (x *NexusProgressReleaseTask) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NexusProgressReleaseTask.ProtoReflect.Descriptor instead. +func (*NexusProgressReleaseTask) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescGZIP(), []int{0} +} + +var File_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDesc = "" + + "\n" + + "7temporal/server/chasm/lib/workflow/proto/v1/tasks.proto\x12+temporal.server.chasm.lib.workflow.proto.v1\"\x1a\n" + + "\x18NexusProgressReleaseTaskBDZBgo.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpbb\x06proto3" + +var ( + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescOnce sync.Once + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescData []byte +) + +func file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescGZIP() []byte { + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescOnce.Do(func() { + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDesc), len(file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDesc))) + }) + return file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDescData +} + +var file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_goTypes = []any{ + (*NexusProgressReleaseTask)(nil), // 0: temporal.server.chasm.lib.workflow.proto.v1.NexusProgressReleaseTask +} +var file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_init() } +func file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_init() { + if File_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDesc), len(file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_depIdxs, + MessageInfos: file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_msgTypes, + }.Build() + File_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto = out.File + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_goTypes = nil + file_temporal_server_chasm_lib_workflow_proto_v1_tasks_proto_depIdxs = nil +} diff --git a/chasm/lib/workflow/library.go b/chasm/lib/workflow/library.go index 5170f37add2..ae3b9265051 100644 --- a/chasm/lib/workflow/library.go +++ b/chasm/lib/workflow/library.go @@ -82,6 +82,12 @@ func (l *library) Components() []*chasm.RegistrableComponent { } } +func (l *library) Tasks() []*chasm.RegistrableTask { + return []*chasm.RegistrableTask{ + chasm.NewRegistrablePureTask("nexusProgressRelease", nexusProgressReleaseTaskHandler{}), + } +} + // SetEventRegistryOnContext injects the event registry into a CHASM context. This is primarily // useful for tests that construct MockMutableContext directly. func SetEventRegistryOnContext[C chasm.Context](ctx C, registry *Registry) C { diff --git a/chasm/lib/workflow/nexus_progress.go b/chasm/lib/workflow/nexus_progress.go index 1883d9ca028..6ea8586eda9 100644 --- a/chasm/lib/workflow/nexus_progress.go +++ b/chasm/lib/workflow/nexus_progress.go @@ -2,12 +2,14 @@ package workflow import ( "slices" + "time" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/nexusoperation" chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" "go.temporal.io/server/common" + "google.golang.org/protobuf/types/known/timestamppb" ) var _ nexusoperation.ProgressStore = (*Workflow)(nil) @@ -31,8 +33,12 @@ func (w *Workflow) nexusProgressIndex(ctx chasm.Context) *chasmworkflowpb.NexusP // setNexusProgressIndex stores the index, or drops it when it is empty, so a Workflow whose // operations report no progress carries nothing. -func (w *Workflow) setNexusProgressIndex(ctx chasm.MutableContext, index *chasmworkflowpb.NexusProgressState) { - if len(index.GetPending()) == 0 && len(index.GetScheduled()) == 0 && len(index.GetFolded()) == 0 { +func (w *Workflow) setNexusProgressIndex( + ctx chasm.MutableContext, + index *chasmworkflowpb.NexusProgressState, +) { + if len(index.GetPending()) == 0 && len(index.GetScheduled()) == 0 && + len(index.GetFolded()) == 0 && index.GetLastCarriedTime() == nil { if _, ok := w.NexusProgress.TryGet(ctx); ok { w.NexusProgress = chasm.NewEmptyField[*chasmworkflowpb.NexusProgressState]() } @@ -99,10 +105,54 @@ func (w *Workflow) TakeNexusProgress(ctx chasm.MutableContext) []*nexuspb.NexusO } slices.Sort(index.Scheduled) index.Pending = nil + index.ReleaseTime = nil + if len(taken) > 0 { + index.LastCarriedTime = timestamppb.New(ctx.Now(w)) + } w.setNexusProgressIndex(ctx, index) return taken } +// NexusProgressReadyAt answers when pending progress may schedule a Workflow Task of its own: +// minInterval after the last task that carried progress. +func (w *Workflow) NexusProgressReadyAt(ctx chasm.Context, minInterval time.Duration) time.Time { + last := w.nexusProgressIndex(ctx).GetLastCarriedTime() + if last == nil { + return time.Time{} + } + return last.AsTime().Add(minInterval) +} + +// HoldNexusProgress keeps pending progress waiting until readyAt, when a NexusProgressReleaseTask +// lets it schedule a Workflow Task. A task scheduled for any other reason still takes it sooner. +func (w *Workflow) HoldNexusProgress(ctx chasm.MutableContext, readyAt time.Time) { + index := w.nexusProgressIndex(ctx) + // A release armed for the future and no later than readyAt already covers this. One that + // fires early releases nothing, since the progress is held again until readyAt. + if release := index.GetReleaseTime(); release != nil && release.AsTime().After(ctx.Now(w)) && + !release.AsTime().After(readyAt) { + return + } + index = common.CloneProto(index) + index.ReleaseTime = timestamppb.New(readyAt) + w.setNexusProgressIndex(ctx, index) + ctx.AddTask(w, chasm.TaskAttributes{ScheduledTime: readyAt}, &chasmworkflowpb.NexusProgressReleaseTask{}) +} + +// releaseNexusProgress lets held progress schedule a Workflow Task. It changes the Workflow, so +// the transaction that closes after it schedules the task. +func (w *Workflow) releaseNexusProgress(ctx chasm.MutableContext) { + index := common.CloneProto(w.nexusProgressIndex(ctx)) + index.ReleaseTime = nil + w.setNexusProgressIndex(ctx, index) +} + +// nexusProgressHeld reports whether a release is armed for pending progress. +func (w *Workflow) nexusProgressHeld(ctx chasm.Context) bool { + index := w.nexusProgressIndex(ctx) + return index.GetReleaseTime() != nil && len(index.GetPending()) > 0 +} + // ClearScheduledNexusProgress forgets which operations the scheduled Workflow Task carries. Called // once that task starts, fails or times out: from then on newer progress needs a task of its own, // and progress that folded into it waits for the next task, since its scheduled event carries an @@ -124,24 +174,6 @@ func (w *Workflow) ClearScheduledNexusProgress(ctx chasm.MutableContext) { w.setNexusProgressIndex(ctx, index) } -// DropPendingNexusProgress forgets every operation's pending progress. It guards against a Workflow -// Task that was scheduled for progress but could not carry it, which would otherwise schedule -// another task on every transaction. -func (w *Workflow) DropPendingNexusProgress(ctx chasm.MutableContext) { - index := w.nexusProgressIndex(ctx) - if len(index.GetPending()) == 0 { - return - } - for _, key := range index.GetPending() { - if field, ok := w.Operations[key]; ok { - field.Get(ctx).DropPendingProgress(ctx) - } - } - index = common.CloneProto(index) - index.Pending = nil - w.setNexusProgressIndex(ctx, index) -} - // forgetNexusProgress removes a closed operation from the index. func (w *Workflow) forgetNexusProgress(ctx chasm.MutableContext, key int64) { index := w.nexusProgressIndex(ctx) diff --git a/chasm/lib/workflow/nexus_progress_tasks.go b/chasm/lib/workflow/nexus_progress_tasks.go new file mode 100644 index 00000000000..8f46e34e533 --- /dev/null +++ b/chasm/lib/workflow/nexus_progress_tasks.go @@ -0,0 +1,29 @@ +package workflow + +import ( + "go.temporal.io/server/chasm" + chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" +) + +type nexusProgressReleaseTaskHandler struct { + chasm.PureTaskHandlerBase +} + +func (nexusProgressReleaseTaskHandler) Validate( + ctx chasm.Context, + wf *Workflow, + _ chasm.TaskInvocation, + _ *chasmworkflowpb.NexusProgressReleaseTask, +) (bool, error) { + return wf.nexusProgressHeld(ctx), nil +} + +func (nexusProgressReleaseTaskHandler) Execute( + ctx chasm.MutableContext, + wf *Workflow, + _ chasm.TaskAttributes, + _ *chasmworkflowpb.NexusProgressReleaseTask, +) error { + wf.releaseNexusProgress(ctx) + return nil +} diff --git a/chasm/lib/workflow/nexus_progress_test.go b/chasm/lib/workflow/nexus_progress_test.go index dd386039f18..5774b0dc55f 100644 --- a/chasm/lib/workflow/nexus_progress_test.go +++ b/chasm/lib/workflow/nexus_progress_test.go @@ -2,6 +2,7 @@ package workflow import ( "testing" + "time" "github.com/stretchr/testify/require" nexuspb "go.temporal.io/api/nexus/v1" @@ -95,16 +96,53 @@ func TestNexusProgressIndex(t *testing.T) { require.Equal(t, map[int64]int64{9: 2}, counters(wf.TakeNexusProgress(ctx)), "a closed operation's folded progress is gone") wf.removeNexusOperation(ctx, 9) require.False(t, wf.HasScheduledNexusProgress(ctx)) - _, ok := wf.NexusProgress.TryGet(ctx) - require.False(t, ok, "an empty index is not stored") + index, ok := wf.NexusProgress.TryGet(ctx) + require.True(t, ok) + require.Empty(t, index.GetPending()) + require.Empty(t, index.GetScheduled()) + require.Empty(t, index.GetFolded()) + require.NotNil(t, index.GetLastCarriedTime(), "only the last carried time stays, for the rate limit") }) - t.Run("DropsPendingProgressAsALivelockGuard", func(t *testing.T) { + t.Run("StoresNoIndexWithoutProgress", func(t *testing.T) { ctx := &chasm.MockMutableContext{} wf := newWorkflowWithOperations(t, ctx, 5) - deliver(t, ctx, wf, 5, 1) - wf.DropPendingNexusProgress(ctx) - require.False(t, wf.HasPendingNexusProgress(ctx)) + wf.ClearScheduledNexusProgress(ctx) require.Empty(t, wf.TakeNexusProgress(ctx)) + _, ok := wf.NexusProgress.TryGet(ctx) + require.False(t, ok) + }) + + t.Run("HoldsProgressUntilTheIntervalSinceTheLastCarry", func(t *testing.T) { + now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) + ctx := &chasm.MockMutableContext{MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return now }, + }} + wf := newWorkflowWithOperations(t, ctx, 5) + require.True(t, wf.NexusProgressReadyAt(ctx, time.Second).IsZero(), "nothing carried yet") + deliver(t, ctx, wf, 5, 1) + require.Len(t, wf.TakeNexusProgress(ctx), 1) + wf.ClearScheduledNexusProgress(ctx) + readyAt := wf.NexusProgressReadyAt(ctx, time.Second) + require.Equal(t, now.Add(time.Second), readyAt) + + deliver(t, ctx, wf, 5, 2) + wf.HoldNexusProgress(ctx, readyAt) + wf.HoldNexusProgress(ctx, readyAt) + require.Len(t, ctx.Tasks, 1, "one armed release covers the same deadline") + require.Equal(t, readyAt, ctx.Tasks[0].Attributes.ScheduledTime) + require.True(t, wf.nexusProgressHeld(ctx)) + require.True(t, wf.HasPendingNexusProgress(ctx), "held progress still rides any other task") + + wf.releaseNexusProgress(ctx) + require.False(t, wf.nexusProgressHeld(ctx)) + require.True(t, wf.HasPendingNexusProgress(ctx)) + + wf.HoldNexusProgress(ctx, readyAt) + require.Len(t, ctx.Tasks, 2, "a released hold arms again") + now = now.Add(time.Second) + require.Len(t, wf.TakeNexusProgress(ctx), 1) + require.False(t, wf.nexusProgressHeld(ctx), "a task that takes the progress ends the hold") + require.Equal(t, now.Add(time.Second), wf.NexusProgressReadyAt(ctx, time.Second)) }) } diff --git a/chasm/lib/workflow/proto/v1/state.proto b/chasm/lib/workflow/proto/v1/state.proto index 9bad1825ccd..157ab9a3d01 100644 --- a/chasm/lib/workflow/proto/v1/state.proto +++ b/chasm/lib/workflow/proto/v1/state.proto @@ -4,6 +4,8 @@ package temporal.server.chasm.lib.workflow.proto.v1; option go_package = "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpb"; +import "google/protobuf/timestamp.proto"; + // NexusOperationParentData contains workflow-specific data stored in a nexus operation's // parent_data field when the operation is embedded in a workflow. message NexusOperationParentData { @@ -38,4 +40,10 @@ message NexusProgressState { // carries. Once the task starts, fails or times out they wait like pending ones, so one follow-up // task carries the highest counter. repeated int64 folded = 3; + // When a Workflow Task last carried progress. Progress waits at least + // nexusoperation.progressMinInterval after it before it schedules a task of its own, so a long + // stream can't fill the caller's History. + google.protobuf.Timestamp last_carried_time = 4; + // When the armed NexusProgressReleaseTask lets waiting progress schedule a task. + google.protobuf.Timestamp release_time = 5; } diff --git a/chasm/lib/workflow/proto/v1/tasks.proto b/chasm/lib/workflow/proto/v1/tasks.proto new file mode 100644 index 00000000000..4930e892fdf --- /dev/null +++ b/chasm/lib/workflow/proto/v1/tasks.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.workflow.proto.v1; + +option go_package = "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb;workflowpb"; + +// Lets Nexus operation progress that waited out nexusoperation.progressMinInterval schedule a +// Workflow Task. +message NexusProgressReleaseTask {} diff --git a/service/history/configs/config.go b/service/history/configs/config.go index f3d0e7921b3..15bd3f832fc 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -82,6 +82,7 @@ type Config struct { EnableCHASMSkipPersistence dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableChasmNexusWorkflowOperations dynamicconfig.BoolPropertyFnWithNamespaceFilter ChasmNexusWorkflowOperationsRolloutPercent dynamicconfig.IntPropertyFnWithNamespaceFilter + NexusOperationProgressMinInterval dynamicconfig.DurationPropertyFnWithNamespaceFilter EnableCHASMCallbacks dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableCHASMSignalBacklinks dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableCHASMWorkflowRootOnStart dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -547,6 +548,7 @@ func NewConfig( EnableCHASMSkipPersistence: dynamicconfig.EnableCHASMSkipPersistence.Get(dc), EnableChasmNexusWorkflowOperations: nexusoperation.EnableChasmWorkflowOperations.Get(dc), ChasmNexusWorkflowOperationsRolloutPercent: nexusoperation.ChasmWorkflowOperationsRolloutPercent.Get(dc), + NexusOperationProgressMinInterval: nexusoperation.ProgressMinInterval.Get(dc), ChasmMaxInMemoryPureTasks: dynamicconfig.ChasmMaxInMemoryPureTasks.Get(dc), ChasmLogicalTaskCountAlertThreshold: dynamicconfig.ChasmLogicalTaskCountAlertThreshold.Get(dc), ChasmDLQScheduledPureTaskOnValidation: dynamicconfig.ChasmDLQScheduledPureTaskOnValidation.Get(dc), diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index e8231074b64..6a577f56b6c 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -37,6 +37,7 @@ import ( chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" @@ -73,6 +74,7 @@ import ( "go.temporal.io/server/service/history/tests" "go.uber.org/mock/gomock" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -2319,6 +2321,93 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskSchedule_CurrentVersionChan s.Equal(0, s.mutableState.hBuilder.NumBufferedEvents()) } +// TestNexusProgressAcrossFailoverConversion checks that a transient Workflow Task a failover +// converts doesn't carry Nexus operation progress, and that the progress waits for a later task +// instead of being dropped. +func (s *mutableStateSuite) TestNexusProgressAcrossFailoverConversion() { + version := int64(2000) + // prepare leaves a failed first task, so the next task is a transient retry. + prepare := func() { + workflowID := "some random workflow ID" + runID := uuid.NewString() + s.mutableState = TestGlobalMutableState( + s.mockShard, s.mockEventsCache, s.logger, version, workflowID, runID) + _, _ = s.prepareTransientWorkflowTaskCompletionFirstBatchApplied(version, workflowID, runID) + s.NoError(s.mutableState.ApplyWorkflowTaskFailedEvent()) + versionHistory, err := versionhistory.GetCurrentVersionHistory( + s.mutableState.GetExecutionInfo().GetVersionHistories()) + s.NoError(err) + s.NoError(versionhistory.AddOrUpdateVersionHistoryItem(versionHistory, &historyspb.VersionHistoryItem{ + EventId: s.mutableState.GetNextEventID() - 1, + Version: version, + })) + + registry := chasm.NewRegistry(s.logger) + s.NoError(registry.Register(&chasm.CoreLibrary{})) + s.NoError(registry.Register(chasmworkflow.NewLibrary(chasmworkflow.NewRegistry()))) + s.NoError(registry.Register(chasmnexus.NewNilLibrary())) + tree := chasm.NewEmptyTree( + registry, s.mutableState, chasm.DefaultPathEncoder, s.logger, metrics.NoopMetricsHandler) + s.mutableState.chasmTree = tree + ctx := chasm.NewMutableContext(context.Background(), tree) + wf := chasmworkflow.NewWorkflow(ctx, chasm.NewMSPointer(s.mutableState)) + parentData, err := anypb.New(&chasmworkflowpb.NexusOperationParentData{ScheduledEventId: 5}) + s.NoError(err) + op := chasmnexus.NewOperation(&nexusoperationpb.OperationState{ + Status: nexusoperationpb.OPERATION_STATUS_STARTED, + ParentData: parentData, + }) + wf.Operations = chasm.Map[int64, *chasmnexus.Operation]{5: chasm.NewComponentField(ctx, op)} + s.NoError(tree.SetRootComponent(wf)) + op.PendingProgress = chasm.NewDataField(ctx, &nexuspb.NexusOperationProgress{Counter: 1}) + s.NoError(wf.OnNexusOperationProgress(ctx, op)) + s.True(s.mutableState.hasPendingNexusProgress()) + } + scheduledProgress := func() []*nexuspb.NexusOperationProgress { + mutation, err := s.mutableState.hBuilder.Finish(true) + s.NoError(err) + var carried []*nexuspb.NexusOperationProgress + for _, batch := range mutation.DBEventsBatches { + for _, event := range batch { + attrs := event.GetWorkflowTaskScheduledEventAttributes() + carried = append(carried, attrs.GetNexusOperationProgress()...) + } + } + return carried + } + + s.Run("ScheduledByProgressAfterTheVersionChanged", func() { + prepare() + s.NoError(s.mutableState.UpdateCurrentVersion(version+1, true)) + s.NoError(s.mutableState.scheduleWorkflowTaskForNexusProgress()) + s.True(s.mutableState.HasPendingWorkflowTask()) + s.Empty(scheduledProgress(), "the converted task doesn't carry the progress") + s.True(s.mutableState.hasPendingNexusProgress(), "the progress waits for a later task") + }) + + s.Run("StartedAfterTheVersionChanged", func() { + prepare() + wt, err := s.mutableState.AddWorkflowTaskScheduledEventAsHeartbeat( + true, timestamp.TimeNowPtrUtc(), enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + s.NoError(s.mutableState.UpdateCurrentVersion(version+1, true)) + f, err := tqid.NewTaskQueueFamily("", "tq") + s.NoError(err) + _, started, err := s.mutableState.AddWorkflowTaskStartedEvent( + wt.ScheduledEventID, + uuid.NewString(), + &taskqueuepb.TaskQueue{Name: f.TaskQueue(enumspb.TASK_QUEUE_TYPE_WORKFLOW).NormalPartition(5).RpcName()}, + "random identity", nil, nil, nil, false, nil, 0, + ) + s.NoError(err) + s.NotNil(started) + s.Equal(int32(1), started.Attempt, "the transient task was converted") + s.Empty(scheduledProgress(), "the converted task doesn't carry the progress") + s.True(s.mutableState.hasPendingNexusProgress()) + }) + +} + func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged() { version := int64(2000) workflowID := "some random workflow ID" diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go index dccf502a8f0..e18a770aaa5 100644 --- a/service/history/workflow/nexus_progress.go +++ b/service/history/workflow/nexus_progress.go @@ -2,6 +2,7 @@ package workflow import ( "context" + "time" historypb "go.temporal.io/api/history/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -66,24 +67,48 @@ func (ms *MutableStateImpl) clearScheduledNexusProgress() { wf.ClearScheduledNexusProgress(mutableCtx) } +// nexusProgressMinInterval is the shortest time between a Workflow Task that carried progress and +// the next task progress schedules. +func (ms *MutableStateImpl) nexusProgressMinInterval() time.Duration { + if ms.config.NexusOperationProgressMinInterval == nil { + return 0 + } + return ms.config.NexusOperationProgressMinInterval(ms.GetNamespaceEntry().Name().String()) +} + // scheduleWorkflowTaskForNexusProgress schedules a Workflow Task to carry pending Nexus operation -// progress when no task is pending to carry it. +// progress when no task is pending to carry it, unless a task carried progress too recently. func (ms *MutableStateImpl) scheduleWorkflowTaskForNexusProgress() error { - if ms.HasPendingWorkflowTask() || ms.IsWorkflowExecutionStatusPaused() || !ms.hasPendingNexusProgress() { + if ms.HasPendingWorkflowTask() || ms.IsWorkflowExecutionStatusPaused() { return nil } + wf, chasmCtx, ok := ms.chasmWorkflowView() + if !ok || !wf.HasPendingNexusProgress(chasmCtx) { + return nil + } + minInterval := ms.nexusProgressMinInterval() + now := chasmCtx.Now(wf) + if readyAt := wf.NexusProgressReadyAt(chasmCtx, minInterval); now.Before(readyAt) { + return ms.waitNexusProgress(readyAt) + } if _, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL); err != nil { return err } - // A task that was scheduled for progress but did not carry it would leave the progress - // pending, and every later transaction would schedule another task for it. - if ms.hasPendingNexusProgress() { - ms.logger.Warn("dropped Nexus operation progress that a scheduled Workflow Task did not carry") - wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) - if err != nil { - return err - } - wf.DropPendingNexusProgress(chasmCtx) + if !ms.hasPendingNexusProgress() { + return nil + } + // The task can't carry the progress, for example after a failover converted it. Without a + // hold, every later transaction would schedule another task for it. The floor stops that loop + // when the interval is zero. + ms.logger.Info("held Nexus operation progress that a scheduled Workflow Task can't carry") + return ms.waitNexusProgress(now.Add(max(minInterval, time.Second))) +} + +func (ms *MutableStateImpl) waitNexusProgress(readyAt time.Time) error { + wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + return err } + wf.HoldNexusProgress(chasmCtx, readyAt) return nil } diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index ac20224dc66..b2905fa64cf 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -731,3 +731,139 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks func (s *NexusWorkflowTestSuite) scheduledEventID(hist []*historypb.HistoryEvent) int64 { return s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() } + +// rawProgressCaller drives a caller Workflow with raw Workflow Task polls, so a test controls +// when each task starts and how it ends. +type rawProgressCaller struct { + s *NexusWorkflowTestSuite + env *NexusTestEnv + ctx context.Context + taskQueue string + run client.WorkflowRun + callbackToken string + callbackURL string +} + +func (s *NexusWorkflowTestSuite) newRawProgressCaller( + opts ...testcore.TestOption, +) *rawProgressCaller { + opts = append(opts, testcore.WithDynamicConfig(chasmnexus.EnableProgress, true)) + c := &rawProgressCaller{ + s: s, + env: s.newTestEnv(true, opts...), + ctx: s.Context(), + taskQueue: testcore.RandomizeStr(s.T().Name()), + } + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + c.callbackToken = options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + c.callbackURL = options.CallbackURL + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil + }, + } + endpointName := c.env.createRandomExternalNexusServer(c.ctx, s.T(), h) + run, err := c.env.SdkClient().ExecuteWorkflow( + c.ctx, client.StartWorkflowOptions{TaskQueue: c.taskQueue}, "workflow") + s.NoError(err) + c.run = run + c.respond(c.poll(), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpointName, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(s.T(), "input"), + }, + }, + }) + started := c.poll() + s.RequireHistoryEvent(started.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + c.respond(started) + return c +} + +func (c *rawProgressCaller) poll() *workflowservice.PollWorkflowTaskQueueResponse { + resp, err := c.env.FrontendClient().PollWorkflowTaskQueue(c.ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: c.env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + Identity: "test", + }) + c.s.NoError(err) + c.s.NotEmpty(resp.GetTaskToken()) + return resp +} + +func (c *rawProgressCaller) respond( + task *workflowservice.PollWorkflowTaskQueueResponse, + commands ...*commandpb.Command, +) { + _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: c.env.Namespace().String(), + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + }) + c.s.NoError(err) +} + +func (c *rawProgressCaller) complete(task *workflowservice.PollWorkflowTaskQueueResponse) { + c.respond(task, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }) + c.s.NoError(c.run.Get(c.ctx, nil)) +} + +func (c *rawProgressCaller) postProgress(counter int) { + body := fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter) + status, err := postNexusProgress(c.ctx, c.callbackURL, c.callbackToken, body) + c.s.NoError(err) + c.s.Equal(http.StatusOK, status) +} + +// lastScheduled is the scheduled event of the task a poll answered. +func lastScheduled(task *workflowservice.PollWorkflowTaskQueueResponse) *historypb.HistoryEvent { + events := task.GetHistory().GetEvents() + for i := len(events) - 1; i >= 0; i-- { + if events[i].GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + return events[i] + } + } + return nil +} + +// TestNexusOperationProgressMinInterval checks that progress schedules a task of its own no sooner +// than nexusoperation.progressMinInterval after the last task that carried progress, so a long +// stream can't fill the caller's History. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressMinInterval(chasmEnabled bool) { + if !chasmEnabled { + // An HSM caller refuses progress; TestNexusOperationProgress covers it. + return + } + const interval = 2 * time.Second + c := s.newRawProgressCaller(testcore.WithDynamicConfig(chasmnexus.ProgressMinInterval, interval)) + + c.postProgress(1) + first := c.poll() + s.Equal(int64(1), lastScheduled(first).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress()[0].GetCounter()) + c.respond(first) + + c.postProgress(2) + c.postProgress(3) + second := c.poll() + carried := lastScheduled(second).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + s.Len(carried, 1) + s.Equal(int64(3), carried[0].GetCounter(), "progress that waited folds into one task") + gap := lastScheduled(second).GetEventTime().AsTime().Sub(lastScheduled(first).GetEventTime().AsTime()) + s.GreaterOrEqual(gap, interval, "progress waits out the interval since the last task that carried it") + c.complete(second) +} From b41378520d07cb43009ab0234fd871a50d71a6af Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 10 Oct 2026 13:02:00 -0700 Subject: [PATCH 7/9] Skipped the Nexus progress check when nothing it depends on changed. It ran on every transaction close and resolved the CHASM root each time. --- service/history/workflow/nexus_progress.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go index e18a770aaa5..a23aaaf9b80 100644 --- a/service/history/workflow/nexus_progress.go +++ b/service/history/workflow/nexus_progress.go @@ -79,6 +79,11 @@ func (ms *MutableStateImpl) nexusProgressMinInterval() time.Duration { // scheduleWorkflowTaskForNexusProgress schedules a Workflow Task to carry pending Nexus operation // progress when no task is pending to carry it, unless a task carried progress too recently. func (ms *MutableStateImpl) scheduleWorkflowTaskForNexusProgress() error { + // Only a CHASM change (progress arrived, or a hold was released) or a Workflow Task change can + // make progress need a task, so most transactions skip resolving the CHASM root. + if !ms.workflowTaskUpdated && !ms.chasmTree.IsStateDirty() { + return nil + } if ms.HasPendingWorkflowTask() || ms.IsWorkflowExecutionStatusPaused() { return nil } From 217a46628c122c4d9c626c7286f4100a5787ff51 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 10 Oct 2026 13:02:00 -0700 Subject: [PATCH 8/9] Checked where progress is after the task that carried it fails. --- tests/nexus_progress_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index b2905fa64cf..174f2010fe3 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -867,3 +867,35 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressMinInterval(chasmEnab s.GreaterOrEqual(gap, interval, "progress waits out the interval since the last task that carried it") c.complete(second) } + +// TestNexusOperationProgressOnFailedTask checks where progress is after the task that carried it +// fails. It stays on that task's scheduled event, which the retry's History includes, and the +// retry's own scheduled event doesn't repeat it. So a worker reads progress from every scheduled +// event since its last completed task. +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressOnFailedTask(chasmEnabled bool) { + if !chasmEnabled { + // An HSM caller refuses progress; TestNexusOperationProgress covers it. + return + } + c := s.newRawProgressCaller() + + c.postProgress(1) + failing := c.poll() + s.Len(lastScheduled(failing).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), 1) + _, err := c.env.FrontendClient().RespondWorkflowTaskFailed(c.ctx, &workflowservice.RespondWorkflowTaskFailedRequest{ + Namespace: c.env.Namespace().String(), + Identity: "test", + TaskToken: failing.GetTaskToken(), + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, + }) + s.NoError(err) + + retry := c.poll() + s.Equal(int32(2), retry.GetAttempt()) + s.Empty(lastScheduled(retry).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), + "the retry doesn't repeat what the failed task carried") + carried := carriedNexusProgress(retry.GetHistory().GetEvents()) + s.Len(carried, 1, "the failed task's scheduled event still carries the progress") + s.Equal(int64(1), carried[0].GetCounter()) + c.complete(retry) +} From 3208cdcc5df1b2171aaa3be5f94569a3bbaa5e06 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 10 Oct 2026 13:06:36 -0700 Subject: [PATCH 9/9] Wrapped the progress folding code at 100 columns. --- chasm/lib/nexusoperation/operation.go | 14 +- chasm/lib/nexusoperation/progress_test.go | 35 +- chasm/lib/workflow/nexus_progress.go | 11 +- chasm/lib/workflow/nexus_progress_test.go | 34 +- common/nexus/nexusrpc/completion.go | 4 +- common/nexus/nexusrpc/completion_test.go | 15 +- .../api/respondworkflowtaskcompleted/api.go | 6 +- service/history/interfaces/mutable_state.go | 4 +- .../history/workflow/mutable_state_impl.go | 4 +- .../workflow/mutable_state_impl_test.go | 32 +- service/history/workflow/nexus_progress.go | 7 +- .../workflow/workflow_task_state_machine.go | 7 +- tests/nexus_progress_test.go | 435 +++++++++++------- tests/xdc/nexus_progress_forwarding_test.go | 139 ++++-- 14 files changed, 502 insertions(+), 245 deletions(-) diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index cdb8423090c..252fe4c12c7 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -345,7 +345,10 @@ func (o *Operation) HandleNexusCompletion( // onProgress keeps the progress with the highest counter for the caller. Progress before the // operation started is dropped, since it cannot stand in for the start response, and so is progress // after it closed, since its completion supersedes it. -func (o *Operation) onProgress(ctx chasm.MutableContext, progress *nexuspb.NexusOperationProgress) error { +func (o *Operation) onProgress( + ctx chasm.MutableContext, + progress *nexuspb.NexusOperationProgress, +) error { if o.Status != nexusoperationpb.OPERATION_STATUS_STARTED { return nil } @@ -357,7 +360,10 @@ func (o *Operation) onProgress(ctx chasm.MutableContext, progress *nexuspb.Nexus if !ok { return nil } - for _, seen := range []chasm.Field[*nexuspb.NexusOperationProgress]{o.PendingProgress, o.DeliveredProgress} { + for _, seen := range []chasm.Field[*nexuspb.NexusOperationProgress]{ + o.PendingProgress, + o.DeliveredProgress, + } { if previous, ok := seen.TryGet(ctx); ok && previous.GetCounter() >= progress.GetCounter() { return nil } @@ -367,7 +373,9 @@ func (o *Operation) onProgress(ctx chasm.MutableContext, progress *nexuspb.Nexus } // TakePendingProgress returns the pending progress and records it as delivered. -func (o *Operation) TakePendingProgress(ctx chasm.MutableContext) (*nexuspb.NexusOperationProgress, bool) { +func (o *Operation) TakePendingProgress( + ctx chasm.MutableContext, +) (*nexuspb.NexusOperationProgress, bool) { progress, ok := o.PendingProgress.TryGet(ctx) if !ok { return nil, false diff --git a/chasm/lib/nexusoperation/progress_test.go b/chasm/lib/nexusoperation/progress_test.go index 5d3c3f0a831..c80346fe765 100644 --- a/chasm/lib/nexusoperation/progress_test.go +++ b/chasm/lib/nexusoperation/progress_test.go @@ -54,11 +54,36 @@ func TestHandleNexusProgress(t *testing.T) { counters []int64 wantPending int64 }{ - {name: "HighestCounterWins", status: nexusoperationpb.OPERATION_STATUS_STARTED, store: &progressStoreComponent{}, counters: []int64{1, 3, 2}, wantPending: 3}, - {name: "DroppedBeforeStart", status: nexusoperationpb.OPERATION_STATUS_SCHEDULED, store: &progressStoreComponent{}, counters: []int64{1}}, - {name: "DroppedAfterClose", status: nexusoperationpb.OPERATION_STATUS_SUCCEEDED, store: &progressStoreComponent{}, counters: []int64{1}}, - {name: "DroppedWithoutCaller", status: nexusoperationpb.OPERATION_STATUS_STARTED, counters: []int64{1}}, - {name: "DroppedByAParentThatTakesNoProgress", status: nexusoperationpb.OPERATION_STATUS_STARTED, store: &mockStoreComponent{}, counters: []int64{1}}, + { + name: "HighestCounterWins", + status: nexusoperationpb.OPERATION_STATUS_STARTED, + store: &progressStoreComponent{}, + counters: []int64{1, 3, 2}, + wantPending: 3, + }, + { + name: "DroppedBeforeStart", + status: nexusoperationpb.OPERATION_STATUS_SCHEDULED, + store: &progressStoreComponent{}, + counters: []int64{1}, + }, + { + name: "DroppedAfterClose", + status: nexusoperationpb.OPERATION_STATUS_SUCCEEDED, + store: &progressStoreComponent{}, + counters: []int64{1}, + }, + { + name: "DroppedWithoutCaller", + status: nexusoperationpb.OPERATION_STATUS_STARTED, + counters: []int64{1}, + }, + { + name: "DroppedByAParentThatTakesNoProgress", + status: nexusoperationpb.OPERATION_STATUS_STARTED, + store: &mockStoreComponent{}, + counters: []int64{1}, + }, } { t.Run(tc.name, func(t *testing.T) { ctx := &chasm.MockMutableContext{} diff --git a/chasm/lib/workflow/nexus_progress.go b/chasm/lib/workflow/nexus_progress.go index 6ea8586eda9..f093abfba29 100644 --- a/chasm/lib/workflow/nexus_progress.go +++ b/chasm/lib/workflow/nexus_progress.go @@ -50,7 +50,10 @@ func (w *Workflow) setNexusProgressIndex( // OnNexusOperationProgress records that an operation holds progress for the next Workflow Task // scheduled event. Progress for an operation that rides a scheduled Workflow Task that has not // started folds into that task: it asks for no task of its own until that one starts. -func (w *Workflow) OnNexusOperationProgress(ctx chasm.MutableContext, op *nexusoperation.Operation) error { +func (w *Workflow) OnNexusOperationProgress( + ctx chasm.MutableContext, + op *nexusoperation.Operation, +) error { key, ok := nexusProgressKey(op) if !ok { op.DropPendingProgress(ctx) @@ -136,7 +139,11 @@ func (w *Workflow) HoldNexusProgress(ctx chasm.MutableContext, readyAt time.Time index = common.CloneProto(index) index.ReleaseTime = timestamppb.New(readyAt) w.setNexusProgressIndex(ctx, index) - ctx.AddTask(w, chasm.TaskAttributes{ScheduledTime: readyAt}, &chasmworkflowpb.NexusProgressReleaseTask{}) + ctx.AddTask( + w, + chasm.TaskAttributes{ScheduledTime: readyAt}, + &chasmworkflowpb.NexusProgressReleaseTask{}, + ) } // releaseNexusProgress lets held progress schedule a Workflow Task. It changes the Workflow, so diff --git a/chasm/lib/workflow/nexus_progress_test.go b/chasm/lib/workflow/nexus_progress_test.go index 5774b0dc55f..536a3ddb4fd 100644 --- a/chasm/lib/workflow/nexus_progress_test.go +++ b/chasm/lib/workflow/nexus_progress_test.go @@ -15,10 +15,16 @@ import ( ) func TestNexusProgressIndex(t *testing.T) { - newWorkflowWithOperations := func(t *testing.T, ctx chasm.MutableContext, keys ...int64) *Workflow { + newWorkflowWithOperations := func( + t *testing.T, + ctx chasm.MutableContext, + keys ...int64, + ) *Workflow { wf := &Workflow{} for _, key := range keys { - parentData, err := anypb.New(&chasmworkflowpb.NexusOperationParentData{ScheduledEventId: key}) + parentData, err := anypb.New( + &chasmworkflowpb.NexusOperationParentData{ScheduledEventId: key}, + ) require.NoError(t, err) op := nexusoperation.NewOperation(&nexusoperationpb.OperationState{ Status: nexusoperationpb.OPERATION_STATUS_STARTED, @@ -30,11 +36,12 @@ func TestNexusProgressIndex(t *testing.T) { return wf } deliver := func(t *testing.T, ctx chasm.MutableContext, wf *Workflow, key, counter int64) { - err := wf.Operations[key].Get(ctx).HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ - Outcome: &persistencespb.ChasmNexusCompletion_Progress{ - Progress: &nexuspb.NexusOperationProgress{Counter: counter}, - }, - }) + err := wf.Operations[key].Get(ctx). + HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + Outcome: &persistencespb.ChasmNexusCompletion_Progress{ + Progress: &nexuspb.NexusOperationProgress{Counter: counter}, + }, + }) require.NoError(t, err) } counters := func(taken []*nexuspb.NexusOperationProgress) map[int64]int64 { @@ -93,7 +100,12 @@ func TestNexusProgressIndex(t *testing.T) { deliver(t, ctx, wf, 9, 2) wf.removeNexusOperation(ctx, 5) wf.ClearScheduledNexusProgress(ctx) - require.Equal(t, map[int64]int64{9: 2}, counters(wf.TakeNexusProgress(ctx)), "a closed operation's folded progress is gone") + require.Equal( + t, + map[int64]int64{9: 2}, + counters(wf.TakeNexusProgress(ctx)), + "a closed operation's folded progress is gone", + ) wf.removeNexusOperation(ctx, 9) require.False(t, wf.HasScheduledNexusProgress(ctx)) index, ok := wf.NexusProgress.TryGet(ctx) @@ -101,7 +113,11 @@ func TestNexusProgressIndex(t *testing.T) { require.Empty(t, index.GetPending()) require.Empty(t, index.GetScheduled()) require.Empty(t, index.GetFolded()) - require.NotNil(t, index.GetLastCarriedTime(), "only the last carried time stays, for the rate limit") + require.NotNil( + t, + index.GetLastCarriedTime(), + "only the last carried time stays, for the rate limit", + ) }) t.Run("StoresNoIndexWithoutProgress", func(t *testing.T) { diff --git a/common/nexus/nexusrpc/completion.go b/common/nexus/nexusrpc/completion.go index cc215999b49..9c5e8b2792b 100644 --- a/common/nexus/nexusrpc/completion.go +++ b/common/nexus/nexusrpc/completion.go @@ -106,8 +106,8 @@ type CompleteOperationOptions struct { // Result to deliver with the completion. Uses the client's serializer to serialize the result into the request body. // Only used for successful completions and progress. Result any - // Progress makes the request a progress delivery rather than a completion: the state is running and Result holds - // the progress body. The operation stays open. + // Progress makes the request a progress delivery rather than a completion: the state is running + // and Result holds the progress body. The operation stays open. Progress bool } diff --git a/common/nexus/nexusrpc/completion_test.go b/common/nexus/nexusrpc/completion_test.go index 9aead3292ab..5e918f80159 100644 --- a/common/nexus/nexusrpc/completion_test.go +++ b/common/nexus/nexusrpc/completion_test.go @@ -265,7 +265,10 @@ type progressHandler struct { body chan []byte } -func (h *progressHandler) CompleteOperation(ctx context.Context, completion *nexusrpc.CompletionRequest) error { +func (h *progressHandler) CompleteOperation( + ctx context.Context, + completion *nexusrpc.CompletionRequest, +) error { body, err := io.ReadAll(completion.Result.Reader) if err != nil { return err @@ -276,13 +279,19 @@ func (h *progressHandler) CompleteOperation(ctx context.Context, completion *nex } func TestProgressDelivery(t *testing.T) { - h := &progressHandler{got: make(chan *nexusrpc.CompletionRequest, 1), body: make(chan []byte, 1)} + h := &progressHandler{ + got: make(chan *nexusrpc.CompletionRequest, 1), + body: make(chan []byte, 1), + } ctx, callbackURL, teardown := setupForCompletion(t, h, nil, nil) defer teardown() c := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{}) err := c.CompleteOperation(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ - Result: &nexus.Content{Header: nexus.Header{"type": "application/json"}, Data: []byte(`{"counter": 2}`)}, + Result: &nexus.Content{ + Header: nexus.Header{"type": "application/json"}, + Data: []byte(`{"counter": 2}`), + }, Progress: true, OperationToken: "test-operation-token", }) diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index f1b79ca46ed..44b8a2313d4 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -627,9 +627,9 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( } if request.GetForceCreateNewWorkflowTask() { - // A worker forces a new task to heartbeat a long local activity. SDKs replay such a task - // as part of a heartbeat chain, so progress on it could reach the Workflow in a different - // activation on replay than live. It waits for the next task instead. + // A worker forces a new task to heartbeat a long local activity. SDKs replay such a + // task as part of a heartbeat chain, so progress on it could reach the Workflow in a + // different activation on replay than live. It waits for the next task instead. ms.HoldNexusProgress() } diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 8dd7b195321..9ee29a387ce 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -80,8 +80,8 @@ type ( AddFirstWorkflowTaskScheduled(parentClock *clockspb.VectorClock, event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error) AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *timestamppb.Timestamp, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error) - // HoldNexusProgress keeps pending Nexus operation progress off the Workflow Task scheduled in - // this transaction and off its retries, so it waits for a later task. + // HoldNexusProgress keeps pending Nexus operation progress off the Workflow Task scheduled + // in this transaction and off its retries, so it waits for a later task. HoldNexusProgress() AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion, int64) (*historypb.HistoryEvent, *WorkflowTaskInfo, error) AddWorkflowTaskTimedOutEvent(workflowTask *WorkflowTaskInfo) (*historypb.HistoryEvent, error) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 1e1041c8fbe..ccfca206ba5 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -210,8 +210,8 @@ type ( // (via AttachChasmRequestID). It gates the lazy request-ID sweep at transaction close so only // transactions that added an ID pay for the scan. chasmRequestIDsAdded bool - // holdNexusProgress marks the Workflow Task scheduled in the current transaction as one that - // holds Nexus operation progress (see HoldNexusProgress). + // holdNexusProgress marks the Workflow Task scheduled in the current transaction as one + // that holds Nexus operation progress (see HoldNexusProgress). holdNexusProgress bool updateInfoUpdated map[string]struct{} // following xxxUserDataUpdated fields are for tracking if activity/timer user data updated. diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 6a577f56b6c..52ccaba2e95 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -2337,17 +2337,27 @@ func (s *mutableStateSuite) TestNexusProgressAcrossFailoverConversion() { versionHistory, err := versionhistory.GetCurrentVersionHistory( s.mutableState.GetExecutionInfo().GetVersionHistories()) s.NoError(err) - s.NoError(versionhistory.AddOrUpdateVersionHistoryItem(versionHistory, &historyspb.VersionHistoryItem{ - EventId: s.mutableState.GetNextEventID() - 1, - Version: version, - })) + s.NoError( + versionhistory.AddOrUpdateVersionHistoryItem( + versionHistory, + &historyspb.VersionHistoryItem{ + EventId: s.mutableState.GetNextEventID() - 1, + Version: version, + }, + ), + ) registry := chasm.NewRegistry(s.logger) s.NoError(registry.Register(&chasm.CoreLibrary{})) s.NoError(registry.Register(chasmworkflow.NewLibrary(chasmworkflow.NewRegistry()))) s.NoError(registry.Register(chasmnexus.NewNilLibrary())) tree := chasm.NewEmptyTree( - registry, s.mutableState, chasm.DefaultPathEncoder, s.logger, metrics.NoopMetricsHandler) + registry, + s.mutableState, + chasm.DefaultPathEncoder, + s.logger, + metrics.NoopMetricsHandler, + ) s.mutableState.chasmTree = tree ctx := chasm.NewMutableContext(context.Background(), tree) wf := chasmworkflow.NewWorkflow(ctx, chasm.NewMSPointer(s.mutableState)) @@ -2396,8 +2406,16 @@ func (s *mutableStateSuite) TestNexusProgressAcrossFailoverConversion() { _, started, err := s.mutableState.AddWorkflowTaskStartedEvent( wt.ScheduledEventID, uuid.NewString(), - &taskqueuepb.TaskQueue{Name: f.TaskQueue(enumspb.TASK_QUEUE_TYPE_WORKFLOW).NormalPartition(5).RpcName()}, - "random identity", nil, nil, nil, false, nil, 0, + &taskqueuepb.TaskQueue{ + Name: f.TaskQueue(enumspb.TASK_QUEUE_TYPE_WORKFLOW).NormalPartition(5).RpcName(), + }, + "random identity", + nil, + nil, + nil, + false, + nil, + 0, ) s.NoError(err) s.NotNil(started) diff --git a/service/history/workflow/nexus_progress.go b/service/history/workflow/nexus_progress.go index a23aaaf9b80..41529b7f918 100644 --- a/service/history/workflow/nexus_progress.go +++ b/service/history/workflow/nexus_progress.go @@ -48,7 +48,9 @@ func (ms *MutableStateImpl) attachNexusProgress(event *historypb.HistoryEvent) { return } attrs := event.GetWorkflowTaskScheduledEventAttributes() - attrs.NexusOperationProgress = append(attrs.NexusOperationProgress, wf.TakeNexusProgress(chasmCtx)...) + attrs.NexusOperationProgress = append( + attrs.NexusOperationProgress, + wf.TakeNexusProgress(chasmCtx)...) } // clearScheduledNexusProgress forgets which operations the scheduled Workflow Task carries, once @@ -96,7 +98,8 @@ func (ms *MutableStateImpl) scheduleWorkflowTaskForNexusProgress() error { if readyAt := wf.NexusProgressReadyAt(chasmCtx, minInterval); now.Before(readyAt) { return ms.waitNexusProgress(readyAt) } - if _, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL); err != nil { + _, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + if err != nil { return err } if !ms.hasPendingNexusProgress() { diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index 948003ca759..3126780b79f 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -331,8 +331,8 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskScheduledEventAsHeartbeat( createWorkflowTaskScheduledEvent := !m.ms.IsTransientWorkflowTask() && workflowTaskType != enumsspb.WORKFLOW_TASK_TYPE_SPECULATIVE // A cluster rebuilt from replicated events never learns workflow_task_holds_nexus_progress, so - // a task a failover converts may retry a heartbeat task without the hold. Such a task carries no - // progress; the progress waits for the task after it. + // a task a failover converts may retry a heartbeat task without the hold. Such a task carries + // no progress; the progress waits for the task after it. holdForFailover := false // If while scheduling a workflow task and new events has come, then this workflow task cannot be a transient/speculative. @@ -601,7 +601,8 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent( startTime, ) // The worker has not seen this task yet, so its scheduled event can carry what is pending, - // unless a failover converted it (see holdForFailover in AddWorkflowTaskScheduledEventAsHeartbeat). + // unless a failover converted it (see holdForFailover in + // AddWorkflowTaskScheduledEventAsHeartbeat). if workflowTask.Version == m.ms.GetCurrentVersion() { m.ms.attachNexusProgress(scheduledEvent) } diff --git a/tests/nexus_progress_test.go b/tests/nexus_progress_test.go index 174f2010fe3..15eb61bd7ed 100644 --- a/tests/nexus_progress_test.go +++ b/tests/nexus_progress_test.go @@ -6,6 +6,7 @@ import ( "net/http" "strings" "sync" + "testing" "time" "github.com/nexus-rpc/sdk-go/nexus" @@ -153,9 +154,11 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressIntake(chasmEnabled b accepted := `{"position": "cursor-2", "counter": "2", "metadata": {"topic": "t"}}` s.Equal(http.StatusOK, progress(accepted)) // An accepted delivery rides the next Workflow Task's scheduled event. - scheduledEventID := s.RequireHistoryEvent(started, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() + scheduledEventID := s.RequireHistoryEvent( + started, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED).GetEventId() s.Await(func(s *NexusWorkflowTestSuite) { - carried := carriedNexusProgress(env.GetHistory(env.Namespace().String(), wfExec)[len(started):]) + hist := env.GetHistory(env.Namespace().String(), wfExec) + carried := carriedNexusProgress(hist[len(started):]) s.Len(carried, 1) s.Equal(scheduledEventID, carried[0].GetScheduledEventId()) s.Equal(int64(2), carried[0].GetCounter()) @@ -182,11 +185,28 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressIntake(chasmEnabled b } } +func scheduleNexusOperationCmd(t *testing.T, endpoint string) *commandpb.Command { + attrs := &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpoint, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(t, "input"), + } + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: attrs, + }, + } +} + // carriedNexusProgress lists the Nexus operation progress the scheduled events in hist carry. func carriedNexusProgress(hist []*historypb.HistoryEvent) []*nexuspb.NexusOperationProgress { var carried []*nexuspb.NexusOperationProgress for _, event := range hist { - carried = append(carried, event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) + carried = append( + carried, + event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) } return carried } @@ -218,13 +238,15 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { callerWF := func(ctx workflow.Context) (string, error) { c := workflow.NewNexusClient(endpointName, "service") var result string - if err := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}).Get(ctx, &result); err != nil { + fut := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}) + if err := fut.Get(ctx, &result); err != nil { return "", err } workflow.GetSignalChannel(ctx, "finish").Receive(ctx, nil) return result, nil } - run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, callerWF) + run, err := env.SdkClient(). + ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, callerWF) s.NoError(err) newWorker := func() worker.Worker { w := worker.New(env.SdkClient(), taskQueue, worker.Options{}) @@ -239,7 +261,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { return env.GetHistory(env.Namespace().String(), wfExec) } // Idle means the Workflow is blocked on the operation or the Signal with no task in flight. - waitIdle := func(check func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent)) []*historypb.HistoryEvent { + waitIdle := func( + check func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent), + ) []*historypb.HistoryEvent { var hist []*historypb.HistoryEvent s.Await(func(s *NexusWorkflowTestSuite) { desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID()) @@ -252,7 +276,12 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { return hist } progress := func(counter int) int { - status, err := postNexusProgress(ctx, callbackURL, callbackToken, fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter)) + status, err := postNexusProgress( + ctx, + callbackURL, + callbackToken, + fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter), + ) s.NoError(err) return status } @@ -270,7 +299,11 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) }) if !chasmEnabled { - s.Equal(http.StatusBadRequest, progress(1), "an HSM caller refuses progress, so the handler stops sending it") + s.Equal( + http.StatusBadRequest, + progress(1), + "an HSM caller refuses progress, so the handler stops sending it", + ) w.Stop() return } @@ -290,14 +323,22 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { afterFold := waitIdle(func(s *NexusWorkflowTestSuite, hist []*historypb.HistoryEvent) { carried := carriedNexusProgress(hist[len(beforeBurst):]) s.NotEmpty(carried) - s.Equal(int64(3), carried[len(carried)-1].GetCounter(), "the Workflow sees the highest counter") + s.Equal( + int64(3), + carried[len(carried)-1].GetCounter(), + "the Workflow sees the highest counter", + ) }) folded := afterFold[len(beforeBurst):] onlyWorkflowTaskEvents(folded) carried := carriedNexusProgress(folded) s.Len(carried, 2) s.Equal(scheduledEventID, carried[0].GetScheduledEventId()) - s.Equal(int64(1), carried[0].GetCounter(), "the task that was already scheduled carries what it had") + s.Equal( + int64(1), + carried[0].GetCounter(), + "the task that was already scheduled carries what it had", + ) s.Equal(int64(3), carried[1].GetCounter(), "one follow-up task carries what folded into it") var scheduledTasks int for _, event := range folded { @@ -336,10 +377,11 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgress(chasmEnabled bool) { replayer.RegisterWorkflow(callerWF) s.NoError(replayer.ReplayWorkflowHistory(nil, &historypb.History{Events: history()})) - resp, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ - Namespace: env.Namespace().String(), - Execution: wfExec, - }) + resp, err := env.FrontendClient(). + GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: wfExec, + }) s.NoError(err) s.Len(carriedNexusProgress(resp.GetHistory().GetEvents()), 3, "the progress is part of History") } @@ -388,8 +430,12 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c if err := fut.GetNexusOperationExecution().Get(ctx, nil); err != nil { return "", err } - laCtx := workflow.WithLocalActivityOptions(ctx, workflow.LocalActivityOptions{StartToCloseTimeout: time.Minute}) - if err := workflow.ExecuteLocalActivity(laCtx, longLocalActivity).Get(ctx, nil); err != nil { + laCtx := workflow.WithLocalActivityOptions( + ctx, + workflow.LocalActivityOptions{StartToCloseTimeout: time.Minute}, + ) + la := workflow.ExecuteLocalActivity(laCtx, longLocalActivity) + if err := la.Get(ctx, nil); err != nil { return "", err } var result string @@ -419,7 +465,12 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c } beforeProgress := history() - status, err := postNexusProgress(ctx, callbackURL, callbackToken, `{"position": "p1", "counter": 1}`) + status, err := postNexusProgress( + ctx, + callbackURL, + callbackToken, + `{"position": "p1", "counter": 1}`, + ) s.NoError(err) s.Equal(http.StatusOK, status) countScheduled := func(hist []*historypb.HistoryEvent) int { @@ -434,7 +485,11 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c var duringLA []*historypb.HistoryEvent s.Await(func(s *NexusWorkflowTestSuite) { duringLA = history()[len(beforeProgress):] - s.GreaterOrEqual(countScheduled(duringLA), 2, "the worker keeps heartbeating the local activity") + s.GreaterOrEqual( + countScheduled(duringLA), + 2, + "the worker keeps heartbeating the local activity", + ) }, 10*time.Second, 50*time.Millisecond) s.Empty(carriedNexusProgress(duringLA), "no heartbeat task carries progress") @@ -455,7 +510,11 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c } } s.Positive(markerIdx, "the local activity's result is recorded") - s.Greater(carrierIdx, markerIdx, "the first normal task after the heartbeats carries the progress") + s.Greater( + carrierIdx, + markerIdx, + "the first normal task after the heartbeats carries the progress", + ) s.Equal(int64(1), carriedNexusProgress(afterLA)[0].GetCounter()) s.NoError(s.sendNexusCompletionRequest(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ @@ -474,7 +533,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatTasks(c // TestNexusOperationProgressSkipsHeartbeatRetries checks that the retry of a failed heartbeat task // holds progress like the task it retries, and that the next normal task carries it. A Signal // arrives before the retry starts, so the retry gets a scheduled event of its own. -func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries(chasmEnabled bool) { +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries( + chasmEnabled bool, +) { if !chasmEnabled { // An HSM caller refuses progress; TestNexusOperationProgress covers it. return @@ -499,31 +560,47 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries } endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) - run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + run, err := env.SdkClient(). + ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") s.NoError(err) frontend := env.FrontendClient() poll := func() *workflowservice.PollWorkflowTaskQueueResponse { - resp, err := frontend.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: ns, - TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := frontend.PollWorkflowTaskQueue( + ctx, + &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }, + ) s.NoError(err) s.NotEmpty(resp.GetTaskToken()) return resp } - respond := func(task *workflowservice.PollWorkflowTaskQueueResponse, forceCreate bool, commands ...*commandpb.Command) { - _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: ns, - Identity: "test", - TaskToken: task.GetTaskToken(), - Commands: commands, - ForceCreateNewWorkflowTask: forceCreate, - }) + respond := func( + task *workflowservice.PollWorkflowTaskQueueResponse, + forceCreate bool, + commands ...*commandpb.Command, + ) { + _, err := frontend.RespondWorkflowTaskCompleted( + ctx, + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + ForceCreateNewWorkflowTask: forceCreate, + }, + ) s.NoError(err) } // lastScheduled is the scheduled event of the task a poll answered. - lastScheduled := func(task *workflowservice.PollWorkflowTaskQueueResponse) *historypb.HistoryEvent { + lastScheduled := func( + task *workflowservice.PollWorkflowTaskQueueResponse, + ) *historypb.HistoryEvent { events := task.GetHistory().GetEvents() for i := len(events) - 1; i >= 0; i-- { if events[i].GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { @@ -534,52 +611,64 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries return nil } - respond(poll(), false, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: endpointName, - Service: "service", - Operation: "operation", - Input: testcore.MustToPayload(s.T(), "input"), - }, - }, - }) + respond(poll(), false, scheduleNexusOperationCmd(s.T(), endpointName)) startedTask := poll() - s.RequireHistoryEvent(startedTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + s.RequireHistoryEvent( + startedTask.GetHistory().GetEvents(), + enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, + ) // The worker forces a heartbeat task, takes it, gets progress meanwhile, and fails it. respond(startedTask, true) heartbeatTask := poll() - s.Empty(lastScheduled(heartbeatTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()) - status, err := postNexusProgress(ctx, callbackURL, callbackToken, `{"position": "p1", "counter": 1}`) + s.Empty( + lastScheduled( + heartbeatTask, + ).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress(), + ) + status, err := postNexusProgress( + ctx, + callbackURL, + callbackToken, + `{"position": "p1", "counter": 1}`, + ) s.NoError(err) s.Equal(http.StatusOK, status) - _, err = frontend.RespondWorkflowTaskFailed(ctx, &workflowservice.RespondWorkflowTaskFailedRequest{ - Namespace: ns, - Identity: "test", - TaskToken: heartbeatTask.GetTaskToken(), - Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, - }) + _, err = frontend.RespondWorkflowTaskFailed( + ctx, + &workflowservice.RespondWorkflowTaskFailedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: heartbeatTask.GetTaskToken(), + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, + }, + ) s.NoError(err) s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "wake", nil)) retryTask := poll() - s.RequireHistoryEvent(retryTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED) - s.Empty(lastScheduled(retryTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), - "the retry of a heartbeat task carries no progress") + s.RequireHistoryEvent( + retryTask.GetHistory().GetEvents(), + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, + ) + s.Empty( + lastScheduled( + retryTask, + ).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress(), + "the retry of a heartbeat task carries no progress", + ) respond(retryTask, false) normalTask := poll() - carried := lastScheduled(normalTask).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + carried := lastScheduled( + normalTask, + ).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress() s.Len(carried, 1, "the next normal task carries the progress") s.Equal(int64(1), carried[0].GetCounter()) - respond(normalTask, false, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }) + respond(normalTask, false, completeWorkflowCmd()) s.NoError(run.Get(ctx, nil)) } @@ -587,7 +676,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressSkipsHeartbeatRetries // arrives while a speculative task for an Update is pending, and the Update is then rejected. The // progress transaction turns the speculative task into a normal one: one that has not started // carries the progress, and one already started is kept, so a follow-up task carries it. -func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks(chasmEnabled bool) { +func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks( + chasmEnabled bool, +) { if !chasmEnabled { // An HSM caller refuses progress; TestNexusOperationProgress covers it. return @@ -614,85 +705,107 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks } endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) - run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + run, err := env.SdkClient(). + ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") s.NoError(err) wfExec := &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()} frontend := env.FrontendClient() poll := func() *workflowservice.PollWorkflowTaskQueueResponse { - resp, err := frontend.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: ns, - TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := frontend.PollWorkflowTaskQueue( + ctx, + &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }, + ) s.NoError(err) s.NotEmpty(resp.GetTaskToken()) return resp } - respond := func(task *workflowservice.PollWorkflowTaskQueueResponse, commands ...*commandpb.Command) { - _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: ns, - Identity: "test", - TaskToken: task.GetTaskToken(), - Commands: commands, - }) + respond := func( + task *workflowservice.PollWorkflowTaskQueueResponse, + commands ...*commandpb.Command, + ) { + _, err := frontend.RespondWorkflowTaskCompleted( + ctx, + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + }, + ) s.NoError(err) } rejectUpdate := func(task *workflowservice.PollWorkflowTaskQueueResponse) { s.Len(task.GetMessages(), 1, "the task carries the Update request") - _, err := frontend.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: ns, - Identity: "test", - TaskToken: task.GetTaskToken(), - Messages: updates.UpdateRejectMessages(tv, task.GetMessages()[0]), - }) + _, err := frontend.RespondWorkflowTaskCompleted( + ctx, + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: ns, + Identity: "test", + TaskToken: task.GetTaskToken(), + Messages: updates.UpdateRejectMessages(tv, task.GetMessages()[0]), + }, + ) s.NoError(err) } sendUpdate := func(updateID string) <-chan *updatepb.Outcome { done := make(chan *updatepb.Outcome, 1) go func() { - resp, err := frontend.UpdateWorkflowExecution(ctx, &workflowservice.UpdateWorkflowExecutionRequest{ - Namespace: ns, - WorkflowExecution: wfExec, - Request: &updatepb.Request{ - Meta: &updatepb.Meta{UpdateId: updateID}, - Input: &updatepb.Input{Name: "update"}, + resp, err := frontend.UpdateWorkflowExecution( + ctx, + &workflowservice.UpdateWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: wfExec, + Request: &updatepb.Request{ + Meta: &updatepb.Meta{UpdateId: updateID}, + Input: &updatepb.Input{Name: "update"}, + }, + WaitPolicy: &updatepb.WaitPolicy{ + LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, + }, }, - WaitPolicy: &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}, - }) + ) s.NoError(err) done <- resp.GetOutcome() }() return done } // carried is the progress on the scheduled event of the task a poll answered. - carried := func(task *workflowservice.PollWorkflowTaskQueueResponse) []*nexuspb.NexusOperationProgress { + carried := func( + task *workflowservice.PollWorkflowTaskQueueResponse, + ) []*nexuspb.NexusOperationProgress { events := task.GetHistory().GetEvents() for i := len(events) - 1; i >= 0; i-- { if events[i].GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { - return events[i].GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + return events[i].GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress() } } return nil } postProgress := func(counter int) { - status, err := postNexusProgress(ctx, callbackURL, callbackToken, fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter)) + status, err := postNexusProgress( + ctx, + callbackURL, + callbackToken, + fmt.Sprintf(`{"position": "p%d", "counter": %d}`, counter, counter), + ) s.NoError(err) s.Equal(http.StatusOK, status) } - respond(poll(), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: endpointName, - Service: "service", - Operation: "operation", - Input: testcore.MustToPayload(s.T(), "input"), - }, - }, - }) + respond(poll(), scheduleNexusOperationCmd(s.T(), endpointName)) startedTask := poll() - s.RequireHistoryEvent(startedTask.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + s.RequireHistoryEvent( + startedTask.GetHistory().GetEvents(), + enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, + ) respond(startedTask) // An Update schedules a speculative task, and progress arrives before any worker takes it. @@ -719,12 +832,7 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressRidesSpeculativeTasks followUp := poll() s.Len(carried(followUp), 1, "a follow-up task carries the progress the started task could not") s.Equal(int64(2), carried(followUp)[0].GetCounter()) - respond(followUp, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }) + respond(followUp, completeWorkflowCmd()) s.NoError(run.Get(ctx, nil)) } @@ -771,29 +879,26 @@ func (s *NexusWorkflowTestSuite) newRawProgressCaller( c.ctx, client.StartWorkflowOptions{TaskQueue: c.taskQueue}, "workflow") s.NoError(err) c.run = run - c.respond(c.poll(), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: endpointName, - Service: "service", - Operation: "operation", - Input: testcore.MustToPayload(s.T(), "input"), - }, - }, - }) + c.respond(c.poll(), scheduleNexusOperationCmd(s.T(), endpointName)) started := c.poll() - s.RequireHistoryEvent(started.GetHistory().GetEvents(), enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED) + s.RequireHistoryEvent( + started.GetHistory().GetEvents(), + enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, + ) c.respond(started) return c } func (c *rawProgressCaller) poll() *workflowservice.PollWorkflowTaskQueueResponse { - resp, err := c.env.FrontendClient().PollWorkflowTaskQueue(c.ctx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: c.env.Namespace().String(), - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := c.env.FrontendClient(). + PollWorkflowTaskQueue(c.ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: c.env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{ + Name: c.taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }) c.s.NoError(err) c.s.NotEmpty(resp.GetTaskToken()) return resp @@ -803,22 +908,18 @@ func (c *rawProgressCaller) respond( task *workflowservice.PollWorkflowTaskQueueResponse, commands ...*commandpb.Command, ) { - _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: c.env.Namespace().String(), - Identity: "test", - TaskToken: task.GetTaskToken(), - Commands: commands, - }) + _, err := c.env.FrontendClient(). + RespondWorkflowTaskCompleted(c.ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: c.env.Namespace().String(), + Identity: "test", + TaskToken: task.GetTaskToken(), + Commands: commands, + }) c.s.NoError(err) } func (c *rawProgressCaller) complete(task *workflowservice.PollWorkflowTaskQueueResponse) { - c.respond(task, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }) + c.respond(task, completeWorkflowCmd()) c.s.NoError(c.run.Get(c.ctx, nil)) } @@ -849,7 +950,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressMinInterval(chasmEnab return } const interval = 2 * time.Second - c := s.newRawProgressCaller(testcore.WithDynamicConfig(chasmnexus.ProgressMinInterval, interval)) + c := s.newRawProgressCaller( + testcore.WithDynamicConfig(chasmnexus.ProgressMinInterval, interval), + ) c.postProgress(1) first := c.poll() @@ -860,11 +963,22 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressMinInterval(chasmEnab c.postProgress(2) c.postProgress(3) second := c.poll() - carried := lastScheduled(second).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress() + carried := lastScheduled( + second, + ).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress() s.Len(carried, 1) s.Equal(int64(3), carried[0].GetCounter(), "progress that waited folds into one task") - gap := lastScheduled(second).GetEventTime().AsTime().Sub(lastScheduled(first).GetEventTime().AsTime()) - s.GreaterOrEqual(gap, interval, "progress waits out the interval since the last task that carried it") + gap := lastScheduled( + second, + ).GetEventTime(). + AsTime(). + Sub(lastScheduled(first).GetEventTime().AsTime()) + s.GreaterOrEqual( + gap, + interval, + "progress waits out the interval since the last task that carried it", + ) c.complete(second) } @@ -881,19 +995,28 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationProgressOnFailedTask(chasmEna c.postProgress(1) failing := c.poll() - s.Len(lastScheduled(failing).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), 1) - _, err := c.env.FrontendClient().RespondWorkflowTaskFailed(c.ctx, &workflowservice.RespondWorkflowTaskFailedRequest{ - Namespace: c.env.Namespace().String(), - Identity: "test", - TaskToken: failing.GetTaskToken(), - Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, - }) + s.Len( + lastScheduled( + failing, + ).GetWorkflowTaskScheduledEventAttributes(). + GetNexusOperationProgress(), + 1, + ) + _, err := c.env.FrontendClient(). + RespondWorkflowTaskFailed(c.ctx, &workflowservice.RespondWorkflowTaskFailedRequest{ + Namespace: c.env.Namespace().String(), + Identity: "test", + TaskToken: failing.GetTaskToken(), + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, + }) s.NoError(err) retry := c.poll() s.Equal(int32(2), retry.GetAttempt()) - s.Empty(lastScheduled(retry).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), - "the retry doesn't repeat what the failed task carried") + s.Empty( + lastScheduled(retry).GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress(), + "the retry doesn't repeat what the failed task carried", + ) carried := carriedNexusProgress(retry.GetHistory().GetEvents()) s.Len(carried, 1, "the failed task's scheduled event still carries the progress") s.Equal(int64(1), carried[0].GetCounter()) diff --git a/tests/xdc/nexus_progress_forwarding_test.go b/tests/xdc/nexus_progress_forwarding_test.go index e3394614ad9..871f1ce1d2f 100644 --- a/tests/xdc/nexus_progress_forwarding_test.go +++ b/tests/xdc/nexus_progress_forwarding_test.go @@ -44,15 +44,16 @@ func TestNexusProgressForwardingSuite(t *testing.T) { func (s *NexusProgressForwardingSuite) SetupSuite() { s.dynamicConfigOverrides = map[dynamicconfig.Key]any{ - dynamicconfig.FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS.Key(): 1000, - dynamicconfig.RefreshNexusEndpointsMinWait.Key(): 1 * time.Millisecond, - dynamicconfig.EnableChasm.Key(): true, - dynamicconfig.EnableCHASMCallbacks.Key(): true, - chasmnexus.Enabled.Key(): true, - chasmnexus.EnableChasmWorkflowOperations.Key(): true, - chasmnexus.ChasmWorkflowOperationsRolloutPercent.Key(): 100, - chasmnexus.EnableProgress.Key(): true, + dynamicconfig.RefreshNexusEndpointsMinWait.Key(): 1 * time.Millisecond, + dynamicconfig.EnableChasm.Key(): true, + dynamicconfig.EnableCHASMCallbacks.Key(): true, + chasmnexus.Enabled.Key(): true, + chasmnexus.EnableChasmWorkflowOperations.Key(): true, + chasmnexus.ChasmWorkflowOperationsRolloutPercent.Key(): 100, + chasmnexus.EnableProgress.Key(): true, } + rpsKey := dynamicconfig.FrontendGlobalNamespaceNamespaceReplicationInducingAPIsRPS.Key() + s.dynamicConfigOverrides[rpsKey] = 1000 s.setupSuite() } @@ -68,7 +69,9 @@ func (s *NexusProgressForwardingSuite) TearDownSuite() { // to the active one instead of refusing it, so the caller's next Workflow Task carries it. func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive() { // The callback URL points at the standby cluster, so every delivery lands there first. - standbyCallbackURL := "http://" + s.clusters[1].Host().FrontendHTTPAddress() + "/namespaces/{{.NamespaceName}}/nexus/callback" + standbyCallbackURL := "http://" + s.clusters[1].Host(). + FrontendHTTPAddress() + + "/namespaces/{{.NamespaceName}}/nexus/callback" s.clusters[0].OverrideDynamicConfig(s.T(), chasmnexus.CallbackURLTemplate, standbyCallbackURL) s.clusters[1].OverrideDynamicConfig(s.T(), chasmnexus.CallbackURLTemplate, standbyCallbackURL) @@ -79,7 +82,13 @@ func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive( var callbackToken, callbackURL string h := nexustest.Handler{ - OnStartOperation: func(ctx context.Context, service, operation string, input *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + OnStartOperation: func( + ctx context.Context, + service, + operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { callbackToken = options.CallbackHeader.Get(cnexus.CallbackTokenHeader) callbackURL = options.CallbackURL return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil @@ -108,43 +117,52 @@ func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive( Logger: log.NewSdkLogger(s.logger), }) s.NoError(err) - run, err := activeSDKClient.ExecuteWorkflow(ctx, client.StartWorkflowOptions{TaskQueue: taskQueue}, "workflow") + run, err := activeSDKClient.ExecuteWorkflow( + ctx, + client.StartWorkflowOptions{TaskQueue: taskQueue}, + "workflow", + ) s.NoError(err) active := s.clusters[0].FrontendClient() poll := func() *workflowservice.PollWorkflowTaskQueueResponse { - resp, err := active.PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: ns, - TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := active.PollWorkflowTaskQueue( + ctx, + &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }, + ) s.NoError(err) return resp } - respond := func(resp *workflowservice.PollWorkflowTaskQueueResponse, commands ...*commandpb.Command) { - _, err := active.RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: resp.TaskToken, - Commands: commands, - }) + respond := func( + resp *workflowservice.PollWorkflowTaskQueueResponse, + commands ...*commandpb.Command, + ) { + _, err := active.RespondWorkflowTaskCompleted( + ctx, + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Identity: "test", + TaskToken: resp.TaskToken, + Commands: commands, + }, + ) s.NoError(err) } - respond(poll(), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: endpointName, - Service: "service", - Operation: "operation", - Input: testcore.MustToPayload(s.T(), "input"), - }, - }, - }) + respond(poll(), scheduleNexusOperationCmd(s.T(), endpointName)) startedTask := poll() - scheduledIdx := slices.IndexFunc(startedTask.History.Events, func(e *historypb.HistoryEvent) bool { - return e.GetNexusOperationScheduledEventAttributes() != nil - }) + scheduledIdx := slices.IndexFunc( + startedTask.History.Events, + func(e *historypb.HistoryEvent) bool { + return e.GetNexusOperationScheduledEventAttributes() != nil + }, + ) s.Positive(scheduledIdx) s.Positive(slices.IndexFunc(startedTask.History.Events, func(e *historypb.HistoryEvent) bool { return e.GetNexusOperationStartedEventAttributes() != nil @@ -153,16 +171,23 @@ func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive( // The standby has to know the operation before it can forward progress for it. await.RequireTrue(s.T(), func() bool { - resp, err := s.clusters[1].FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ - Namespace: ns, - Execution: &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()}, - }) + resp, err := s.clusters[1].FrontendClient(). + DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + }, + }) return err == nil && len(resp.PendingNexusOperations) > 0 }, 5*time.Second, 200*time.Millisecond) c := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{}) err = c.CompleteOperation(ctx, callbackURL, nexusrpc.CompleteOperationOptions{ - Result: &nexus.Content{Header: nexus.Header{"type": "application/json"}, Data: []byte(`{"position": "p1", "counter": 1}`)}, + Result: &nexus.Content{ + Header: nexus.Header{"type": "application/json"}, + Data: []byte(`{"position": "p1", "counter": 1}`), + }, Progress: true, Header: nexus.Header{cnexus.CallbackTokenHeader: callbackToken}, }) @@ -171,17 +196,39 @@ func (s *NexusProgressForwardingSuite) TestProgressForwardedFromStandbyToActive( progressTask := poll() var carried []*nexuspb.NexusOperationProgress for _, event := range progressTask.History.Events { - carried = append(carried, event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) + carried = append( + carried, + event.GetWorkflowTaskScheduledEventAttributes().GetNexusOperationProgress()...) } s.Len(carried, 1) s.Equal(startedTask.History.Events[scheduledIdx].GetEventId(), carried[0].GetScheduledEventId()) s.Equal(int64(1), carried[0].GetCounter()) s.Equal("p1", carried[0].GetPosition()) - respond(progressTask, &commandpb.Command{ + respond(progressTask, completeWorkflowCmd()) + s.NoError(run.Get(ctx, nil)) +} + +func scheduleNexusOperationCmd(t *testing.T, endpoint string) *commandpb.Command { + attrs := &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpoint, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(t, "input"), + } + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: attrs, + }, + } +} + +func completeWorkflowCmd() *commandpb.Command { + attrs := &commandpb.CompleteWorkflowExecutionCommandAttributes{} + return &commandpb.Command{ CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + CompleteWorkflowExecutionCommandAttributes: attrs, }, - }) - s.NoError(run.Get(ctx, nil)) + } }