From 3f07c70bdab27408a62210b3796c1e1d32ab58a2 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:10:06 +0100 Subject: [PATCH 01/11] [AI-3334] Count scheduled occurrences, campaign dials, simulation runs, MCP events and 10DLC reports (WIP) --- acceleration/cmd/router/main.go | 9 +++-- acceleration/internal/activity/activity.go | 14 ++++---- acceleration/internal/campaign/campaign.go | 34 +++++++++++++++++-- acceleration/internal/dlc/service.go | 5 ++- acceleration/internal/mcpevents/mcpevents.go | 5 ++- acceleration/internal/scheduling/scheduler.go | 33 ++++++++++++++++++ .../internal/simulation/simulation.go | 20 ++++++++--- 7 files changed, 103 insertions(+), 17 deletions(-) diff --git a/acceleration/cmd/router/main.go b/acceleration/cmd/router/main.go index 9420447e9d..249ac072d1 100644 --- a/acceleration/cmd/router/main.go +++ b/acceleration/cmd/router/main.go @@ -1237,9 +1237,10 @@ func run(settings config.Config, logger *slog.Logger) error { LLM: streams.LLM, // Speech is what an audio simulation needs and a text one does not, so it is // passed where it exists rather than required. - TTS: streams.TTS, - STT: streams.STT, - Logger: logger, + TTS: streams.TTS, + STT: streams.STT, + Activity: activityRecord, + Logger: logger, }) if err != nil { return err @@ -1623,6 +1624,7 @@ func run(settings config.Config, logger *slog.Logger) error { var scheduler *scheduling.Scheduler if pgStore != nil && sessions != nil { scheduler = scheduling.NewScheduler(pgStore, server.DeliverScheduledTask, logger) + scheduler.SetActivity(activityRecord) // Before Start: a session the first delivery opens takes the kick then. server.SetSchedulerKick(scheduler.Kick) scheduler.Start(ctx) @@ -1637,6 +1639,7 @@ func run(settings config.Config, logger *slog.Logger) error { Dialer: campaign.NewPhoneDialer(telephony, sessions, pgStore, logger), Transcripts: server, Classifier: campaign.NewClassifier(streams.LLM), + Activity: activityRecord, Logger: logger, }) if err != nil { diff --git a/acceleration/internal/activity/activity.go b/acceleration/internal/activity/activity.go index e4e05b3421..57bd511c4d 100644 --- a/acceleration/internal/activity/activity.go +++ b/acceleration/internal/activity/activity.go @@ -1,8 +1,9 @@ // Package activity counts the requests each app makes, by area of the API and by client, and // writes the sums to customer_activity (store.RecordActivity). // -// Every authenticated request is one, and so is a hook checked as an app's (NoteCustomer) and a -// forward to an app's own endpoint, so the counting is in memory and the writing is not on +// Every authenticated request is one, and so is a hook checked as an app's (NoteCustomer), and +// the work the router does for an app on its own: a forward to its endpoint, a scheduled task's +// occurrence, a campaign's dial and a simulation run. So the counting is in memory and the writing is not on // the request path: a minute's requests for one app, area, client and day are one row's // update. Nothing here fails a request: counting it is not what the caller asked for. package activity @@ -39,8 +40,8 @@ const ( AreaOther = "other" ) -// ClientRouter is the client work the router starts itself is counted under: a forward to a -// customer's own endpoint. It is the API's, the client a caller that named none is counted +// ClientRouter is the client work the router starts itself is counted under, such as a forward to +// a customer's own endpoint or a scheduled task's occurrence. It is the API's, the client a caller that named none is counted // under, so no row says a client sent what nobody sent. const ClientRouter = store.AuditSourceAPI @@ -180,9 +181,10 @@ func (r *Recorder) Close() { }) } -// Record counts one request. A request naming no app is nobody's and is not counted. +// Record counts one request. A request naming no app is nobody's and is not counted, and a nil +// Recorder, a router with no database, counts nothing. func (r *Recorder) Record(request Request) { - if request.AppID == "" { + if r == nil || request.AppID == "" { return } at := request.At.UTC() diff --git a/acceleration/internal/campaign/campaign.go b/acceleration/internal/campaign/campaign.go index 21c40759c9..6999eefef3 100644 --- a/acceleration/internal/campaign/campaign.go +++ b/acceleration/internal/campaign/campaign.go @@ -23,6 +23,7 @@ import ( "github.com/google/uuid" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/chatlog" "github.com/GetStream/Vision-Agents/acceleration/internal/dlc" "github.com/GetStream/Vision-Agents/acceleration/internal/phone" @@ -115,8 +116,10 @@ type Options struct { // Now is the clock. Nil is time.Now. Now func() time.Time // Tick is how often to look for work. Zero is 2 seconds. - Tick time.Duration - Logger *slog.Logger + Tick time.Duration + // Activity counts each dial under its campaign's app, as area phone_sms. Nil counts none. + Activity *activity.Recorder + Logger *slog.Logger } // Runner places the calls of every running campaign that this router claims. @@ -128,6 +131,7 @@ type Runner struct { owner string now func() time.Time every time.Duration + activity *activity.Recorder logger *slog.Logger // calls is what calls are placed under. Close cancels it only once the calls in flight @@ -179,6 +183,7 @@ func New(options Options) (*Runner, error) { owner: options.Owner, now: options.Now, every: cmp.Or(options.Tick, defaultTick), + activity: options.Activity, logger: cmp.Or(options.Logger, slog.Default()), calls: calls, cancelCalls: cancelCalls, @@ -447,6 +452,8 @@ func (r *Runner) ring(ctx context.Context, campaign store.Campaign, contact stor } } + r.countDial(campaign, dialed, err) + written, cancel := context.WithTimeout(context.WithoutCancel(ctx), writeTimeout) defer cancel() ended := r.now() @@ -461,6 +468,29 @@ func (r *Runner) ring(ctx context.Context, campaign store.Campaign, contact stor return dialed, nil } +// codeDialFailed is the code a dial that could not be tried, and gave no reason, is counted +// refused under. +const codeDialFailed = "dial_failed" + +// countDial counts a dial in the activity record, refused when it could not be tried or placed +// no call, with its reason. A dial a shutdown cut off rang nobody and leaves the contact to be +// rung again, so it is counted when that dial is. +func (r *Runner) countDial(campaign store.Campaign, dialed Dialed, err error) { + if dialed.Result == NotPlaced { + return + } + request := activity.Request{ + AppID: campaign.CustomerID, + Area: activity.AreaPhoneSMS, + Client: activity.ClientRouter, + At: r.now(), + } + if err != nil || dialed.Result != "" && dialed.Result != Voicemail { + request.Refused, request.Code = true, cmp.Or(dialed.Reason, string(dialed.Result), codeDialFailed) + } + r.activity.Record(request) +} + // takeOver settles the contacts whose router died mid-call, from the call it placed. None is // rung again. func (r *Runner) takeOver(ctx context.Context) { diff --git a/acceleration/internal/dlc/service.go b/acceleration/internal/dlc/service.go index 3332e66016..b33c191b97 100644 --- a/acceleration/internal/dlc/service.go +++ b/acceleration/internal/dlc/service.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/routing" "github.com/GetStream/Vision-Agents/acceleration/internal/stack" "github.com/GetStream/Vision-Agents/acceleration/internal/store" @@ -217,7 +218,8 @@ func (s *Service) Review(ctx context.Context, id, decision, notes, reviewer stri } // Hook takes a vendor's report on a brand or a campaign, verified, and moves whichever use -// cases it is about. +// cases it is about. A report on a campaign is counted as its use case's app's phone work. One on +// a brand is not: the hook reads no app for it, and has every waiting use case registered. func (s *Service) Hook(ctx context.Context, header http.Header, body []byte) error { if s.registrar == nil { return fmt.Errorf("%w: no registrar is configured", ErrRefused) @@ -234,6 +236,7 @@ func (s *Service) Hook(ctx context.Context, header http.Header, body []byte) err if err != nil { return err } + activity.NoteCustomer(ctx, useCase.CustomerID, activity.AreaPhoneSMS) return s.refresh(ctx, useCase, body) } if brandID != "" { diff --git a/acceleration/internal/mcpevents/mcpevents.go b/acceleration/internal/mcpevents/mcpevents.go index 8553f559a4..65bb694e3b 100644 --- a/acceleration/internal/mcpevents/mcpevents.go +++ b/acceleration/internal/mcpevents/mcpevents.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/auth" "github.com/GetStream/Vision-Agents/acceleration/internal/connectors/core" "github.com/GetStream/Vision-Agents/acceleration/internal/options" @@ -593,7 +594,8 @@ func (s *Service) owners(configs []store.AgentConfig, connection store.Connector // Receive answers one delivery to a subscription's callback: the challenge a server checks the // callback with, or an event, which opens a conversation. A delivery not signed with the -// subscription's own secret is refused before its body is read. +// subscription's own secret is refused before its body is read, and counted for nobody: a signed +// one is the subscription's app's hook, whatever is answered. func (s *Service) Receive(ctx context.Context, token string, header http.Header, body []byte) Reply { sub, err := s.store.ConnectionEventSubscriptionByToken(ctx, token) if errors.Is(err, store.ErrNoConnectionEventSubscription) { @@ -610,6 +612,7 @@ func (s *Service) Receive(ctx context.Context, token string, header http.Header, if err := webhook.Verify(secret, header, body, time.Now()); err != nil { return Reply{Status: http.StatusUnauthorized, Body: failure("the delivery is not signed with the subscription's secret")} } + activity.NoteCustomer(ctx, sub.CustomerID, activity.AreaHooksIn) var delivered webhook.Event if err := json.Unmarshal(body, &delivered); err != nil { diff --git a/acceleration/internal/scheduling/scheduler.go b/acceleration/internal/scheduling/scheduler.go index fc1f8c8266..739120af16 100644 --- a/acceleration/internal/scheduling/scheduler.go +++ b/acceleration/internal/scheduling/scheduler.go @@ -1,6 +1,7 @@ package scheduling import ( + "cmp" "context" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "sync" "time" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/store" ) @@ -188,6 +190,8 @@ type Scheduler struct { store *store.Store deliver DeliverFunc logger *slog.Logger + // activity counts each occurrence settled under its task's app. Nil counts none. + activity *activity.Recorder now func() time.Time lease time.Duration @@ -225,6 +229,12 @@ func NewScheduler(db *store.Store, deliver DeliverFunc, logger *slog.Logger) *Sc } } +// SetActivity has each occurrence the worker settles counted in record, as area scheduling. Call +// it before Start. +func (s *Scheduler) SetActivity(record *activity.Recorder) { + s.activity = record +} + // Start runs the worker until ctx ends or Close: it looks at start, when Kick // wakes it, at the next task due, and at least once a lease. Once ctx ends it // claims nothing more, and the deliveries in flight run on until they finish @@ -601,8 +611,31 @@ func (s *Scheduler) finish(claim store.ClaimedScheduledTask, outcome store.Sched log := s.logger.With("task", claim.Task.ID, "run", claim.Run.ID, "customer", claim.Task.CustomerID) switch { case err != nil: + // The run is left running, and whoever takes it up again settles and counts it. log.Error("could not record a scheduled task run", "error", err) + return case !applied: log.Info("a scheduled task changed under its run; the schedule was left alone") } + s.count(claim, outcome) +} + +// count counts an occurrence settled under claim in the activity record, once and not per +// attempt: delivered, or refused with the reason its run failed under, or missed. A run settled +// before claim took it was counted by whoever settled it, unless the claim itself failed it, its +// workers having been lost. +func (s *Scheduler) count(claim store.ClaimedScheduledTask, outcome store.ScheduledTaskOutcome) { + if claim.AlreadySettled && !claim.WorkersLost { + return + } + request := activity.Request{ + AppID: claim.Task.CustomerID, + Area: activity.AreaScheduling, + Client: activity.ClientRouter, + At: s.now(), + } + if outcome.RunState != store.ScheduledTaskRunDelivered { + request.Refused, request.Code = true, cmp.Or(outcome.Error, outcome.RunState) + } + s.activity.Record(request) } diff --git a/acceleration/internal/simulation/simulation.go b/acceleration/internal/simulation/simulation.go index eb9e45058e..7fe5dec86e 100644 --- a/acceleration/internal/simulation/simulation.go +++ b/acceleration/internal/simulation/simulation.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/llmrouter" "github.com/GetStream/Vision-Agents/acceleration/internal/routing" "github.com/GetStream/Vision-Agents/acceleration/internal/session" @@ -70,9 +71,11 @@ type Options struct { LLM *llmrouter.Router // TTS and STT give the simulated caller a voice and ears. Absent means text // simulations only, and an audio one is refused rather than quietly held in writing. - TTS *ttsrouter.Router - STT *sttrouter.Router - Logger *slog.Logger + TTS *ttsrouter.Router + STT *sttrouter.Router + // Activity counts each run that ends under its app, as area evals. Nil counts none. + Activity *activity.Recorder + Logger *slog.Logger } // Runner works through the simulation runs that have been started. @@ -82,6 +85,7 @@ type Runner struct { llm *llmrouter.Router tts *ttsrouter.Router stt *sttrouter.Router + activity *activity.Recorder logger *slog.Logger mu sync.Mutex @@ -118,6 +122,7 @@ func New(options Options) (*Runner, error) { llm: options.LLM, tts: options.TTS, stt: options.STT, + activity: options.Activity, logger: options.Logger, running: map[string]context.CancelFunc{}, stopKeeping: stop, @@ -461,7 +466,9 @@ func (r *Runner) finish(ctx context.Context, kase store.SimulationCase) { } } -// settle records how the run as a whole came out. +// settle records how the run as a whole came out, and counts it in the activity record: refused +// when it errored, which is the router not getting a ruling on every conversation. A run the judge +// failed is still a ruling, and one cancelled was stopped before it had them all. func (r *Runner) settle(ctx context.Context, run store.SimulationRun) { written, cancel := context.WithTimeout(context.WithoutCancel(ctx), writeTimeout) defer cancel() @@ -469,6 +476,11 @@ func (r *Runner) settle(ctx context.Context, run store.SimulationRun) { if err := r.store.FinishSimulationRun(written, run); err != nil { r.logger.Error("could not record how a run ended", "run", run.ID, "error", err) } + request := activity.Request{AppID: run.CustomerID, Area: activity.AreaEvals, Client: activity.ClientRouter, At: time.Now()} + if run.State == store.SimulationErrored { + request.Refused, request.Code = true, store.SimulationErrored + } + r.activity.Record(request) } // keep renews the runs this router holds, stops the ones it was asked to stop from From 9a45f2b0db5ea1ee6fe3c1d415367e6ae00b8e6e Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:11:24 +0100 Subject: [PATCH 02/11] [AI-3334] Test scheduled occurrence counting --- acceleration/internal/activity/activity.go | 6 +- .../scheduling/scheduler_activity_test.go | 152 ++++++++++++++++++ 2 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 acceleration/internal/scheduling/scheduler_activity_test.go diff --git a/acceleration/internal/activity/activity.go b/acceleration/internal/activity/activity.go index 57bd511c4d..d0473448c3 100644 --- a/acceleration/internal/activity/activity.go +++ b/acceleration/internal/activity/activity.go @@ -3,9 +3,9 @@ // // Every authenticated request is one, and so is a hook checked as an app's (NoteCustomer), and // the work the router does for an app on its own: a forward to its endpoint, a scheduled task's -// occurrence, a campaign's dial and a simulation run. So the counting is in memory and the writing is not on -// the request path: a minute's requests for one app, area, client and day are one row's -// update. Nothing here fails a request: counting it is not what the caller asked for. +// occurrence, a campaign's dial and a simulation run. So the counting is in memory and the +// writing is not on the request path: a minute's requests for one app, area, client and day are +// one row's update. Nothing here fails a request: counting it is not what the caller asked for. package activity import ( diff --git a/acceleration/internal/scheduling/scheduler_activity_test.go b/acceleration/internal/scheduling/scheduler_activity_test.go new file mode 100644 index 0000000000..2bed5c63b7 --- /dev/null +++ b/acceleration/internal/scheduling/scheduler_activity_test.go @@ -0,0 +1,152 @@ +//go:build integration + +package scheduling + +import ( + "database/sql" + "errors" + "log/slog" + "time" + + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" + "github.com/GetStream/Vision-Agents/acceleration/internal/store" +) + +// occurrencesCounted is a task's app's scheduling row in the activity record. +type occurrencesCounted struct { + requests, refused int64 + code string +} + +// counting starts a scheduler over deliver, as started does, that counts in an activity record +// of its own, which writes only when the test calls the stop it returns. +func (s *SchedulerSuite) counting(deliver DeliverFunc, tune func(*Scheduler)) (stop func()) { + record, err := activity.New(activity.Options{Store: s.db, Interval: time.Hour, Logger: slog.New(slog.DiscardHandler)}) + s.Require().NoError(err) + scheduler := s.started(deliver, func(scheduler *Scheduler) { + scheduler.SetActivity(record) + if tune != nil { + tune(scheduler) + } + }) + // The scheduler first, so an occurrence it is settling is counted before the record writes. + return func() { + scheduler.Close() + record.Close() + } +} + +// counted is what the activity record holds for task's app under area scheduling and client api. +func (s *SchedulerSuite) counted(task store.ScheduledTask) occurrencesCounted { + var ( + row occurrencesCounted + code sql.NullString + ) + err := s.db.DB().QueryRowContext(s.ctx, ` +SELECT COALESCE(sum(requests), 0), COALESCE(sum(refused), 0), max(last_refusal_code) +FROM customer_activity WHERE app_id = ? AND area = ? AND client = ?`, + task.CustomerID, activity.AreaScheduling, activity.ClientRouter).Scan(&row.requests, &row.refused, &code) + s.Require().NoError(err) + row.code = code.String + return row +} + +// Defect: a delivered occurrence left out of its app's scheduling work, or counted refused. +func (s *SchedulerSuite) TestADeliveredOccurrenceIsCountedOnceUnderItsApp() { + task := s.task(time.Now().UTC().Add(-time.Hour).Truncate(time.Second), nil) + stop := s.counting((&deliverer{}).deliver, nil) + + s.Require().Equal(store.ScheduledTaskRunDelivered, s.settled(task)[0].State) + stop() + + s.Equal(occurrencesCounted{requests: 1}, s.counted(task)) +} + +// Defect: each attempt counted, so an occurrence tried three times reads as three, or a failed +// one counted as delivered, or refused without the reason its run failed under. +func (s *SchedulerSuite) TestAFailedOccurrenceIsCountedOnceRefusedUnderItsReason() { + task := s.task(time.Now().UTC().Add(-time.Hour).Truncate(time.Second), nil) + deliver := &deliverer{err: &Failure{Reason: ReasonNoNumber, Err: errors.New("no number to text")}, only: task.ID} + stop := s.counting(deliver.deliver, func(scheduler *Scheduler) { + scheduler.retries = []time.Duration{time.Millisecond, time.Millisecond} + }) + + s.Require().Equal(store.ScheduledTaskRunFailed, s.settled(task)[0].State) + stop() + + s.Require().Len(deliver.callsFor(task.ID), 3, "the first attempt and both retries") + s.Equal(occurrencesCounted{requests: 1, refused: 1, code: ReasonNoNumber}, s.counted(task)) +} + +// Defect: an occurrence too late to deliver counted as delivered, so a router that was down at +// 09:00 shows the reminder as sent. +func (s *SchedulerSuite) TestAMissedOccurrenceIsCountedRefusedAsMissed() { + deliver := &deliverer{} + task := s.task(time.Now().UTC().Add(-25*time.Hour).Truncate(time.Second), nil) + stop := s.counting(deliver.deliver, nil) + + s.Require().Equal(store.ScheduledTaskRunMissed, s.settled(task)[0].State) + stop() + + s.Require().Empty(deliver.callsFor(task.ID)) + s.Equal(occurrencesCounted{requests: 1, refused: 1, code: store.ScheduledTaskRunMissed}, s.counted(task)) +} + +// Defect: a quiet-hours hold counted as an occurrence, so a reminder held and then delivered +// reads as two, one of them refused. +func (s *SchedulerSuite) TestAHeldOccurrenceIsCountedOnceWhenItIsDelivered() { + task := s.task(time.Now().UTC().Add(-time.Minute).Truncate(time.Second), nil) + deliver := &holdDeliverer{hold: 300 * time.Millisecond, task: task.ID} + stop := s.counting(deliver.deliver, nil) + + s.Require().Eventually(func() bool { + return s.read(task).State == store.ScheduledTaskCompleted + }, 10*time.Second, 10*time.Millisecond) + stop() + + s.Require().Len(deliver.calls, 2, "held once, then delivered") + s.Equal(occurrencesCounted{requests: 1}, s.counted(task)) +} + +// Defect: a run settled before its advance was lost counted again by the worker that only +// advances the task, so the occurrence reads as two. +func (s *SchedulerSuite) TestAnOccurrenceSettledBeforeItsAdvanceIsNotCountedAgain() { + due := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + task := s.task(due, nil) + started := due.Add(-time.Minute) + run := store.ScheduledTaskRun{ + CustomerID: task.CustomerID, TaskID: task.ID, ScheduledFor: due, + Trigger: store.ScheduledTaskTriggerSchedule, Prompt: task.Instructions, + State: store.ScheduledTaskRunDelivered, DeliveredTo: "conversation", + Attempts: 1, StartedAt: &started, FinishedAt: &due, + } + s.Require().NoError(s.db.InsertScheduledTaskRun(s.ctx, &run)) + stop := s.counting((&deliverer{}).deliver, nil) + + s.Require().Eventually(func() bool { + return s.read(task).State == store.ScheduledTaskCompleted + }, 10*time.Second, 10*time.Millisecond) + stop() + + s.Equal(occurrencesCounted{}, s.counted(task)) +} + +// Defect: a run failed because each of its workers was lost skipped as settled already, though +// no worker that settled it was left to count it. +func (s *SchedulerSuite) TestAnOccurrenceWhoseWorkersKeepDyingIsCountedRefused() { + due := time.Now().UTC().Add(-time.Minute).Truncate(time.Second) + task := s.task(due, nil) + // Three workers took it up and died: the store's cap. + started := time.Now().Add(-time.Hour) + stale := store.ScheduledTaskRun{TaskID: task.ID, CustomerID: task.CustomerID, ScheduledFor: due, + Attempts: 3, StartedAt: &started} + s.Require().NoError(s.db.InsertScheduledTaskRun(s.ctx, &stale)) + stop := s.counting((&deliverer{}).deliver, nil) + + s.Require().Eventually(func() bool { + return s.read(task).State == store.ScheduledTaskFailed + }, 10*time.Second, 10*time.Millisecond) + stop() + + s.Equal(occurrencesCounted{requests: 1, refused: 1, code: ReasonFailed}, s.counted(task)) +} From 08b686d55945ad4dd58e1c8fe2a963196951b2d3 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:12:16 +0100 Subject: [PATCH 03/11] [AI-3334] Test campaign dial counting --- .../internal/campaign/runner_activity_test.go | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 acceleration/internal/campaign/runner_activity_test.go diff --git a/acceleration/internal/campaign/runner_activity_test.go b/acceleration/internal/campaign/runner_activity_test.go new file mode 100644 index 0000000000..1477f9657d --- /dev/null +++ b/acceleration/internal/campaign/runner_activity_test.go @@ -0,0 +1,138 @@ +//go:build integration + +package campaign + +import ( + "context" + "database/sql" + "errors" + "log/slog" + "time" + + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" +) + +// dialsCounted is the campaign's app's phone_sms row in the activity record. +type dialsCounted struct { + requests, refused int64 + code string +} + +// failingDialer could not try any dial. +type failingDialer struct{} + +func (failingDialer) Dial(context.Context, Dial) (Dialed, error) { + return Dialed{}, errors.New("the vendor did not answer") +} + +// counting is a runner over dialer, as newRunner makes, that counts in an activity record of its +// own, which writes only when the test calls the write it returns. +func (s *RunnerSuite) counting(dialer Dialer) (runner *Runner, write func()) { + record, err := activity.New(activity.Options{Store: s.store, Interval: time.Hour, Logger: slog.New(slog.DiscardHandler)}) + s.Require().NoError(err) + runner, err = New(Options{ + Store: s.store, Dialer: dialer, Transcripts: s.transcripts, Classifier: s.classifier, + Owner: "router-counting", Now: s.clock.now, Activity: record, Logger: slog.New(slog.DiscardHandler), + }) + s.Require().NoError(err) + s.T().Cleanup(runner.Close) + return runner, record.Close +} + +// counted is what the activity record holds for the suite's app under phone_sms and client api. +func (s *RunnerSuite) counted() dialsCounted { + var ( + row dialsCounted + code sql.NullString + ) + err := s.store.DB().QueryRowContext(s.ctx, ` +SELECT COALESCE(sum(requests), 0), COALESCE(sum(refused), 0), max(last_refusal_code) +FROM customer_activity WHERE app_id = ? AND area = ? AND client = ?`, + s.customerID, activity.AreaPhoneSMS, activity.ClientRouter).Scan(&row.requests, &row.refused, &code) + s.Require().NoError(err) + row.code = code.String + return row +} + +// Defect: a call nobody answered counted refused, or the second dial of a double dial, which +// rings the person again, left uncounted. +func (s *RunnerSuite) TestEachDialOfADoubleDialIsCountedAndARingNobodyAnsweredIsNotRefused() { + runner, write := s.counting(s.dialer) + s.launched(DefaultSequence("config-1"), 1, "UTC", "+15550000001") + + s.tick(runner) + write() + + s.Require().Len(s.dialer.dialed(), 2) + s.Equal(dialsCounted{requests: 2}, s.counted()) +} + +// Defect: a call an answering machine took, which the dial placed, counted refused. +func (s *RunnerSuite) TestADialAMachineAnsweredIsNotCountedRefused() { + s.dialer.results["+15550000001"] = []Dialed{{Result: Voicemail}} + runner, write := s.counting(s.dialer) + s.launched(s.single(), 1, "UTC", "+15550000001") + + s.tick(runner) + write() + + s.Equal(dialsCounted{requests: 1}, s.counted()) +} + +// Defect: a dial refused for money, which placed no call, counted as placed, or refused without +// the reason the campaign paused under. +func (s *RunnerSuite) TestADialRefusedForMoneyIsCountedRefusedUnderItsReason() { + s.dialer.results["+15550000001"] = []Dialed{{Result: Paused, Reason: "insufficient_credits"}} + runner, write := s.counting(s.dialer) + s.launched(s.single(), 1, "UTC", "+15550000001") + + s.tick(runner) + write() + + s.Equal(dialsCounted{requests: 1, refused: 1, code: "insufficient_credits"}, s.counted()) +} + +// Defect: a dial that placed no call and gave no reason counted refused with no code, so the +// digest cannot say why. +func (s *RunnerSuite) TestADialToANumberThatCannotBeDialedIsCountedRefusedUnderItsResult() { + s.dialer.results["+15550000001"] = []Dialed{{Result: InvalidNumber}} + runner, write := s.counting(s.dialer) + s.launched(s.single(), 1, "UTC", "+15550000001") + + s.tick(runner) + write() + + s.Equal(dialsCounted{requests: 1, refused: 1, code: string(InvalidNumber)}, s.counted()) +} + +// Defect: a dial that could not be tried, which may have rung nobody, counted as a call placed. +func (s *RunnerSuite) TestADialThatCouldNotBeTriedIsCountedRefused() { + runner, write := s.counting(failingDialer{}) + s.launched(s.single(), 1, "UTC", "+15550000001") + + s.tick(runner) + write() + + s.Equal(dialsCounted{requests: 1, refused: 1, code: codeDialFailed}, s.counted()) +} + +// Defect: a dial a stopping router never placed counted refused, so the contact, rung by +// another router later, reads as two dials, one of them refused. +func (s *RunnerSuite) TestADialTheRouterDidNotPlaceAsItStopsIsNotCounted() { + s.launched(s.single(), 1, "UTC", "+15550000001") + runner, write := s.counting(s.dialer) + stopped, stop := context.WithCancel(s.ctx) + stop() + s.Require().NoError(runner.Start(stopped)) + s.Require().Eventually(runner.stopping, 5*time.Second, 10*time.Millisecond) + claimed, err := s.store.ClaimDueContacts(s.ctx, "router-counting", s.clock.now(), lease, 10) + s.Require().NoError(err) + s.Require().Len(claimed, 1) + + dialed, err := runner.ring(s.ctx, claimed[0].Campaign, claimed[0].Contact, s.single().First(), 1) + s.Require().NoError(err) + write() + + s.Require().Equal(NotPlaced, dialed.Result) + s.Equal(dialsCounted{}, s.counted()) +} From 34a2bf7dde96c75948886807580bc9fbc80e6c84 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:13:15 +0100 Subject: [PATCH 04/11] [AI-3334] Test simulation run counting --- .../internal/simulation/activity_test.go | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 acceleration/internal/simulation/activity_test.go diff --git a/acceleration/internal/simulation/activity_test.go b/acceleration/internal/simulation/activity_test.go new file mode 100644 index 0000000000..b78889d3da --- /dev/null +++ b/acceleration/internal/simulation/activity_test.go @@ -0,0 +1,88 @@ +//go:build integration + +package simulation + +import ( + "context" + "crypto/rand" + "database/sql" + "log/slog" + "os" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" + "github.com/GetStream/Vision-Agents/acceleration/internal/store" + "github.com/GetStream/Vision-Agents/acceleration/internal/testenv" +) + +// ActivitySuite covers how a run that ends is counted in the activity record. +type ActivitySuite struct { + suite.Suite + ctx context.Context + db *store.Store +} + +func TestActivitySuite(t *testing.T) { + suite.Run(t, new(ActivitySuite)) +} + +func (s *ActivitySuite) SetupSuite() { + dsn := os.Getenv("ROUTER_POSTGRES_DSN") + if dsn == "" { + s.T().Skip("ROUTER_POSTGRES_DSN must be set") + } + s.ctx = context.Background() + db, err := store.Open(testenv.Database(dsn, "simulation")) + s.Require().NoError(err) + s.Require().NoError(db.Migrate(s.ctx)) + s.db = db + s.T().Cleanup(func() { s.Require().NoError(db.Close()) }) +} + +// runsCounted is an app's evals row in the activity record. +type runsCounted struct { + requests, refused int64 + code string +} + +// settled settles a run of an app of its own that ended in state, with a runner that counts in +// an activity record of its own, and returns what the record wrote for the app. +func (s *ActivitySuite) settled(state string) runsCounted { + record, err := activity.New(activity.Options{Store: s.db, Interval: time.Hour, Logger: slog.New(slog.DiscardHandler)}) + s.Require().NoError(err) + runner := &Runner{store: s.db, activity: record, logger: slog.New(slog.DiscardHandler)} + run := store.SimulationRun{ID: "run-" + rand.Text(), CustomerID: "app-" + rand.Text(), State: state, Cases: 1} + + runner.settle(s.ctx, run) + record.Close() + + var ( + row runsCounted + code sql.NullString + ) + s.Require().NoError(s.db.DB().QueryRowContext(s.ctx, ` +SELECT COALESCE(sum(requests), 0), COALESCE(sum(refused), 0), max(last_refusal_code) +FROM customer_activity WHERE app_id = ? AND area = ? AND client = ?`, + run.CustomerID, activity.AreaEvals, activity.ClientRouter).Scan(&row.requests, &row.refused, &code)) + row.code = code.String + return row +} + +// Defect: a run the router could not get a ruling on counted as a run that worked, or left out. +func (s *ActivitySuite) TestARunThatErroredIsCountedRefused() { + s.Equal(runsCounted{requests: 1, refused: 1, code: store.SimulationErrored}, s.settled(store.SimulationErrored)) +} + +// Defect: a run the judge ruled against counted as the router refusing it: the agent failed the +// check, and the router did what it was asked. +func (s *ActivitySuite) TestARunTheJudgeFailedIsCountedAndNotRefused() { + s.Equal(runsCounted{requests: 1}, s.settled(store.SimulationFailed)) +} + +// Defect: a run stopped before it had every ruling counted refused. +func (s *ActivitySuite) TestARunThatWasCancelledIsCountedAndNotRefused() { + s.Equal(runsCounted{requests: 1}, s.settled(store.SimulationCancelled)) +} From 1b1903279fcf644d9e278ac3f2e217670031d958 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:15:07 +0100 Subject: [PATCH 05/11] [AI-3334] Test MCP event and 10DLC report counting --- .../internal/api/hook_activity_test.go | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/acceleration/internal/api/hook_activity_test.go b/acceleration/internal/api/hook_activity_test.go index 1f83077d1a..e163bb662d 100644 --- a/acceleration/internal/api/hook_activity_test.go +++ b/acceleration/internal/api/hook_activity_test.go @@ -4,6 +4,8 @@ package api import ( "context" + "crypto/ed25519" + "crypto/rand" "net/http" "net/url" "time" @@ -256,3 +258,76 @@ func (s *TelnyxHookSuite) TestACallWhoseTrunkCannotBeRenderedIsCountedRefused() s.Equal(int64(1), row.refused) s.Equal(internalError().Code, row.code) } + +// activityWritten waits until what the suite's activity record holds now is written: a row +// recorded after it is written in the same write, one transaction, or a later one. +func (s *RouterSuite) activityWritten() { + marker := "written-" + rand.Text() + s.activity.Record(activity.Request{AppID: s.customerID(), Area: marker, Client: store.AuditSourceAPI, At: time.Now()}) + s.Require().Eventually(func() bool { return s.counted(marker, store.AuditSourceAPI).requests == 1 }, + settleFor, 10*time.Millisecond, "the activity record was not written") +} + +// hooksAfter delivers, and returns the app's hooks_in row before and after it. +func (s *RouterSuite) hooksAfter(deliver func()) (before, after counted) { + s.activityWritten() + before = s.counted(activity.AreaHooksIn, store.AuditSourceAPI) + deliver() + s.activityWritten() + return before, s.counted(activity.AreaHooksIn, store.AuditSourceAPI) +} + +// Defect: an event the app's MCP server signed with the subscription's secret left out of the +// app's hooks, or one signed with another subscription's secret counted under the app. +func (s *ConnectionEventsSuite) TestASignedMCPEventIsCountedAsTheAppsHookAndAForgedOneIsNot() { + connection := s.connection() + binding := s.binding(connection, issueCreated) + binding["events"] = append(binding["events"].([]map[string]any), map[string]any{"event": "issue.closed"}) + s.subscribed(binding) + subs := s.atTheFake(connection) + s.Require().Len(subs, 2) + + before, after := s.hooksAfter(func() { + s.Require().Equal(http.StatusUnauthorized, s.provider.DeliverEvent(subs[0].URL, subs[1].Secret, subs[0].ID, + "evt_"+s.utils.uuid(), subs[0].Name, map[string]any{})) + s.Require().Equal(http.StatusAccepted, s.provider.DeliverEvent(subs[0].URL, subs[0].Secret, subs[0].ID, + "evt_"+s.utils.uuid(), subs[0].Name, map[string]any{})) + }) + + s.Equal(before.requests+1, after.requests) + s.Equal(before.refused, after.refused) +} + +// Defect: a signed delivery the router then turned down left out of the app's hooks, so the +// digest shows none of the app's failing MCP events. +func (s *ConnectionEventsSuite) TestASignedMCPEventTheRouterTurnsDownIsCountedRefused() { + connection := s.subscribed(s.binding(s.connection(), issueCreated)) + sub := s.atTheFake(connection)[0] + + before, after := s.hooksAfter(func() { + s.Require().Equal(http.StatusBadRequest, s.provider.DeliverEvent(sub.URL, sub.Secret, sub.ID, + "evt_"+s.utils.uuid(), "issue.closed", map[string]any{})) + }) + + s.Equal(before.requests+1, after.requests) + s.Equal(before.refused+1, after.refused) +} + +// Defect: a report the registrar signed about an app's campaign left out of the app's phone +// work, or one signed with another key counted. +func (s *DLCSuite) TestASignedReportOnACampaignIsCountedAsItsAppsPhoneWork() { + _, campaign := s.pendingWithTheVendor() + _, stranger, err := ed25519.GenerateKey(rand.Reader) + s.Require().NoError(err) + s.activityWritten() + before := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI) + + s.Require().Equal(http.StatusUnauthorized, s.report(`{"campaignId":"`+campaign+`"}`, stranger)) + s.Require().Equal(http.StatusNoContent, s.report(`{"campaignId":"`+campaign+`"}`, s.signing)) + s.activityWritten() + + after := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI) + s.Equal(before.requests+1, after.requests) + s.Equal(before.refused, after.refused) + s.Equal(int64(0), s.counted(activity.AreaHooksIn, store.AuditSourceAPI).requests, "a report is phone work") +} From 2a5f2239267924b9e6434ad1d45ac1471d50b3c6 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:27:20 +0100 Subject: [PATCH 06/11] [AI-3334] Wrap the ClientRouter comment --- acceleration/internal/activity/activity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceleration/internal/activity/activity.go b/acceleration/internal/activity/activity.go index d0473448c3..af8499352a 100644 --- a/acceleration/internal/activity/activity.go +++ b/acceleration/internal/activity/activity.go @@ -41,8 +41,8 @@ const ( ) // ClientRouter is the client work the router starts itself is counted under, such as a forward to -// a customer's own endpoint or a scheduled task's occurrence. It is the API's, the client a caller that named none is counted -// under, so no row says a client sent what nobody sent. +// a customer's own endpoint or a scheduled task's occurrence. It is the API's, the client a +// caller that named none is counted under, so no row says a client sent what nobody sent. const ClientRouter = store.AuditSourceAPI // Note is who a request no credential named was for: a hook a vendor or Stream signed carries From 6c53e3e5c75bd7db2292186b657f2654af02d7c7 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:28:01 +0100 Subject: [PATCH 07/11] [AI-3334] CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c78378c0bd..3cdfd1c7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Features +### The router counts its own work for an app + +A scheduled task's occurrence (`scheduling`), a campaign's dial (`phone_sms`) and a simulation run (`evals`) are counted in the activity record under their app, once each, refused with why when the occurrence failed or was missed, the dial placed no call or the run errored. A signed MCP connection event counts under `hooks_in` and a 10DLC campaign report under `phone_sms` (#2071). + ### The router counts hooks, forwards and socket refusals under the app A hook Stream, Telnyx or a channel's provider signed is counted in the activity record under its app and organization once its signature or token checks out: `hooks_in`, or `phone_sms` for phone and SMS. A forward to an event destination counts once under `webhooks_out`, refused with why when it is given up, and a `/v1/agents/socket` start refused in a frame counts as refused with its code rather than as answered (#2066). From efdaa0d68696e1532bfbdd96ce76df0081dbc908 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 13:55:35 +0100 Subject: [PATCH 08/11] AI-3334: Test a fenced settle, an unwritten run, a non-JSON MCP event and a failed 10DLC report Co-Authored-By: Claude Opus 5.5 --- acceleration/internal/api/dlc_test.go | 8 +++ .../internal/api/hook_activity_test.go | 58 +++++++++++++++++++ .../scheduling/scheduler_activity_test.go | 52 +++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/acceleration/internal/api/dlc_test.go b/acceleration/internal/api/dlc_test.go index 0cf61a8139..616b99982b 100644 --- a/acceleration/internal/api/dlc_test.go +++ b/acceleration/internal/api/dlc_test.go @@ -37,6 +37,10 @@ type campaignRegistry struct { assigned map[string]string } +// campaignRegistryFails is a campaign status no vendor sends: the fake answers a read of a +// campaign set to it with a 500. +const campaignRegistryFails = "REGISTRY_FAILS" + func (r *campaignRegistry) ServeHTTP(w http.ResponseWriter, request *http.Request) { r.mu.Lock() defer r.mu.Unlock() @@ -55,6 +59,10 @@ func (r *campaignRegistry) ServeHTTP(w http.ResponseWriter, request *http.Reques fmt.Fprintf(w, `{"campaignId":%q,"campaignStatus":"TCR_PENDING"}`, id) case request.Method == http.MethodGet && strings.HasPrefix(path, "/v2/10dlc/campaign/"): id := strings.TrimPrefix(path, "/v2/10dlc/campaign/") + if r.campaigns[id] == campaignRegistryFails { + w.WriteHeader(http.StatusInternalServerError) + return + } fmt.Fprintf(w, `{"campaignId":%q,"campaignStatus":%q}`, id, r.campaigns[id]) case request.Method == http.MethodPost && path == "/v2/10dlc/phone_number_campaigns": r.assigned[body["phoneNumber"]] = body["campaignId"] diff --git a/acceleration/internal/api/hook_activity_test.go b/acceleration/internal/api/hook_activity_test.go index e163bb662d..45b2a8f713 100644 --- a/acceleration/internal/api/hook_activity_test.go +++ b/acceleration/internal/api/hook_activity_test.go @@ -3,11 +3,17 @@ package api import ( + "bytes" "context" "crypto/ed25519" + "crypto/hmac" "crypto/rand" + "crypto/sha256" + "encoding/base64" "net/http" "net/url" + "strconv" + "strings" "time" "github.com/GetStream/Vision-Agents/acceleration/internal/activity" @@ -331,3 +337,55 @@ func (s *DLCSuite) TestASignedReportOnACampaignIsCountedAsItsAppsPhoneWork() { s.Equal(before.refused, after.refused) s.Equal(int64(0), s.counted(activity.AreaHooksIn, store.AuditSourceAPI).requests, "a report is phone work") } + +// Defect: a signed delivery that is not JSON, answered 400, left out of the app's hooks. +func (s *ConnectionEventsSuite) TestASignedDeliveryThatIsNotJSONIsCountedRefused() { + connection := s.subscribed(s.binding(s.connection(), issueCreated)) + sub := s.atTheFake(connection)[0] + + before, after := s.hooksAfter(func() { + s.Require().Equal(http.StatusBadRequest, s.signedRaw(sub.URL, sub.Secret, sub.ID, []byte("not json"))) + }) + + s.Equal(before.requests+1, after.requests) + s.Equal(before.refused+1, after.refused) +} + +// signedRaw posts body signed with secret, as the MCP server signs a delivery, and returns the +// status the router answered. +func (s *ConnectionEventsSuite) signedRaw(url, secret, subscriptionID string, body []byte) int { + key, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(secret, "whsec_")) + s.Require().NoError(err) + id := "msg_" + s.utils.uuid() + stamp := strconv.FormatInt(time.Now().Unix(), 10) + mac := hmac.New(sha256.New, key) + mac.Write([]byte(id + "." + stamp + ".")) + mac.Write(body) + request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + s.Require().NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("webhook-id", id) + request.Header.Set("webhook-timestamp", stamp) + request.Header.Set("webhook-signature", "v1,"+base64.StdEncoding.EncodeToString(mac.Sum(nil))) + request.Header.Set("X-MCP-Subscription-Id", subscriptionID) + response, err := http.DefaultClient.Do(request) + s.Require().NoError(err) + _ = response.Body.Close() + return response.StatusCode +} + +// Defect: a signed report on an app's campaign the router failed to act on (5xx) left out of +// the app's phone work, so the digest shows none of its failing 10DLC reports. +func (s *DLCSuite) TestASignedReportTheRouterFailsOnIsCountedRefused() { + _, campaign := s.pendingWithTheVendor() + s.registry.set(campaign, campaignRegistryFails) + s.activityWritten() + before := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI) + + s.Require().GreaterOrEqual(s.report(`{"campaignId":"`+campaign+`"}`, s.signing), http.StatusInternalServerError) + s.activityWritten() + + after := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI) + s.Equal(before.requests+1, after.requests) + s.Equal(before.refused+1, after.refused) +} diff --git a/acceleration/internal/scheduling/scheduler_activity_test.go b/acceleration/internal/scheduling/scheduler_activity_test.go index 2bed5c63b7..3b5bd8dbd8 100644 --- a/acceleration/internal/scheduling/scheduler_activity_test.go +++ b/acceleration/internal/scheduling/scheduler_activity_test.go @@ -3,6 +3,7 @@ package scheduling import ( + "context" "database/sql" "errors" "log/slog" @@ -150,3 +151,54 @@ func (s *SchedulerSuite) TestAnOccurrenceWhoseWorkersKeepDyingIsCountedRefused() s.Equal(occurrencesCounted{requests: 1, refused: 1, code: ReasonFailed}, s.counted(task)) } + +// Defect: an occurrence settled under a claim an edit fenced (the run recorded delivered, the +// schedule left alone) left uncounted. +func (s *SchedulerSuite) TestAnOccurrenceSettledUnderAFencedClaimIsCounted() { + task := s.task(time.Now().UTC().Add(-time.Hour).Truncate(time.Second), nil) + fence := func(ctx context.Context, claimed store.ClaimedScheduledTask) (string, string, error) { + if claimed.Task.ID == task.ID { + // An edit during the delivery takes the claim away. + _, err := s.db.DB().ExecContext(ctx, `UPDATE scheduled_tasks SET claim_id = 'fenced' WHERE id = ?`, task.ID) + s.Require().NoError(err) + } + return "conversation", "", nil + } + stop := s.counting(fence, nil) + + s.Require().Eventually(func() bool { + runs := s.runs(task) + return len(runs) == 1 && runs[0].State == store.ScheduledTaskRunDelivered + }, 10*time.Second, 10*time.Millisecond) + stop() + + s.Equal(occurrencesCounted{requests: 1}, s.counted(task)) +} + +// Defect: an occurrence whose run could not be written counted now, and again by the claim that +// takes it up and settles it. +func (s *SchedulerSuite) TestAnOccurrenceWhoseRunWasNotWrittenIsNotCounted() { + task := s.task(time.Now().UTC().Add(-time.Hour).Truncate(time.Second), nil) + delivered := make(chan struct{}, 1) + lose := func(ctx context.Context, claimed store.ClaimedScheduledTask) (string, string, error) { + if claimed.Task.ID == task.ID { + // The run row gone, so FinishScheduledTaskRun's read of it fails. + _, err := s.db.DB().ExecContext(ctx, `DELETE FROM scheduled_task_runs WHERE id = ?`, claimed.Run.ID) + s.Require().NoError(err) + delivered <- struct{}{} + } + return "conversation", "", nil + } + var scheduler *Scheduler + stop := s.counting(lose, func(started *Scheduler) { scheduler = started }) + + select { + case <-delivered: + case <-time.After(10 * time.Second): + s.FailNow("the occurrence was never delivered") + } + s.Require().Eventually(func() bool { return scheduler.Running() == 0 }, 10*time.Second, 10*time.Millisecond) + stop() + + s.Equal(occurrencesCounted{}, s.counted(task)) +} From 42729c506ad9246cf52525b04453c57a642e0164 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 14:00:59 +0100 Subject: [PATCH 09/11] AI-3334: Count a scheduled task run by hand Co-Authored-By: Claude Opus 5.5 --- acceleration/internal/api/scheduled_tasks.go | 10 +++++ .../internal/api/scheduled_tasks_test.go | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/acceleration/internal/api/scheduled_tasks.go b/acceleration/internal/api/scheduled_tasks.go index e18fb2fbb6..7972e9ef56 100644 --- a/acceleration/internal/api/scheduled_tasks.go +++ b/acceleration/internal/api/scheduled_tasks.go @@ -1,6 +1,7 @@ package api import ( + "cmp" "context" "errors" "net/http" @@ -10,6 +11,7 @@ import ( "github.com/danielgtaylor/huma/v2" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/auth" "github.com/GetStream/Vision-Agents/acceleration/internal/routing" "github.com/GetStream/Vision-Agents/acceleration/internal/scheduling" @@ -622,6 +624,14 @@ func (s *Server) deliverManualRun(task store.ScheduledTask, run store.ScheduledT s.logger.Error("could not record a scheduled task run by hand", "task", task.ID, "run", run.ID, "error", err) } + // Counted here whether or not its run was written: no worker takes a run by hand up again. + request := activity.Request{ + AppID: task.CustomerID, Area: activity.AreaScheduling, Client: activity.ClientRouter, At: time.Now(), + } + if state != store.ScheduledTaskRunDelivered { + request.Refused, request.Code = true, cmp.Or(failure, state) + } + s.activity.Record(request) } func (s *Server) deleteScheduledTask(ctx context.Context, request *deleteScheduledTaskRequest) (*struct{}, error) { diff --git a/acceleration/internal/api/scheduled_tasks_test.go b/acceleration/internal/api/scheduled_tasks_test.go index aabefc58cb..e0ae3a6037 100644 --- a/acceleration/internal/api/scheduled_tasks_test.go +++ b/acceleration/internal/api/scheduled_tasks_test.go @@ -20,6 +20,7 @@ import ( "github.com/gorilla/websocket" "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/activity" "github.com/GetStream/Vision-Agents/acceleration/internal/auth" "github.com/GetStream/Vision-Agents/acceleration/internal/channels" "github.com/GetStream/Vision-Agents/acceleration/internal/conversation" @@ -3467,6 +3468,44 @@ func (s *ScheduledTasksSuite) TestARunByHandThatFailsListsTheReasonItFailedUnder s.Equal(scheduling.ReasonNoNumber, *page.Items[0].Error) } +// Defect: a run by hand that landed counted refused, or not at all past the request that set +// it going. +func (s *ScheduledTasksSuite) TestARunByHandThatLandsIsCountedDelivered() { + config := s.agentWithScheduling(nil) + created := s.createTask(s.data.createUser(), taskRequest(config)) + s.activityWritten() + before := s.counted(activity.AreaScheduling, activity.ClientRouter) + + s.Require().Equal(http.StatusAccepted, s.serverClient.do(http.MethodPost, "/v1/agents/scheduled-tasks/"+created.Id+"/run", nil, nil)) + + // The request that set it going, and the run. + s.Require().Eventually(func() bool { + return s.counted(activity.AreaScheduling, activity.ClientRouter).requests == before.requests+2 + }, settleFor, 10*time.Millisecond, "the run by hand was not counted") + s.Equal(before.refused, s.counted(activity.AreaScheduling, activity.ClientRouter).refused) +} + +// Defect: a run by hand that failed counted only as the request that set it going, so the +// app's scheduling work shows no refusal, or one without the reason its run failed under. +func (s *ScheduledTasksSuite) TestARunByHandThatFailsIsCountedRefusedUnderItsReason() { + config := s.agentWithScheduling(nil) + user := s.data.createUser() + opened := s.openScheduling(user, config, nil) + created := s.taskOn(user, config, opened.Id, []string{"phone"}) + s.activityWritten() + before := s.counted(activity.AreaScheduling, activity.ClientRouter) + + s.Require().Equal(http.StatusAccepted, s.serverClient.do(http.MethodPost, "/v1/agents/scheduled-tasks/"+created.Id+"/run", nil, nil)) + + s.Require().Eventually(func() bool { + return s.counted(activity.AreaScheduling, activity.ClientRouter).refused == before.refused+1 + }, settleFor, 10*time.Millisecond, "the failed run by hand was not counted refused") + after := s.counted(activity.AreaScheduling, activity.ClientRouter) + // The request that set it going, and the run. + s.Equal(before.requests+2, after.requests) + s.Equal(scheduling.ReasonNoNumber, after.code) +} + func (s *ScheduledTasksSuite) TestATaskForAnAgentWithoutANumberHasNoLineToRingFrom() { s.retryAtOnce() config := s.agentWithScheduling(nil) From f30cdb03d20a025566f65feee7469570d1dd3cd0 Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 14:05:40 +0100 Subject: [PATCH 10/11] AI-3334: Say a run failed for lost workers whose finish write fails is never counted Co-Authored-By: Claude Opus 5.5 --- acceleration/internal/scheduling/scheduler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/acceleration/internal/scheduling/scheduler.go b/acceleration/internal/scheduling/scheduler.go index 739120af16..c33e39a928 100644 --- a/acceleration/internal/scheduling/scheduler.go +++ b/acceleration/internal/scheduling/scheduler.go @@ -611,7 +611,9 @@ func (s *Scheduler) finish(claim store.ClaimedScheduledTask, outcome store.Sched log := s.logger.With("task", claim.Task.ID, "run", claim.Run.ID, "customer", claim.Task.CustomerID) switch { case err != nil: - // The run is left running, and whoever takes it up again settles and counts it. + // The run is left running, and whoever takes it up again settles and counts it. A + // run this claim failed itself, its workers lost, is not running: the next claim + // finds it settled by somebody else, and it is never counted. log.Error("could not record a scheduled task run", "error", err) return case !applied: From efbcb3978899a2dc6fd988690166ad9ca478951f Mon Sep 17 00:00:00 2001 From: Kanat Date: Wed, 18 Nov 2026 14:05:40 +0100 Subject: [PATCH 11/11] AI-3334: List the router's own work and MCP/10DLC reports in the ops activity counts Co-Authored-By: Claude Opus 5.5 --- acceleration/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/acceleration/README.md b/acceleration/README.md index d01eedb5dd..8de320455a 100644 --- a/acceleration/README.md +++ b/acceleration/README.md @@ -2156,7 +2156,11 @@ characters, time and cost), and `first_seen` and `last_seen` the first and last area. A hook counts under the app its signature or token was checked for, from client `api`: `phone_sms` for a phone or SMS vendor's, else `hooks_in`. A forward to an event destination counts once under `webhooks_out` from `api`, refused when given up. A socket refused in a frame -after its 101 counts as refused, with the code. +after its 101 counts as refused, with the code. Work the router starts itself also counts from +`api`, once each: a scheduled task's occurrence or run by hand under `scheduling` (refused when +missed or failed), a campaign dial under `phone_sms`, and a simulation run under `evals` +(refused when it errored). A signed MCP event counts under `hooks_in`, and a signed 10DLC report +on an app's campaign under `phone_sms`. `furthest_step` is the furthest of `dashboard_reads`, `agent_setup` and `conversations` with a request answered rather than refused, or `steady_use` once conversations or model calls