Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions CHANGELOG.md

Large diffs are not rendered by default.

6 changes: 5 additions & 1 deletion acceleration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2156,7 +2156,11 @@ characters, time and cost), and `first_seen` and `last_seen` the first and last
area. A hook counts under the app its signature or token was checked for, from client `api`:
`phone_sms` for a phone or SMS vendor's, else `hooks_in`. A forward to an event destination
counts once under `webhooks_out` from `api`, refused when given up. A socket refused in a frame
after its 101 counts as refused, with the code.
after its 101 counts as refused, with the code. Work the router starts itself also counts from
`api`, once each: a scheduled task's occurrence or run by hand under `scheduling` (refused when
missed or failed), a campaign dial under `phone_sms`, and a simulation run under `evals`
(refused when it errored). A signed MCP event counts under `hooks_in`, and a signed 10DLC report
on an app's campaign under `phone_sms`.

`furthest_step` is the furthest of `dashboard_reads`, `agent_setup`, `workers`, `conversations`,
`voice_video` and `phone_sms` (`activity.Steps`), in that order, with a request answered rather than refused, or `steady_use` once conversations or model calls
Expand Down
9 changes: 6 additions & 3 deletions acceleration/cmd/router/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -1238,9 +1238,10 @@ func run(settings config.Config, logger *slog.Logger) error {
LLM: streams.LLM,
// Speech is what an audio simulation needs and a text one does not, so it is
// passed where it exists rather than required.
TTS: streams.TTS,
STT: streams.STT,
Logger: logger,
TTS: streams.TTS,
STT: streams.STT,
Activity: activityRecord,
Logger: logger,
})
if err != nil {
return err
Expand Down Expand Up @@ -1624,6 +1625,7 @@ func run(settings config.Config, logger *slog.Logger) error {
var scheduler *scheduling.Scheduler
if pgStore != nil && sessions != nil {
scheduler = scheduling.NewScheduler(pgStore, server.DeliverScheduledTask, logger)
scheduler.SetActivity(activityRecord)
// Before Start: a session the first delivery opens takes the kick then.
server.SetSchedulerKick(scheduler.Kick)
scheduler.Start(ctx)
Expand All @@ -1638,6 +1640,7 @@ func run(settings config.Config, logger *slog.Logger) error {
Dialer: campaign.NewPhoneDialer(telephony, sessions, pgStore, logger),
Transcripts: server,
Classifier: campaign.NewClassifier(streams.LLM),
Activity: activityRecord,
Logger: logger,
})
if err != nil {
Expand Down
20 changes: 11 additions & 9 deletions acceleration/internal/activity/activity.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
// Package activity counts the requests each app makes, by area of the API and by client, and
// writes the sums to customer_activity (store.RecordActivity).
//
// Every authenticated request is one, and so is a hook checked as an app's (NoteCustomer) and a
// forward to an app's own endpoint, so the counting is in memory and the writing is not on
// the request path: a minute's requests for one app, area, client and day are one row's
// update. Nothing here fails a request: counting it is not what the caller asked for.
// Every authenticated request is one, and so is a hook checked as an app's (NoteCustomer), and
// the work the router does for an app on its own: a forward to its endpoint, a scheduled task's
// occurrence, a campaign's dial and a simulation run. So the counting is in memory and the
// writing is not on the request path: a minute's requests for one app, area, client and day are
// one row's update. Nothing here fails a request: counting it is not what the caller asked for.
package activity

import (
Expand Down Expand Up @@ -39,9 +40,9 @@ const (
AreaOther = "other"
)

// ClientRouter is the client work the router starts itself is counted under: a forward to a
// customer's own endpoint. It is the API's, the client a caller that named none is counted
// under, so no row says a client sent what nobody sent.
// ClientRouter is the client work the router starts itself is counted under, such as a forward to
// a customer's own endpoint or a scheduled task's occurrence. It is the API's, the client a
// caller that named none is counted under, so no row says a client sent what nobody sent.
const ClientRouter = store.AuditSourceAPI

// Note is who a request no credential named was for: a hook a vendor or Stream signed carries
Expand Down Expand Up @@ -189,9 +190,10 @@ func (r *Recorder) Close() {
})
}

// Record counts one request. A request naming no app is nobody's and is not counted.
// Record counts one request. A request naming no app is nobody's and is not counted, and a nil
// Recorder, a router with no database, counts nothing.
func (r *Recorder) Record(request Request) {
if request.AppID == "" {
if r == nil || request.AppID == "" {
return
}
at := request.At.UTC()
Expand Down
8 changes: 8 additions & 0 deletions acceleration/internal/api/dlc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ type campaignRegistry struct {
assigned map[string]string
}

// campaignRegistryFails is a campaign status no vendor sends: the fake answers a read of a
// campaign set to it with a 500.
const campaignRegistryFails = "REGISTRY_FAILS"

func (r *campaignRegistry) ServeHTTP(w http.ResponseWriter, request *http.Request) {
r.mu.Lock()
defer r.mu.Unlock()
Expand All @@ -55,6 +59,10 @@ func (r *campaignRegistry) ServeHTTP(w http.ResponseWriter, request *http.Reques
fmt.Fprintf(w, `{"campaignId":%q,"campaignStatus":"TCR_PENDING"}`, id)
case request.Method == http.MethodGet && strings.HasPrefix(path, "/v2/10dlc/campaign/"):
id := strings.TrimPrefix(path, "/v2/10dlc/campaign/")
if r.campaigns[id] == campaignRegistryFails {
w.WriteHeader(http.StatusInternalServerError)
return
}
fmt.Fprintf(w, `{"campaignId":%q,"campaignStatus":%q}`, id, r.campaigns[id])
case request.Method == http.MethodPost && path == "/v2/10dlc/phone_number_campaigns":
r.assigned[body["phoneNumber"]] = body["campaignId"]
Expand Down
133 changes: 133 additions & 0 deletions acceleration/internal/api/hook_activity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,17 @@
package api

import (
"bytes"
"context"
"crypto/ed25519"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"net/http"
"net/url"
"strconv"
"strings"
"time"

"github.com/GetStream/Vision-Agents/acceleration/internal/activity"
Expand Down Expand Up @@ -256,3 +264,128 @@ func (s *TelnyxHookSuite) TestACallWhoseTrunkCannotBeRenderedIsCountedRefused()
s.Equal(int64(1), row.refused)
s.Equal(internalError().Code, row.code)
}

// activityWritten waits until what the suite's activity record holds now is written: a row
// recorded after it is written in the same write, one transaction, or a later one.
func (s *RouterSuite) activityWritten() {
marker := "written-" + rand.Text()
s.activity.Record(activity.Request{AppID: s.customerID(), Area: marker, Client: store.AuditSourceAPI, At: time.Now()})
s.Require().Eventually(func() bool { return s.counted(marker, store.AuditSourceAPI).requests == 1 },
settleFor, 10*time.Millisecond, "the activity record was not written")
}

// hooksAfter delivers, and returns the app's hooks_in row before and after it.
func (s *RouterSuite) hooksAfter(deliver func()) (before, after counted) {
s.activityWritten()
before = s.counted(activity.AreaHooksIn, store.AuditSourceAPI)
deliver()
s.activityWritten()
return before, s.counted(activity.AreaHooksIn, store.AuditSourceAPI)
}

// Defect: an event the app's MCP server signed with the subscription's secret left out of the
// app's hooks, or one signed with another subscription's secret counted under the app.
func (s *ConnectionEventsSuite) TestASignedMCPEventIsCountedAsTheAppsHookAndAForgedOneIsNot() {
connection := s.connection()
binding := s.binding(connection, issueCreated)
binding["events"] = append(binding["events"].([]map[string]any), map[string]any{"event": "issue.closed"})
s.subscribed(binding)
subs := s.atTheFake(connection)
s.Require().Len(subs, 2)

before, after := s.hooksAfter(func() {
s.Require().Equal(http.StatusUnauthorized, s.provider.DeliverEvent(subs[0].URL, subs[1].Secret, subs[0].ID,
"evt_"+s.utils.uuid(), subs[0].Name, map[string]any{}))
s.Require().Equal(http.StatusAccepted, s.provider.DeliverEvent(subs[0].URL, subs[0].Secret, subs[0].ID,
"evt_"+s.utils.uuid(), subs[0].Name, map[string]any{}))
})

s.Equal(before.requests+1, after.requests)
s.Equal(before.refused, after.refused)
}

// Defect: a signed delivery the router then turned down left out of the app's hooks, so the
// digest shows none of the app's failing MCP events.
func (s *ConnectionEventsSuite) TestASignedMCPEventTheRouterTurnsDownIsCountedRefused() {
connection := s.subscribed(s.binding(s.connection(), issueCreated))
sub := s.atTheFake(connection)[0]

before, after := s.hooksAfter(func() {
s.Require().Equal(http.StatusBadRequest, s.provider.DeliverEvent(sub.URL, sub.Secret, sub.ID,
"evt_"+s.utils.uuid(), "issue.closed", map[string]any{}))
})

s.Equal(before.requests+1, after.requests)
s.Equal(before.refused+1, after.refused)
}

// Defect: a report the registrar signed about an app's campaign left out of the app's phone
// work, or one signed with another key counted.
func (s *DLCSuite) TestASignedReportOnACampaignIsCountedAsItsAppsPhoneWork() {
_, campaign := s.pendingWithTheVendor()
_, stranger, err := ed25519.GenerateKey(rand.Reader)
s.Require().NoError(err)
s.activityWritten()
before := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI)

s.Require().Equal(http.StatusUnauthorized, s.report(`{"campaignId":"`+campaign+`"}`, stranger))
s.Require().Equal(http.StatusNoContent, s.report(`{"campaignId":"`+campaign+`"}`, s.signing))
s.activityWritten()

after := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI)
s.Equal(before.requests+1, after.requests)
s.Equal(before.refused, after.refused)
s.Equal(int64(0), s.counted(activity.AreaHooksIn, store.AuditSourceAPI).requests, "a report is phone work")
}

// Defect: a signed delivery that is not JSON, answered 400, left out of the app's hooks.
func (s *ConnectionEventsSuite) TestASignedDeliveryThatIsNotJSONIsCountedRefused() {
connection := s.subscribed(s.binding(s.connection(), issueCreated))
sub := s.atTheFake(connection)[0]

before, after := s.hooksAfter(func() {
s.Require().Equal(http.StatusBadRequest, s.signedRaw(sub.URL, sub.Secret, sub.ID, []byte("not json")))
})

s.Equal(before.requests+1, after.requests)
s.Equal(before.refused+1, after.refused)
}

// signedRaw posts body signed with secret, as the MCP server signs a delivery, and returns the
// status the router answered.
func (s *ConnectionEventsSuite) signedRaw(url, secret, subscriptionID string, body []byte) int {
key, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(secret, "whsec_"))
s.Require().NoError(err)
id := "msg_" + s.utils.uuid()
stamp := strconv.FormatInt(time.Now().Unix(), 10)
mac := hmac.New(sha256.New, key)
mac.Write([]byte(id + "." + stamp + "."))
mac.Write(body)
request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
s.Require().NoError(err)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("webhook-id", id)
request.Header.Set("webhook-timestamp", stamp)
request.Header.Set("webhook-signature", "v1,"+base64.StdEncoding.EncodeToString(mac.Sum(nil)))
request.Header.Set("X-MCP-Subscription-Id", subscriptionID)
response, err := http.DefaultClient.Do(request)
s.Require().NoError(err)
_ = response.Body.Close()
return response.StatusCode
}

// Defect: a signed report on an app's campaign the router failed to act on (5xx) left out of
// the app's phone work, so the digest shows none of its failing 10DLC reports.
func (s *DLCSuite) TestASignedReportTheRouterFailsOnIsCountedRefused() {
_, campaign := s.pendingWithTheVendor()
s.registry.set(campaign, campaignRegistryFails)
s.activityWritten()
before := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI)

s.Require().GreaterOrEqual(s.report(`{"campaignId":"`+campaign+`"}`, s.signing), http.StatusInternalServerError)
s.activityWritten()

after := s.counted(activity.AreaPhoneSMS, store.AuditSourceAPI)
s.Equal(before.requests+1, after.requests)
s.Equal(before.refused+1, after.refused)
}
10 changes: 10 additions & 0 deletions acceleration/internal/api/scheduled_tasks.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package api

import (
"cmp"
"context"
"errors"
"net/http"
Expand All @@ -10,6 +11,7 @@ import (

"github.com/danielgtaylor/huma/v2"

"github.com/GetStream/Vision-Agents/acceleration/internal/activity"
"github.com/GetStream/Vision-Agents/acceleration/internal/auth"
"github.com/GetStream/Vision-Agents/acceleration/internal/routing"
"github.com/GetStream/Vision-Agents/acceleration/internal/scheduling"
Expand Down Expand Up @@ -622,6 +624,14 @@ func (s *Server) deliverManualRun(task store.ScheduledTask, run store.ScheduledT
s.logger.Error("could not record a scheduled task run by hand",
"task", task.ID, "run", run.ID, "error", err)
}
// Counted here whether or not its run was written: no worker takes a run by hand up again.
request := activity.Request{
AppID: task.CustomerID, Area: activity.AreaScheduling, Client: activity.ClientRouter, At: time.Now(),
}
if state != store.ScheduledTaskRunDelivered {
request.Refused, request.Code = true, cmp.Or(failure, state)
}
s.activity.Record(request)
}

func (s *Server) deleteScheduledTask(ctx context.Context, request *deleteScheduledTaskRequest) (*struct{}, error) {
Expand Down
39 changes: 39 additions & 0 deletions acceleration/internal/api/scheduled_tasks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/gorilla/websocket"
"github.com/uptrace/bun"

"github.com/GetStream/Vision-Agents/acceleration/internal/activity"
"github.com/GetStream/Vision-Agents/acceleration/internal/auth"
"github.com/GetStream/Vision-Agents/acceleration/internal/channels"
"github.com/GetStream/Vision-Agents/acceleration/internal/conversation"
Expand Down Expand Up @@ -3467,6 +3468,44 @@ func (s *ScheduledTasksSuite) TestARunByHandThatFailsListsTheReasonItFailedUnder
s.Equal(scheduling.ReasonNoNumber, *page.Items[0].Error)
}

// Defect: a run by hand that landed counted refused, or not at all past the request that set
// it going.
func (s *ScheduledTasksSuite) TestARunByHandThatLandsIsCountedDelivered() {
config := s.agentWithScheduling(nil)
created := s.createTask(s.data.createUser(), taskRequest(config))
s.activityWritten()
before := s.counted(activity.AreaScheduling, activity.ClientRouter)

s.Require().Equal(http.StatusAccepted, s.serverClient.do(http.MethodPost, "/v1/agents/scheduled-tasks/"+created.Id+"/run", nil, nil))

// The request that set it going, and the run.
s.Require().Eventually(func() bool {
return s.counted(activity.AreaScheduling, activity.ClientRouter).requests == before.requests+2
}, settleFor, 10*time.Millisecond, "the run by hand was not counted")
s.Equal(before.refused, s.counted(activity.AreaScheduling, activity.ClientRouter).refused)
}

// Defect: a run by hand that failed counted only as the request that set it going, so the
// app's scheduling work shows no refusal, or one without the reason its run failed under.
func (s *ScheduledTasksSuite) TestARunByHandThatFailsIsCountedRefusedUnderItsReason() {
config := s.agentWithScheduling(nil)
user := s.data.createUser()
opened := s.openScheduling(user, config, nil)
created := s.taskOn(user, config, opened.Id, []string{"phone"})
s.activityWritten()
before := s.counted(activity.AreaScheduling, activity.ClientRouter)

s.Require().Equal(http.StatusAccepted, s.serverClient.do(http.MethodPost, "/v1/agents/scheduled-tasks/"+created.Id+"/run", nil, nil))

s.Require().Eventually(func() bool {
return s.counted(activity.AreaScheduling, activity.ClientRouter).refused == before.refused+1
}, settleFor, 10*time.Millisecond, "the failed run by hand was not counted refused")
after := s.counted(activity.AreaScheduling, activity.ClientRouter)
// The request that set it going, and the run.
s.Equal(before.requests+2, after.requests)
s.Equal(scheduling.ReasonNoNumber, after.code)
}

func (s *ScheduledTasksSuite) TestATaskForAnAgentWithoutANumberHasNoLineToRingFrom() {
s.retryAtOnce()
config := s.agentWithScheduling(nil)
Expand Down
Loading
Loading