diff --git a/agent/internal/catalog/error_context_test.go b/agent/internal/catalog/error_context_test.go index fc0571b19..738fddb4d 100644 --- a/agent/internal/catalog/error_context_test.go +++ b/agent/internal/catalog/error_context_test.go @@ -7,6 +7,7 @@ import ( "net/url" "strings" "testing" + "testing/synctest" "github.com/muxable/rtchat/agent/internal/callable" "github.com/muxable/rtchat/agent/internal/clients" @@ -93,3 +94,30 @@ func TestUserTokenFailurePreservesCause(t *testing.T) { t.Fatalf("token failure context lost: %v", err) } } + +func TestEmotesRecoverFromTransientProviderError(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s, _, tokens := fixture() + tokens.user["uid:twitch"] = "token" + calls := 0 + s.HTTP = &http.Client{Transport: transport(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "api.twitch.tv" { + return response(404, `{}`), nil + } + if r.URL.Path == "/helix/users" { + calls++ + if calls == 1 { + return response(503, `{}`), nil + } + return response(200, `{"data":[{"id":"7","login":"viewer"}]}`), nil + } + return response(200, `{"data":[]}`), nil + })} + core, logs := observer.New(zap.ErrorLevel) + defer zap.ReplaceGlobals(zap.New(core))() + status, result := invoke(t, s, "getEmotes", `{"provider":"twitch","channelId":"123"}`, "uid") + if status != 200 || result["result"] == nil || calls != 2 || logs.Len() != 0 { + t.Fatalf("status=%d response=%v calls=%d error logs=%d", status, result, calls, logs.Len()) + } + }) +} diff --git a/agent/internal/clients/client.go b/agent/internal/clients/client.go index b32d5338a..3197d6649 100644 --- a/agent/internal/clients/client.go +++ b/agent/internal/clients/client.go @@ -92,15 +92,25 @@ func (c Client) Do(ctx context.Context, method, path string, body any) (*Respons return &Response{StatusCode: response.StatusCode, Header: response.Header, Body: data}, nil } +// Get retries provider 5xx responses up to three times, honoring server backoff +// and the caller's context. Do remains single-attempt for callers with their own policy. func (c Client) Get(ctx context.Context, path string, out any) error { - response, err := c.Do(ctx, http.MethodGet, path, nil) - if err != nil { - return err - } - if err := response.Err(); err != nil { - return err + for attempt := 0; ; attempt++ { + response, err := c.Do(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + if response.StatusCode >= 500 && response.StatusCode < 600 && attempt < 3 { + if err := Wait(ctx, RetryDelay(response.Header, attempt)); err != nil { + return err + } + continue + } + if err := response.Err(); err != nil { + return err + } + return response.Decode(out) } - return response.Decode(out) } func (c Client) DoJSON(ctx context.Context, method, path string, body any) (int, map[string]any, error) { diff --git a/agent/internal/clients/get_retry_test.go b/agent/internal/clients/get_retry_test.go new file mode 100644 index 000000000..c67c782a7 --- /dev/null +++ b/agent/internal/clients/get_retry_test.go @@ -0,0 +1,132 @@ +package clients_test + +import ( + "context" + "errors" + "io" + "net/http" + "strconv" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/muxable/rtchat/agent/internal/clients" + "github.com/muxable/rtchat/agent/internal/clients/twitch" +) + +type retryBody struct { + io.Reader + closed bool +} + +func (b *retryBody) Close() error { b.closed = true; return nil } + +func TestGetRecoversFromServerErrors(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls []time.Time + var bodies []*retryBody + statuses := []int{500, 502, 503, 200} + client := twitch.New(&http.Client{Transport: transport(func(r *http.Request) (*http.Response, error) { + if len(bodies) > 0 && !bodies[len(bodies)-1].closed { + t.Fatal("previous response was not closed before retry") + } + if r.Header.Get("Authorization") != "Bearer token" || r.URL.RequestURI() != "/helix/users?id=123" { + t.Fatal("retry changed the provider request") + } + status := statuses[len(calls)] + calls = append(calls, time.Now()) + body := &retryBody{Reader: strings.NewReader(`{"id":"123"}`)} + bodies = append(bodies, body) + return &http.Response{StatusCode: status, Header: http.Header{}, Body: body}, nil + })}, "client", "token") + var result struct{ ID string } + if err := client.Get(context.Background(), "/users?id=123", &result); err != nil || result.ID != "123" { + t.Fatalf("recovery failed: result=%+v err=%v", result, err) + } + if len(calls) != 4 { + t.Fatalf("got %d requests, want initial plus three retries", len(calls)) + } + for i := 1; i < len(calls); i++ { + if delay := calls[i].Sub(calls[i-1]); delay != time.Second<