From be6d5de12196ce1a1f9076d483883215d40fc3e3 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Tue, 10 Nov 2026 16:16:01 +0100 Subject: [PATCH 1/4] Retry OCI requests on a closed connection, and the workload identity token fetch A request whose connection the server closes before answering fails with a url.Error wrapping io.EOF. The OCI SDK's retry predicate compares the error to io.EOF by equality, so the wrapped one gets a single attempt and the upload is lost. The client's policy now adds EOF, unexpected EOF and a server-closed idle connection to the SDK's predicate. It is built on the default without eventual consistency, as the eventual-consistency default replaces a custom predicate with its own. With workload identity the SDK fetches its token with one request, inside the client constructor and again on renewal, so a throttled fetch fails NewOCI or a request mid-run. The provider now repeats the fetch up to five times with jittered backoff, except on a 403. Co-Authored-By: Claude Fable 5.2 --- README.md | 2 +- oci.go | 94 ++++++++++++++++++++++++++++- oci_test.go | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1f31a2c..2dd1b28 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does. It tries every path, and when some are not deleted fails with a `*DeleteError` whose `Failed` names each one with its error (`errors.As`); every other path is deleted, and a retry is safe. - **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. - **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut. An HTTP/2 connection with no inbound frame for 30 seconds is pinged and cut when the ping goes unanswered for 15 seconds, so a dead peer fails every request on it within 45 seconds with an error the SDKs retry. The ping waits behind the frame being written, so a peer accepting less than one 16 KiB frame per 15 seconds is cut as well; any faster transfer, however slow, never is. The context bounds the call. -- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. A stream given to `UploadObject` is sent once on AliOSS. +- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. OCI also retries a request whose connection the server closed before answering, which the SDK does not, and a workload identity token fetch that fails, which the SDK makes once. A stream given to `UploadObject` is sent once on AliOSS. - **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. - **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. diff --git a/oci.go b/oci.go index 77508ff..dd9dfe2 100644 --- a/oci.go +++ b/oci.go @@ -17,10 +17,12 @@ package storage import ( "bytes" "context" + "crypto/rsa" "crypto/tls" "errors" "fmt" "io" + "math/rand/v2" "net/http" "net/url" "os" @@ -80,6 +82,8 @@ func NewOCI(conf *OCIConfig) (Storage, error) { } client.HTTPClient = c } + policy := ociRetryPolicy() + client.Configuration.RetryPolicy = &policy // client.Host keeps endpoint options such as {dualStack?ds.oci.:}, which the // SDK resolves on a copy per request, so resolve them the same way here. @@ -135,9 +139,97 @@ func lookupNamespace(client objectstorage.ObjectStorageClient, req objectstorage } } +// ociRetryPolicy is the SDK's default policy with ociShouldRetry as its +// predicate. The eventual-consistency default swaps a set predicate out. +func ociRetryPolicy() common.RetryPolicy { + policy := common.DefaultRetryPolicyWithoutEventualConsistency() + policy.ShouldRetryOperation = ociShouldRetry + return policy +} + +// ociShouldRetry is the SDK's predicate plus a connection the server closed +// under the request, which the SDK matches only when nothing wraps the error. +// Such a request may have been received, as with a timeout the SDK retries. +func ociShouldRetry(r common.OCIOperationResponse) bool { + if common.DefaultShouldRetryOperation(r) { + return true + } + err := r.Error + if err == nil { + return false + } + return errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + strings.Contains(err.Error(), "http: server closed idle connection") +} + +const ( + ociTokenAttempts = 5 + // A fetch that fails this long after the first began has timed out, and + // is not repeated. + ociTokenRetryWindow = 15 * time.Second +) + +// ociTokenBackoff is the wait before the first retry of a token fetch. Each +// later wait doubles it, and each is jittered. +var ociTokenBackoff = 250 * time.Millisecond + +// ociTokenProvider retries the workload identity token fetch, which the SDK +// makes once, inside whichever of these calls finds no valid token. +type ociTokenProvider struct { + auth.ConfigurationProviderWithClaimAccess +} + +func (p ociTokenProvider) TenancyOCID() (string, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.TenancyOCID) +} + +func (p ociTokenProvider) KeyID() (string, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.KeyID) +} + +func (p ociTokenProvider) PrivateRSAKey() (*rsa.PrivateKey, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.PrivateRSAKey) +} + +func (p ociTokenProvider) GetClaim(key string) (interface{}, error) { + return retryOCIToken(func() (interface{}, error) { + return p.ConfigurationProviderWithClaimAccess.GetClaim(key) + }) +} + +// Refreshable keeps the SDK's retry of a 401 on a token it can renew. +func (p ociTokenProvider) Refreshable() bool { + r, ok := p.ConfigurationProviderWithClaimAccess.(common.RefreshableConfigurationProvider) + return ok && r.Refreshable() +} + +func retryOCIToken[T any](fetch func() (T, error)) (T, error) { + start := time.Now() + backoff := ociTokenBackoff + for attempt := 1; ; attempt++ { + v, err := fetch() + if err == nil || attempt == ociTokenAttempts || !ociTokenRetryable(err) || time.Since(start) > ociTokenRetryWindow { + return v, err + } + time.Sleep(backoff/2 + rand.N(backoff/2+1)) + backoff *= 2 + } +} + +// ociTokenRetryable is any failure but the 403 of a cluster without workload +// identity. The SDK flattens the response's status into the error's text. +func ociTokenRetryable(err error) bool { + return !strings.Contains(err.Error(), "Status: 403") +} + func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { if conf.UseWorkloadIdentity { - return auth.OkeWorkloadIdentityConfigurationProvider() + provider, err := auth.OkeWorkloadIdentityConfigurationProvider() + if err != nil { + return nil, err + } + return ociTokenProvider{provider}, nil } if conf.TenancyOCID != "" && conf.UserOCID != "" && conf.PrivateKey != "" { diff --git a/oci_test.go b/oci_test.go index 0d3d4c6..062d76b 100644 --- a/oci_test.go +++ b/oci_test.go @@ -15,13 +15,23 @@ package storage import ( + "context" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" + "errors" + "io" + "net/http" + "net/http/httptest" "net/url" + "slices" + "sync/atomic" + "syscall" "testing" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" "github.com/stretchr/testify/require" ) @@ -90,3 +100,160 @@ func TestOCILocationHost(t *testing.T) { require.NoError(t, err) require.Equal(t, "objectstorage.us-chicago-1.oraclecloud.com", u.Host) } + +// ociTestConfig is API key credentials for endpoint. A set Namespace keeps +// NewOCI off the network. +func ociTestConfig(t *testing.T, endpoint string) *OCIConfig { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + return &OCIConfig{ + TenancyOCID: "ocid1.tenancy.oc1..tenancy", + UserOCID: "ocid1.user.oc1..user", + Fingerprint: "aa:bb:cc", + PrivateKey: string(pemKey), + Region: "us-chicago-1", + Namespace: "ns", + Bucket: "bucket", + Endpoint: endpoint, + } +} + +// A request whose connection the server closes before answering is sent +// again, where the SDK's own predicate leaves it failed. +func TestOCIRetriesClosedConnection(t *testing.T) { + t.Run("upload", func(t *testing.T) { + // Without the header the server writes nothing before it closes, so + // the client reads a bare EOF. + t.Setenv(common.UsingExpectHeaderEnvVar, "false") + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if requests.Add(1) > 1 { + return + } + if c, _, err := w.(http.Hijacker).Hijack(); err == nil { + _ = c.Close() + } + })) + t.Cleanup(srv.Close) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.NoError(t, s.PutObject(context.Background(), "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 2, requests.Load()) + }) + + put := func(err error) common.OCIOperationResponse { + return common.OCIOperationResponse{ + Error: &url.Error{Op: "Put", URL: "https://objectstorage.us-chicago-1.oraclecloud.com/n/ns/b/bucket/o/k", Err: err}, + } + } + + t.Run("errors of a closed connection", func(t *testing.T) { + for _, err := range []error{ + io.EOF, + io.ErrUnexpectedEOF, + errors.New("http: server closed idle connection"), + } { + r := put(err) + require.False(t, common.DefaultShouldRetryOperation(r), "the SDK's predicate, which this package adds to: %v", err) + require.True(t, ociShouldRetry(r), "%v", err) + } + }) + + t.Run("unchanged otherwise", func(t *testing.T) { + require.True(t, ociShouldRetry(put(syscall.ECONNRESET))) + require.False(t, ociShouldRetry(put(errors.New("x509: certificate signed by unknown authority")))) + require.False(t, ociShouldRetry(common.OCIOperationResponse{Error: &ociServiceError{status: http.StatusNotFound}})) + }) +} + +// flakyTokenProvider fails its token-backed calls with failures, in turn, then +// serves them as the API key provider it wraps. +type flakyTokenProvider struct { + common.ConfigurationProvider + failures []error + fetches int +} + +func (p *flakyTokenProvider) fetch() error { + p.fetches++ + if len(p.failures) == 0 { + return nil + } + err := p.failures[0] + p.failures = p.failures[1:] + return err +} + +func (p *flakyTokenProvider) TenancyOCID() (string, error) { + if err := p.fetch(); err != nil { + return "", err + } + return p.ConfigurationProvider.TenancyOCID() +} + +func (p *flakyTokenProvider) KeyID() (string, error) { + if err := p.fetch(); err != nil { + return "", err + } + return p.ConfigurationProvider.KeyID() +} + +func (p *flakyTokenProvider) PrivateRSAKey() (*rsa.PrivateKey, error) { + if err := p.fetch(); err != nil { + return nil, err + } + return p.ConfigurationProvider.PrivateRSAKey() +} + +func (p *flakyTokenProvider) GetClaim(string) (interface{}, error) { + return nil, p.fetch() +} + +func (p *flakyTokenProvider) Refreshable() bool { return true } + +// A workload identity token fetch that is throttled is made again, so a client +// is built where the SDK alone fails on the first refusal. +func TestOCIRetriesTokenFetch(t *testing.T) { + backoff := ociTokenBackoff + ociTokenBackoff = 0 + t.Cleanup(func() { ociTokenBackoff = backoff }) + + // The SDK's errors, which carry the response's status only as text. + throttled := errors.New("failed to renew security token: failed to get security token: failed to get a Workload Identity token. Status: 429 Too Many Requests, Message: . Please contact OKE team") + forbidden := errors.New("failed to renew security token: failed to get security token: please ensure the cluster type is enhanced: Status: 403 Forbidden, Message: ") + + flaky := func(failures ...error) *flakyTokenProvider { + keys, err := newConfigProvider(ociTestConfig(t, "")) + require.NoError(t, err) + return &flakyTokenProvider{ConfigurationProvider: keys, failures: failures} + } + + t.Run("a client is built over throttled fetches", func(t *testing.T) { + _, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(flaky(throttled)) + require.Error(t, err, "the SDK's one fetch, which this package repeats") + + p := flaky(throttled, throttled) + client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(ociTokenProvider{p}) + require.NoError(t, err) + require.Empty(t, p.failures) + require.True(t, client.IsRefreshableAuthType()) + }) + + t.Run("gives up after its attempts", func(t *testing.T) { + p := flaky(slices.Repeat([]error{throttled}, ociTokenAttempts+1)...) + _, err := ociTokenProvider{p}.KeyID() + require.ErrorIs(t, err, throttled) + require.Equal(t, ociTokenAttempts, p.fetches) + }) + + t.Run("a cluster without workload identity fails at once", func(t *testing.T) { + p := flaky(forbidden, forbidden) + _, err := ociTokenProvider{p}.GetClaim("res_tenant") + require.ErrorIs(t, err, forbidden) + require.Equal(t, 1, p.fetches) + }) +} From 475eb818983a965e35208e8e7b350ff1709d3b34 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Tue, 10 Nov 2026 20:26:41 +0100 Subject: [PATCH 2/4] Build the OCI client on the first call, not in NewOCI With workload identity the SDK fetches a token as its client is built. In NewOCI that fetch has no context to bound it, a throttled fetch leaves the caller with no client for good, and a client that is never used fetches a token all the same. NewOCI now makes no request. The first call builds the client, under a mutex and released when its context ends, and a build that fails is not kept, so the next call tries again. A namespace the config leaves out is looked up by the first call as well. A client with API key credentials is still built in NewOCI, which needs no request, so a bad key fails there. Co-Authored-By: Claude Fable 5.2 --- README.md | 1 + oci.go | 116 ++++++++++++++++++++++++++++++++++++--------- oci_test.go | 132 +++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 209 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 2dd1b28..106446c 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. - **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut. An HTTP/2 connection with no inbound frame for 30 seconds is pinged and cut when the ping goes unanswered for 15 seconds, so a dead peer fails every request on it within 45 seconds with an error the SDKs retry. The ping waits behind the frame being written, so a peer accepting less than one 16 KiB frame per 15 seconds is cut as well; any faster transfer, however slow, never is. The context bounds the call. - **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. OCI also retries a request whose connection the server closed before answering, which the SDK does not, and a workload identity token fetch that fails, which the SDK makes once. A stream given to `UploadObject` is sent once on AliOSS. +- **OCI setup**: `NewOCI` makes no request. With workload identity the first call builds the client, which fetches its token, and a namespace the config leaves out is looked up by the first call too. A failure there fails that call, bounded by its context, and the next call tries again. - **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. - **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. diff --git a/oci.go b/oci.go index dd9dfe2..083dc5f 100644 --- a/oci.go +++ b/oci.go @@ -29,6 +29,8 @@ import ( "path" "slices" "strings" + "sync" + "sync/atomic" "time" "github.com/oracle/oci-go-sdk/v65/common" @@ -37,7 +39,13 @@ import ( ) type ociStorage struct { - conf *OCIConfig + conf *OCIConfig + provider common.ConfigurationProvider + + // connected is set once the fields below are, by connect under mu. + connected atomic.Bool + mu sync.Mutex + built bool client objectstorage.ObjectStorageClient namespace string host string @@ -45,15 +53,75 @@ type ociStorage struct { insecure bool } +// NewOCI makes no request. With workload identity the SDK fetches a token as +// its client is built, so the first call builds it. A namespace the config +// leaves out is looked up by the first call as well. func NewOCI(conf *OCIConfig) (Storage, error) { provider, err := newConfigProvider(conf) if err != nil { return nil, err } - client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) + s := &ociStorage{conf: conf, provider: provider, namespace: conf.Namespace} + if !conf.UseWorkloadIdentity { + if err = s.build(); err != nil { + return nil, err + } + s.connected.Store(s.namespace != "") + } + return s, nil +} + +// ready has the client built and the namespace known, by this call where no +// earlier one did. A failure is not kept, so the next call tries again. A call +// whose ctx ends is released, and the work goes on for the next one. +func (s *ociStorage) ready(ctx context.Context) error { + if s.connected.Load() { + return nil + } + done := make(chan error, 1) + go func() { done <- s.connect() }() + select { + case err := <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *ociStorage) connect() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.connected.Load() { + return nil + } + + if !s.built { + if err := s.build(); err != nil { + return err + } + } + if s.namespace == "" { + req := objectstorage.GetNamespaceRequest{} + if s.conf.CompartmentID != "" { + req.CompartmentId = &s.conf.CompartmentID + } + namespace, err := lookupNamespace(s.client, req) + if err != nil { + return err + } + s.namespace = namespace + } + s.connected.Store(true) + return nil +} + +// build makes the SDK client, which with workload identity fetches a token. +func (s *ociStorage) build() error { + conf := s.conf + client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(s.provider) if err != nil { - return nil, err + return err } if conf.Region != "" { @@ -93,25 +161,8 @@ func NewOCI(conf *OCIConfig) (Storage, error) { insecure := strings.HasPrefix(host, "http://") host = strings.TrimPrefix(host, "http://") - s := &ociStorage{ - conf: conf, - client: client, - namespace: conf.Namespace, - host: host, - insecure: insecure, - } - - if s.namespace == "" { - req := objectstorage.GetNamespaceRequest{} - if conf.CompartmentID != "" { - req.CompartmentId = &conf.CompartmentID - } - if s.namespace, err = lookupNamespace(client, req); err != nil { - return nil, err - } - } - - return s, nil + s.client, s.host, s.insecure, s.built = client, host, insecure, true + return nil } // lookupNamespace is the tenancy's namespace, or an error after a minute, as @@ -293,6 +344,9 @@ func (s *ociStorage) DownloadFile(ctx context.Context, storagePath, filepath str // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + if err := s.ready(ctx); err != nil { + return "", err + } name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} @@ -341,6 +395,9 @@ var ociContentDisposition = common.String("inline") // put sends body in one request. It must be seekable, so the SDK can re-read // it when it retries. func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCloser, size int64, opts PutOptions) error { + if err := s.ready(ctx); err != nil { + return err + } req := objectstorage.PutObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -363,6 +420,9 @@ func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCl // multipart upload of sequential parts through one buffer. IfAbsent is checked // at create and again at commit, so it is atomic here. func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if err := s.ready(ctx); err != nil { + return err + } // The condition is checked again at create and commit; this check spares a // refused upload its traffic, and refuses it where a store ignores // if-none-match on multipart uploads, as floci-oci does. @@ -492,6 +552,9 @@ func (s *ociStorage) DownloadObject(ctx context.Context, storagePath string, w i } func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + if err := s.ready(ctx); err != nil { + return 0, err + } resp, err := s.client.GetObject(ctx, objectstorage.GetObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -507,6 +570,9 @@ func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writ } func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + if err := s.ready(ctx); err != nil { + return false, err + } _, err := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -520,6 +586,9 @@ func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool } func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + if err := s.ready(ctx); err != nil { + return nil, err + } var keys []string var start *string for { @@ -545,6 +614,9 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { + if err := s.ready(ctx); err != nil { + return err + } _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, diff --git a/oci_test.go b/oci_test.go index 062d76b..4b75ffa 100644 --- a/oci_test.go +++ b/oci_test.go @@ -26,9 +26,12 @@ import ( "net/http/httptest" "net/url" "slices" + "strings" + "sync" "sync/atomic" "syscall" "testing" + "time" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/objectstorage" @@ -176,9 +179,20 @@ type flakyTokenProvider struct { common.ConfigurationProvider failures []error fetches int + // gate, when set, holds every fetch until it is closed. + gate chan struct{} +} + +func flakyProvider(t *testing.T, failures ...error) *flakyTokenProvider { + keys, err := newConfigProvider(ociTestConfig(t, "")) + require.NoError(t, err) + return &flakyTokenProvider{ConfigurationProvider: keys, failures: failures} } func (p *flakyTokenProvider) fetch() error { + if p.gate != nil { + <-p.gate + } p.fetches++ if len(p.failures) == 0 { return nil @@ -215,28 +229,29 @@ func (p *flakyTokenProvider) GetClaim(string) (interface{}, error) { func (p *flakyTokenProvider) Refreshable() bool { return true } -// A workload identity token fetch that is throttled is made again, so a client -// is built where the SDK alone fails on the first refusal. -func TestOCIRetriesTokenFetch(t *testing.T) { +// The SDK's token fetch errors, which carry the response's status only as text. +var ( + errOCIThrottled = errors.New("failed to renew security token: failed to get security token: failed to get a Workload Identity token. Status: 429 Too Many Requests, Message: . Please contact OKE team") + errOCIForbidden = errors.New("failed to renew security token: failed to get security token: please ensure the cluster type is enhanced: Status: 403 Forbidden, Message: ") +) + +// noOCITokenBackoff has a token fetch retried at once. +func noOCITokenBackoff(t *testing.T) { backoff := ociTokenBackoff ociTokenBackoff = 0 t.Cleanup(func() { ociTokenBackoff = backoff }) +} - // The SDK's errors, which carry the response's status only as text. - throttled := errors.New("failed to renew security token: failed to get security token: failed to get a Workload Identity token. Status: 429 Too Many Requests, Message: . Please contact OKE team") - forbidden := errors.New("failed to renew security token: failed to get security token: please ensure the cluster type is enhanced: Status: 403 Forbidden, Message: ") - - flaky := func(failures ...error) *flakyTokenProvider { - keys, err := newConfigProvider(ociTestConfig(t, "")) - require.NoError(t, err) - return &flakyTokenProvider{ConfigurationProvider: keys, failures: failures} - } +// A workload identity token fetch that is throttled is made again, so a client +// is built where the SDK alone fails on the first refusal. +func TestOCIRetriesTokenFetch(t *testing.T) { + noOCITokenBackoff(t) t.Run("a client is built over throttled fetches", func(t *testing.T) { - _, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(flaky(throttled)) + _, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(flakyProvider(t, errOCIThrottled)) require.Error(t, err, "the SDK's one fetch, which this package repeats") - p := flaky(throttled, throttled) + p := flakyProvider(t, errOCIThrottled, errOCIThrottled) client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(ociTokenProvider{p}) require.NoError(t, err) require.Empty(t, p.failures) @@ -244,16 +259,97 @@ func TestOCIRetriesTokenFetch(t *testing.T) { }) t.Run("gives up after its attempts", func(t *testing.T) { - p := flaky(slices.Repeat([]error{throttled}, ociTokenAttempts+1)...) + p := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, ociTokenAttempts+1)...) _, err := ociTokenProvider{p}.KeyID() - require.ErrorIs(t, err, throttled) + require.ErrorIs(t, err, errOCIThrottled) require.Equal(t, ociTokenAttempts, p.fetches) }) t.Run("a cluster without workload identity fails at once", func(t *testing.T) { - p := flaky(forbidden, forbidden) + p := flakyProvider(t, errOCIForbidden, errOCIForbidden) _, err := ociTokenProvider{p}.GetClaim("res_tenant") - require.ErrorIs(t, err, forbidden) + require.ErrorIs(t, err, errOCIForbidden) require.Equal(t, 1, p.fetches) }) } + +// With workload identity the first call builds the client, not NewOCI: a +// build that fails is that call's failure, and the next call builds again. +func TestOCIBuildsClientOnFirstCall(t *testing.T) { + noOCITokenBackoff(t) + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + requests.Add(1) + })) + t.Cleanup(srv.Close) + + // The store NewOCI makes with workload identity, over provider. + lazy := func(provider *flakyTokenProvider) *ociStorage { + conf := ociTestConfig(t, srv.URL) + return &ociStorage{conf: conf, provider: ociTokenProvider{provider}, namespace: conf.Namespace} + } + ctx := context.Background() + + t.Run("a failed build is tried again", func(t *testing.T) { + requests.Store(0) + s := lazy(flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, ociTokenAttempts)...)) + + err := s.PutObject(ctx, "k", []byte("hello"), PutOptions{}) + require.ErrorContains(t, err, "429 Too Many Requests") + require.Zero(t, requests.Load()) + + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("a call is released when its context ends", func(t *testing.T) { + requests.Store(0) + provider := flakyProvider(t) + provider.gate = make(chan struct{}) + s := lazy(provider) + + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + require.ErrorIs(t, s.PutObject(short, "k", []byte("hello"), PutOptions{}), context.DeadlineExceeded) + + // The build the first call began goes on, and serves the next. + close(provider.gate) + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 1, requests.Load()) + }) +} + +// A namespace the config leaves out is looked up by the first call, so NewOCI +// makes no request. +func TestOCILooksUpNamespaceOnFirstCall(t *testing.T) { + var mu sync.Mutex + var seen []string + requests := func() []string { + mu.Lock() + defer mu.Unlock() + return slices.Clone(seen) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + path := strings.TrimSuffix(r.URL.Path, "/") + mu.Lock() + seen = append(seen, r.Method+" "+path) + mu.Unlock() + if path == "/n" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `"ns"`) + } + })) + t.Cleanup(srv.Close) + + conf := ociTestConfig(t, srv.URL) + conf.Namespace = "" + s, err := NewOCI(conf) + require.NoError(t, err) + require.Empty(t, requests()) + + require.NoError(t, s.PutObject(context.Background(), "k", []byte("hello"), PutOptions{})) + require.Equal(t, []string{"GET /n", "PUT /n/ns/b/bucket/o/k"}, requests()) +} From f5708d9374ae410edeff5abaddba412140ed96e9 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Wed, 11 Nov 2026 10:59:51 +0100 Subject: [PATCH 3/4] OCI: share one client build, keep the configured retries, send a grant once Every call that found the client unbuilt started a build of its own, queued behind the others, so calls that had given up left attempts running: a build per call, each with up to five token requests. The calls waiting now share one build, and DeleteObjects checks the client once for all its paths, not once per path. The client's retry policy replaced the one the SDK installs from OCI_SDK_DEFAULT_RETRY_ENABLED or common.GlobalRetry. The closed connection cases are now added to the policy in effect, on both of the layers the SDK's default has, so its attempts and backoff stay and retries switched off stay off. A pre-authenticated request is not idempotent, and its URI comes back only in its answer. PresignedURL is now sent again only when the server answered with a failure: a request with no answer may have made its grant, and a second one would leave the first with a URI nobody has. Co-Authored-By: Claude Fable 5.2 --- README.md | 4 +- oci.go | 117 +++++++++++++++++++++++++++--------------- oci_test.go | 145 +++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 216 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 106446c..60852f0 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does. It tries every path, and when some are not deleted fails with a `*DeleteError` whose `Failed` names each one with its error (`errors.As`); every other path is deleted, and a retry is safe. - **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. - **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut. An HTTP/2 connection with no inbound frame for 30 seconds is pinged and cut when the ping goes unanswered for 15 seconds, so a dead peer fails every request on it within 45 seconds with an error the SDKs retry. The ping waits behind the frame being written, so a peer accepting less than one 16 KiB frame per 15 seconds is cut as well; any faster transfer, however slow, never is. The context bounds the call. -- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. OCI also retries a request whose connection the server closed before answering, which the SDK does not, and a workload identity token fetch that fails, which the SDK makes once. A stream given to `UploadObject` is sent once on AliOSS. -- **OCI setup**: `NewOCI` makes no request. With workload identity the first call builds the client, which fetches its token, and a namespace the config leaves out is looked up by the first call too. A failure there fails that call, bounded by its context, and the next call tries again. +- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. OCI also retries a request whose connection the server closed before answering, which the SDK does not, and a workload identity token fetch that fails, which the SDK makes once. It adds these to the retry policy the SDK is configured with (`OCI_SDK_DEFAULT_RETRY_ENABLED`, `common.GlobalRetry`), whose attempts and backoff stay. `PresignedURL` on OCI is sent again only when the server answered: a request with no answer may have made its grant. A stream given to `UploadObject` is sent once on AliOSS. +- **OCI setup**: `NewOCI` makes no request. With workload identity the first call builds the client, which fetches its token, and a namespace the config leaves out is looked up by the first call too. Calls waiting on that share one attempt. A failure there fails those calls, each bounded by its context, and a later call tries again. - **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. - **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. diff --git a/oci.go b/oci.go index 083dc5f..ac17667 100644 --- a/oci.go +++ b/oci.go @@ -29,26 +29,29 @@ import ( "path" "slices" "strings" - "sync" "sync/atomic" "time" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/common/auth" "github.com/oracle/oci-go-sdk/v65/objectstorage" + "golang.org/x/sync/singleflight" ) type ociStorage struct { conf *OCIConfig provider common.ConfigurationProvider - // connected is set once the fields below are, by connect under mu. - connected atomic.Bool - mu sync.Mutex - built bool - client objectstorage.ObjectStorageClient - namespace string - host string + // connected is set once the fields below are, by connect, which + // connecting runs one at a time. + connected atomic.Bool + connecting singleflight.Group + built bool + client objectstorage.ObjectStorageClient + // grantRetry is the policy of a call that leaves a grant behind. + grantRetry common.RetryPolicy + namespace string + host string // insecure is an http:// Endpoint, such as an emulator's. insecure bool } @@ -72,26 +75,24 @@ func NewOCI(conf *OCIConfig) (Storage, error) { return s, nil } -// ready has the client built and the namespace known, by this call where no -// earlier one did. A failure is not kept, so the next call tries again. A call -// whose ctx ends is released, and the work goes on for the next one. +// ready has the client built and the namespace known. The calls waiting on +// that share one connect, and a failure is not kept, so a later call starts +// another. A call whose ctx ends is released and leaves nothing running of its +// own: the connect goes on for the calls still waiting, and for the next. func (s *ociStorage) ready(ctx context.Context) error { if s.connected.Load() { return nil } - done := make(chan error, 1) - go func() { done <- s.connect() }() + done := s.connecting.DoChan("", func() (any, error) { return nil, s.connect() }) select { - case err := <-done: - return err + case r := <-done: + return r.Err case <-ctx.Done(): return ctx.Err() } } func (s *ociStorage) connect() error { - s.mu.Lock() - defer s.mu.Unlock() if s.connected.Load() { return nil } @@ -150,7 +151,9 @@ func (s *ociStorage) build() error { } client.HTTPClient = c } - policy := ociRetryPolicy() + // The SDK has set the policy the environment or common.GlobalRetry asks for. + configured := client.Configuration.RetryPolicy + policy := ociRetryPolicy(configured, ociAlsoClosed) client.Configuration.RetryPolicy = &policy // client.Host keeps endpoint options such as {dualStack?ds.oci.:}, which the @@ -162,6 +165,7 @@ func (s *ociStorage) build() error { host = strings.TrimPrefix(host, "http://") s.client, s.host, s.insecure, s.built = client, host, insecure, true + s.grantRetry = ociRetryPolicy(configured, ociOnlyAnswered) return nil } @@ -190,28 +194,52 @@ func lookupNamespace(client objectstorage.ObjectStorageClient, req objectstorage } } -// ociRetryPolicy is the SDK's default policy with ociShouldRetry as its -// predicate. The eventual-consistency default swaps a set predicate out. -func ociRetryPolicy() common.RetryPolicy { - policy := common.DefaultRetryPolicyWithoutEventualConsistency() - policy.ShouldRetryOperation = ociShouldRetry +type ociPredicate = func(common.OCIOperationResponse) bool + +// ociRetryPolicy is the policy in effect, configured or else the SDK's +// default, with its predicate changed by change: its attempts and backoff stay. +// The SDK runs the inner policy outside an eventual-consistency window, so +// the predicates of both change. +func ociRetryPolicy(configured *common.RetryPolicy, change func(ociPredicate) ociPredicate) common.RetryPolicy { + policy := common.DefaultRetryPolicy() + if configured != nil { + policy = *configured + } + policy.ShouldRetryOperation = change(policy.ShouldRetryOperation) + if policy.NonEventuallyConsistentPolicy != nil { + inner := *policy.NonEventuallyConsistentPolicy + inner.ShouldRetryOperation = change(inner.ShouldRetryOperation) + policy.NonEventuallyConsistentPolicy = &inner + } return policy } -// ociShouldRetry is the SDK's predicate plus a connection the server closed -// under the request, which the SDK matches only when nothing wraps the error. -// Such a request may have been received, as with a timeout the SDK retries. -func ociShouldRetry(r common.OCIOperationResponse) bool { - if common.DefaultShouldRetryOperation(r) { - return true +// ociAlsoClosed adds to retry a connection the server closed under the +// request, which the SDK matches only when nothing wraps the error. Such a +// request may have been received, as with a timeout the SDK retries. +func ociAlsoClosed(retry ociPredicate) ociPredicate { + return func(r common.OCIOperationResponse) bool { + if retry(r) { + return true + } + err := r.Error + if err == nil { + return false + } + return errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + strings.Contains(err.Error(), "http: server closed idle connection") } - err := r.Error - if err == nil { - return false +} + +// ociOnlyAnswered keeps of retry the failures the server answered with. A +// request that got no answer may have been carried out, and one sent again +// leaves a second grant, the first with a URI nobody was given. +func ociOnlyAnswered(retry ociPredicate) ociPredicate { + return func(r common.OCIOperationResponse) bool { + _, answered := common.IsServiceError(r.Error) + return answered && retry(r) } - return errors.Is(err, io.EOF) || - errors.Is(err, io.ErrUnexpectedEOF) || - strings.Contains(err.Error(), "http: server closed idle connection") } const ( @@ -255,6 +283,8 @@ func (p ociTokenProvider) Refreshable() bool { return ok && r.Refreshable() } +// The SDK asks for a token with no context, so the fetch of a call that has +// ended runs on, for under 4s. func retryOCIToken[T any](fetch func() (T, error)) (T, error) { start := time.Now() backoff := ociTokenBackoff @@ -351,8 +381,9 @@ func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expir expires := common.SDKTime{Time: time.Now().Add(expiration)} resp, err := s.client.CreatePreauthenticatedRequest(ctx, objectstorage.CreatePreauthenticatedRequestRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + RequestMetadata: common.RequestMetadata{RetryPolicy: &s.grantRetry}, CreatePreauthenticatedRequestDetails: objectstorage.CreatePreauthenticatedRequestDetails{ Name: &name, AccessType: objectstorage.CreatePreauthenticatedRequestDetailsAccessTypeObjectread, @@ -612,11 +643,17 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, } } +// DeleteObjects checks the client once for all its paths, and names every +// path in its error where there is none, as for any path it does not delete. func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { - return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { - if err := s.ready(ctx); err != nil { - return err + if err := s.ready(ctx); err != nil { + failed := make(map[string]error, len(storagePaths)) + for _, p := range storagePaths { + failed[p] = err } + return deleteError(failed) + } + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, diff --git a/oci_test.go b/oci_test.go index 4b75ffa..6510d25 100644 --- a/oci_test.go +++ b/oci_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "net/url" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -162,14 +163,15 @@ func TestOCIRetriesClosedConnection(t *testing.T) { } { r := put(err) require.False(t, common.DefaultShouldRetryOperation(r), "the SDK's predicate, which this package adds to: %v", err) - require.True(t, ociShouldRetry(r), "%v", err) + require.True(t, ociAlsoClosed(common.DefaultShouldRetryOperation)(r), "%v", err) } }) t.Run("unchanged otherwise", func(t *testing.T) { - require.True(t, ociShouldRetry(put(syscall.ECONNRESET))) - require.False(t, ociShouldRetry(put(errors.New("x509: certificate signed by unknown authority")))) - require.False(t, ociShouldRetry(common.OCIOperationResponse{Error: &ociServiceError{status: http.StatusNotFound}})) + retry := ociAlsoClosed(common.DefaultShouldRetryOperation) + require.True(t, retry(put(syscall.ECONNRESET))) + require.False(t, retry(put(errors.New("x509: certificate signed by unknown authority")))) + require.False(t, retry(common.OCIOperationResponse{Error: &ociServiceError{status: http.StatusNotFound}})) }) } @@ -178,7 +180,7 @@ func TestOCIRetriesClosedConnection(t *testing.T) { type flakyTokenProvider struct { common.ConfigurationProvider failures []error - fetches int + fetches atomic.Int32 // gate, when set, holds every fetch until it is closed. gate chan struct{} } @@ -193,7 +195,7 @@ func (p *flakyTokenProvider) fetch() error { if p.gate != nil { <-p.gate } - p.fetches++ + p.fetches.Add(1) if len(p.failures) == 0 { return nil } @@ -262,14 +264,14 @@ func TestOCIRetriesTokenFetch(t *testing.T) { p := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, ociTokenAttempts+1)...) _, err := ociTokenProvider{p}.KeyID() require.ErrorIs(t, err, errOCIThrottled) - require.Equal(t, ociTokenAttempts, p.fetches) + require.EqualValues(t, ociTokenAttempts, p.fetches.Load()) }) t.Run("a cluster without workload identity fails at once", func(t *testing.T) { p := flakyProvider(t, errOCIForbidden, errOCIForbidden) _, err := ociTokenProvider{p}.GetClaim("res_tenant") require.ErrorIs(t, err, errOCIForbidden) - require.Equal(t, 1, p.fetches) + require.EqualValues(t, 1, p.fetches.Load()) }) } @@ -319,6 +321,45 @@ func TestOCIBuildsClientOnFirstCall(t *testing.T) { require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{})) require.EqualValues(t, 1, requests.Load()) }) + + t.Run("a delete checks once for all its paths", func(t *testing.T) { + provider := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, 10*ociTokenAttempts)...) + s := lazy(provider) + + paths := make([]string, 40) + for i := range paths { + paths[i] = strconv.Itoa(i) + } + var failed *DeleteError + require.ErrorAs(t, s.DeleteObjects(ctx, paths), &failed) + require.Len(t, failed.Failed, len(paths)) + require.EqualValues(t, ociTokenAttempts, provider.fetches.Load()) + }) + + t.Run("calls that leave start no build of their own", func(t *testing.T) { + provider := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, 10*ociTokenAttempts)...) + provider.gate = make(chan struct{}) + s := lazy(provider) + + errs := make([]error, 5) + var wg sync.WaitGroup + for i := range errs { + wg.Go(func() { + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + errs[i] = s.PutObject(short, "k", []byte("hello"), PutOptions{}) + }) + } + wg.Wait() + for _, err := range errs { + require.ErrorIs(t, err, context.DeadlineExceeded) + } + + // The one build they shared fails after its attempts, and none follows. + close(provider.gate) + require.Eventually(t, func() bool { return provider.fetches.Load() == ociTokenAttempts }, 5*time.Second, 5*time.Millisecond) + require.Never(t, func() bool { return provider.fetches.Load() > ociTokenAttempts }, 200*time.Millisecond, 10*time.Millisecond) + }) } // A namespace the config leaves out is looked up by the first call, so NewOCI @@ -353,3 +394,91 @@ func TestOCILooksUpNamespaceOnFirstCall(t *testing.T) { require.NoError(t, s.PutObject(context.Background(), "k", []byte("hello"), PutOptions{})) require.Equal(t, []string{"GET /n", "PUT /n/ns/b/bucket/o/k"}, requests()) } + +// hangUp reads the request, then closes its connection with no answer. +func hangUp(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if c, _, err := w.(http.Hijacker).Hijack(); err == nil { + _ = c.Close() + } +} + +// The policy the SDK is configured with stays in effect: retries switched off +// stay off, and a global policy keeps its attempts, with a closed connection +// among what it retries. +func TestOCIKeepsConfiguredRetries(t *testing.T) { + t.Setenv(common.UsingExpectHeaderEnvVar, "false") + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + hangUp(w, r) + })) + t.Cleanup(srv.Close) + ctx := context.Background() + + t.Run("retries switched off", func(t *testing.T) { + t.Setenv("OCI_SDK_DEFAULT_RETRY_ENABLED", "false") + requests.Store(0) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.ErrorIs(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{}), io.EOF) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("a global policy", func(t *testing.T) { + twice := common.NewRetryPolicy(2, common.DefaultShouldRetryOperation, func(common.OCIOperationResponse) time.Duration { return 0 }) + common.GlobalRetry = &twice + t.Cleanup(func() { common.GlobalRetry = nil }) + requests.Store(0) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.ErrorIs(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{}), io.EOF) + require.EqualValues(t, 2, requests.Load()) + }) +} + +// A pre-authenticated request that got no answer may have been created, so +// it is not sent again; one the server refused is. +func TestOCIPresignedURLRetriesOnlyAnswered(t *testing.T) { + var requests atomic.Int32 + var answer atomic.Value // func(http.ResponseWriter, *http.Request, int32) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + answer.Load().(func(http.ResponseWriter, *http.Request, int32))(w, r, requests.Add(1)) + })) + t.Cleanup(srv.Close) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + ctx := context.Background() + + t.Run("no answer", func(t *testing.T) { + requests.Store(0) + answer.Store(func(w http.ResponseWriter, r *http.Request, _ int32) { hangUp(w, r) }) + + _, err := s.PresignedURL(ctx, "k", time.Hour) + require.ErrorIs(t, err, io.EOF) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("refused, then created", func(t *testing.T) { + requests.Store(0) + answer.Store(func(w http.ResponseWriter, r *http.Request, n int32) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"code":"ServiceUnavailable","message":"busy"}`) + return + } + _, _ = io.WriteString(w, `{"accessUri":"/p/token/n/ns/b/bucket/o/k"}`) + }) + + u, err := s.PresignedURL(ctx, "k", time.Hour) + require.NoError(t, err) + require.Equal(t, srv.URL+"/p/token/n/ns/b/bucket/o/k", u) + require.EqualValues(t, 2, requests.Load()) + }) +} From 81363d37790f42b81edaf4a7dfc5ceba771b055e Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Wed, 11 Nov 2026 12:23:07 +0100 Subject: [PATCH 4/4] OCI: leave a one-attempt retry policy as the SDK built it The closed connection cases were added to whichever policy was in effect, a one-attempt policy included. The SDK asks a policy whether to retry before it counts attempts, and with one attempt it never marks the request body as rewindable. So with retries switched off, an upload whose connection closed came back as "Unable to perform Retry on this request body type", an error that does not unwrap to the one behind it. A policy of one attempt is now returned unchanged. Co-Authored-By: Claude Fable 5.2 --- oci.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oci.go b/oci.go index ac17667..398bfb5 100644 --- a/oci.go +++ b/oci.go @@ -205,6 +205,11 @@ func ociRetryPolicy(configured *common.RetryPolicy, change func(ociPredicate) oc if configured != nil { policy = *configured } + // A policy of one attempt stays as it is: the SDK asks it about a retry it + // will not make, and on a yes reports the body as the failure. + if policy.MaximumNumberAttempts == 1 { + return policy + } policy.ShouldRetryOperation = change(policy.ShouldRetryOperation) if policy.NonEventuallyConsistentPolicy != nil { inner := *policy.NonEventuallyConsistentPolicy