diff --git a/README.md b/README.md index 1f31a2c..60852f0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does. It tries every path, and when some are not deleted fails with a `*DeleteError` whose `Failed` names each one with its error (`errors.As`); every other path is deleted, and a retry is safe. - **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. - **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut. An HTTP/2 connection with no inbound frame for 30 seconds is pinged and cut when the ping goes unanswered for 15 seconds, so a dead peer fails every request on it within 45 seconds with an error the SDKs retry. The ping waits behind the frame being written, so a peer accepting less than one 16 KiB frame per 15 seconds is cut as well; any faster transfer, however slow, never is. The context bounds the call. -- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. A stream given to `UploadObject` is sent once on AliOSS. +- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. GCS also retries the errors with which Go's HTTP/2 client ends every request on a connection it is closing (force closed, or the server's GOAWAY), which the SDK does not. OCI also retries a request whose connection the server closed before answering, which the SDK does not, and a workload identity token fetch that fails, which the SDK makes once. It adds these to the retry policy the SDK is configured with (`OCI_SDK_DEFAULT_RETRY_ENABLED`, `common.GlobalRetry`), whose attempts and backoff stay. `PresignedURL` on OCI is sent again only when the server answered: a request with no answer may have made its grant. A stream given to `UploadObject` is sent once on AliOSS. +- **OCI setup**: `NewOCI` makes no request. With workload identity the first call builds the client, which fetches its token, and a namespace the config leaves out is looked up by the first call too. Calls waiting on that share one attempt. A failure there fails those calls, each bounded by its context, and a later call tries again. - **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. - **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. diff --git a/oci.go b/oci.go index 77508ff..398bfb5 100644 --- a/oci.go +++ b/oci.go @@ -17,41 +17,112 @@ package storage import ( "bytes" "context" + "crypto/rsa" "crypto/tls" "errors" "fmt" "io" + "math/rand/v2" "net/http" "net/url" "os" "path" "slices" "strings" + "sync/atomic" "time" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/common/auth" "github.com/oracle/oci-go-sdk/v65/objectstorage" + "golang.org/x/sync/singleflight" ) type ociStorage struct { - conf *OCIConfig - client objectstorage.ObjectStorageClient - namespace string - host string + conf *OCIConfig + provider common.ConfigurationProvider + + // connected is set once the fields below are, by connect, which + // connecting runs one at a time. + connected atomic.Bool + connecting singleflight.Group + built bool + client objectstorage.ObjectStorageClient + // grantRetry is the policy of a call that leaves a grant behind. + grantRetry common.RetryPolicy + namespace string + host string // insecure is an http:// Endpoint, such as an emulator's. insecure bool } +// NewOCI makes no request. With workload identity the SDK fetches a token as +// its client is built, so the first call builds it. A namespace the config +// leaves out is looked up by the first call as well. func NewOCI(conf *OCIConfig) (Storage, error) { provider, err := newConfigProvider(conf) if err != nil { return nil, err } - client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) + s := &ociStorage{conf: conf, provider: provider, namespace: conf.Namespace} + if !conf.UseWorkloadIdentity { + if err = s.build(); err != nil { + return nil, err + } + s.connected.Store(s.namespace != "") + } + return s, nil +} + +// ready has the client built and the namespace known. The calls waiting on +// that share one connect, and a failure is not kept, so a later call starts +// another. A call whose ctx ends is released and leaves nothing running of its +// own: the connect goes on for the calls still waiting, and for the next. +func (s *ociStorage) ready(ctx context.Context) error { + if s.connected.Load() { + return nil + } + done := s.connecting.DoChan("", func() (any, error) { return nil, s.connect() }) + select { + case r := <-done: + return r.Err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *ociStorage) connect() error { + if s.connected.Load() { + return nil + } + + if !s.built { + if err := s.build(); err != nil { + return err + } + } + if s.namespace == "" { + req := objectstorage.GetNamespaceRequest{} + if s.conf.CompartmentID != "" { + req.CompartmentId = &s.conf.CompartmentID + } + namespace, err := lookupNamespace(s.client, req) + if err != nil { + return err + } + s.namespace = namespace + } + s.connected.Store(true) + return nil +} + +// build makes the SDK client, which with workload identity fetches a token. +func (s *ociStorage) build() error { + conf := s.conf + client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(s.provider) if err != nil { - return nil, err + return err } if conf.Region != "" { @@ -80,6 +151,10 @@ func NewOCI(conf *OCIConfig) (Storage, error) { } client.HTTPClient = c } + // The SDK has set the policy the environment or common.GlobalRetry asks for. + configured := client.Configuration.RetryPolicy + policy := ociRetryPolicy(configured, ociAlsoClosed) + client.Configuration.RetryPolicy = &policy // client.Host keeps endpoint options such as {dualStack?ds.oci.:}, which the // SDK resolves on a copy per request, so resolve them the same way here. @@ -89,25 +164,9 @@ func NewOCI(conf *OCIConfig) (Storage, error) { insecure := strings.HasPrefix(host, "http://") host = strings.TrimPrefix(host, "http://") - s := &ociStorage{ - conf: conf, - client: client, - namespace: conf.Namespace, - host: host, - insecure: insecure, - } - - if s.namespace == "" { - req := objectstorage.GetNamespaceRequest{} - if conf.CompartmentID != "" { - req.CompartmentId = &conf.CompartmentID - } - if s.namespace, err = lookupNamespace(client, req); err != nil { - return nil, err - } - } - - return s, nil + s.client, s.host, s.insecure, s.built = client, host, insecure, true + s.grantRetry = ociRetryPolicy(configured, ociOnlyAnswered) + return nil } // lookupNamespace is the tenancy's namespace, or an error after a minute, as @@ -135,9 +194,128 @@ func lookupNamespace(client objectstorage.ObjectStorageClient, req objectstorage } } +type ociPredicate = func(common.OCIOperationResponse) bool + +// ociRetryPolicy is the policy in effect, configured or else the SDK's +// default, with its predicate changed by change: its attempts and backoff stay. +// The SDK runs the inner policy outside an eventual-consistency window, so +// the predicates of both change. +func ociRetryPolicy(configured *common.RetryPolicy, change func(ociPredicate) ociPredicate) common.RetryPolicy { + policy := common.DefaultRetryPolicy() + if configured != nil { + policy = *configured + } + // A policy of one attempt stays as it is: the SDK asks it about a retry it + // will not make, and on a yes reports the body as the failure. + if policy.MaximumNumberAttempts == 1 { + return policy + } + policy.ShouldRetryOperation = change(policy.ShouldRetryOperation) + if policy.NonEventuallyConsistentPolicy != nil { + inner := *policy.NonEventuallyConsistentPolicy + inner.ShouldRetryOperation = change(inner.ShouldRetryOperation) + policy.NonEventuallyConsistentPolicy = &inner + } + return policy +} + +// ociAlsoClosed adds to retry a connection the server closed under the +// request, which the SDK matches only when nothing wraps the error. Such a +// request may have been received, as with a timeout the SDK retries. +func ociAlsoClosed(retry ociPredicate) ociPredicate { + return func(r common.OCIOperationResponse) bool { + if retry(r) { + return true + } + err := r.Error + if err == nil { + return false + } + return errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + strings.Contains(err.Error(), "http: server closed idle connection") + } +} + +// ociOnlyAnswered keeps of retry the failures the server answered with. A +// request that got no answer may have been carried out, and one sent again +// leaves a second grant, the first with a URI nobody was given. +func ociOnlyAnswered(retry ociPredicate) ociPredicate { + return func(r common.OCIOperationResponse) bool { + _, answered := common.IsServiceError(r.Error) + return answered && retry(r) + } +} + +const ( + ociTokenAttempts = 5 + // A fetch that fails this long after the first began has timed out, and + // is not repeated. + ociTokenRetryWindow = 15 * time.Second +) + +// ociTokenBackoff is the wait before the first retry of a token fetch. Each +// later wait doubles it, and each is jittered. +var ociTokenBackoff = 250 * time.Millisecond + +// ociTokenProvider retries the workload identity token fetch, which the SDK +// makes once, inside whichever of these calls finds no valid token. +type ociTokenProvider struct { + auth.ConfigurationProviderWithClaimAccess +} + +func (p ociTokenProvider) TenancyOCID() (string, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.TenancyOCID) +} + +func (p ociTokenProvider) KeyID() (string, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.KeyID) +} + +func (p ociTokenProvider) PrivateRSAKey() (*rsa.PrivateKey, error) { + return retryOCIToken(p.ConfigurationProviderWithClaimAccess.PrivateRSAKey) +} + +func (p ociTokenProvider) GetClaim(key string) (interface{}, error) { + return retryOCIToken(func() (interface{}, error) { + return p.ConfigurationProviderWithClaimAccess.GetClaim(key) + }) +} + +// Refreshable keeps the SDK's retry of a 401 on a token it can renew. +func (p ociTokenProvider) Refreshable() bool { + r, ok := p.ConfigurationProviderWithClaimAccess.(common.RefreshableConfigurationProvider) + return ok && r.Refreshable() +} + +// The SDK asks for a token with no context, so the fetch of a call that has +// ended runs on, for under 4s. +func retryOCIToken[T any](fetch func() (T, error)) (T, error) { + start := time.Now() + backoff := ociTokenBackoff + for attempt := 1; ; attempt++ { + v, err := fetch() + if err == nil || attempt == ociTokenAttempts || !ociTokenRetryable(err) || time.Since(start) > ociTokenRetryWindow { + return v, err + } + time.Sleep(backoff/2 + rand.N(backoff/2+1)) + backoff *= 2 + } +} + +// ociTokenRetryable is any failure but the 403 of a cluster without workload +// identity. The SDK flattens the response's status into the error's text. +func ociTokenRetryable(err error) bool { + return !strings.Contains(err.Error(), "Status: 403") +} + func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { if conf.UseWorkloadIdentity { - return auth.OkeWorkloadIdentityConfigurationProvider() + provider, err := auth.OkeWorkloadIdentityConfigurationProvider() + if err != nil { + return nil, err + } + return ociTokenProvider{provider}, nil } if conf.TenancyOCID != "" && conf.UserOCID != "" && conf.PrivateKey != "" { @@ -201,12 +379,16 @@ func (s *ociStorage) DownloadFile(ctx context.Context, storagePath, filepath str // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + if err := s.ready(ctx); err != nil { + return "", err + } name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} resp, err := s.client.CreatePreauthenticatedRequest(ctx, objectstorage.CreatePreauthenticatedRequestRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + RequestMetadata: common.RequestMetadata{RetryPolicy: &s.grantRetry}, CreatePreauthenticatedRequestDetails: objectstorage.CreatePreauthenticatedRequestDetails{ Name: &name, AccessType: objectstorage.CreatePreauthenticatedRequestDetailsAccessTypeObjectread, @@ -249,6 +431,9 @@ var ociContentDisposition = common.String("inline") // put sends body in one request. It must be seekable, so the SDK can re-read // it when it retries. func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCloser, size int64, opts PutOptions) error { + if err := s.ready(ctx); err != nil { + return err + } req := objectstorage.PutObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -271,6 +456,9 @@ func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCl // multipart upload of sequential parts through one buffer. IfAbsent is checked // at create and again at commit, so it is atomic here. func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if err := s.ready(ctx); err != nil { + return err + } // The condition is checked again at create and commit; this check spares a // refused upload its traffic, and refuses it where a store ignores // if-none-match on multipart uploads, as floci-oci does. @@ -400,6 +588,9 @@ func (s *ociStorage) DownloadObject(ctx context.Context, storagePath string, w i } func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + if err := s.ready(ctx); err != nil { + return 0, err + } resp, err := s.client.GetObject(ctx, objectstorage.GetObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -415,6 +606,9 @@ func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writ } func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + if err := s.ready(ctx); err != nil { + return false, err + } _, err := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -428,6 +622,9 @@ func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool } func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + if err := s.ready(ctx); err != nil { + return nil, err + } var keys []string var start *string for { @@ -451,7 +648,16 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, } } +// DeleteObjects checks the client once for all its paths, and names every +// path in its error where there is none, as for any path it does not delete. func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + if err := s.ready(ctx); err != nil { + failed := make(map[string]error, len(storagePaths)) + for _, p := range storagePaths { + failed[p] = err + } + return deleteError(failed) + } return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ NamespaceName: &s.namespace, diff --git a/oci_test.go b/oci_test.go index 0d3d4c6..6510d25 100644 --- a/oci_test.go +++ b/oci_test.go @@ -15,13 +15,27 @@ package storage import ( + "context" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" + "errors" + "io" + "net/http" + "net/http/httptest" "net/url" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" "testing" + "time" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" "github.com/stretchr/testify/require" ) @@ -90,3 +104,381 @@ func TestOCILocationHost(t *testing.T) { require.NoError(t, err) require.Equal(t, "objectstorage.us-chicago-1.oraclecloud.com", u.Host) } + +// ociTestConfig is API key credentials for endpoint. A set Namespace keeps +// NewOCI off the network. +func ociTestConfig(t *testing.T, endpoint string) *OCIConfig { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + return &OCIConfig{ + TenancyOCID: "ocid1.tenancy.oc1..tenancy", + UserOCID: "ocid1.user.oc1..user", + Fingerprint: "aa:bb:cc", + PrivateKey: string(pemKey), + Region: "us-chicago-1", + Namespace: "ns", + Bucket: "bucket", + Endpoint: endpoint, + } +} + +// A request whose connection the server closes before answering is sent +// again, where the SDK's own predicate leaves it failed. +func TestOCIRetriesClosedConnection(t *testing.T) { + t.Run("upload", func(t *testing.T) { + // Without the header the server writes nothing before it closes, so + // the client reads a bare EOF. + t.Setenv(common.UsingExpectHeaderEnvVar, "false") + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if requests.Add(1) > 1 { + return + } + if c, _, err := w.(http.Hijacker).Hijack(); err == nil { + _ = c.Close() + } + })) + t.Cleanup(srv.Close) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.NoError(t, s.PutObject(context.Background(), "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 2, requests.Load()) + }) + + put := func(err error) common.OCIOperationResponse { + return common.OCIOperationResponse{ + Error: &url.Error{Op: "Put", URL: "https://objectstorage.us-chicago-1.oraclecloud.com/n/ns/b/bucket/o/k", Err: err}, + } + } + + t.Run("errors of a closed connection", func(t *testing.T) { + for _, err := range []error{ + io.EOF, + io.ErrUnexpectedEOF, + errors.New("http: server closed idle connection"), + } { + r := put(err) + require.False(t, common.DefaultShouldRetryOperation(r), "the SDK's predicate, which this package adds to: %v", err) + require.True(t, ociAlsoClosed(common.DefaultShouldRetryOperation)(r), "%v", err) + } + }) + + t.Run("unchanged otherwise", func(t *testing.T) { + retry := ociAlsoClosed(common.DefaultShouldRetryOperation) + require.True(t, retry(put(syscall.ECONNRESET))) + require.False(t, retry(put(errors.New("x509: certificate signed by unknown authority")))) + require.False(t, retry(common.OCIOperationResponse{Error: &ociServiceError{status: http.StatusNotFound}})) + }) +} + +// flakyTokenProvider fails its token-backed calls with failures, in turn, then +// serves them as the API key provider it wraps. +type flakyTokenProvider struct { + common.ConfigurationProvider + failures []error + fetches atomic.Int32 + // gate, when set, holds every fetch until it is closed. + gate chan struct{} +} + +func flakyProvider(t *testing.T, failures ...error) *flakyTokenProvider { + keys, err := newConfigProvider(ociTestConfig(t, "")) + require.NoError(t, err) + return &flakyTokenProvider{ConfigurationProvider: keys, failures: failures} +} + +func (p *flakyTokenProvider) fetch() error { + if p.gate != nil { + <-p.gate + } + p.fetches.Add(1) + if len(p.failures) == 0 { + return nil + } + err := p.failures[0] + p.failures = p.failures[1:] + return err +} + +func (p *flakyTokenProvider) TenancyOCID() (string, error) { + if err := p.fetch(); err != nil { + return "", err + } + return p.ConfigurationProvider.TenancyOCID() +} + +func (p *flakyTokenProvider) KeyID() (string, error) { + if err := p.fetch(); err != nil { + return "", err + } + return p.ConfigurationProvider.KeyID() +} + +func (p *flakyTokenProvider) PrivateRSAKey() (*rsa.PrivateKey, error) { + if err := p.fetch(); err != nil { + return nil, err + } + return p.ConfigurationProvider.PrivateRSAKey() +} + +func (p *flakyTokenProvider) GetClaim(string) (interface{}, error) { + return nil, p.fetch() +} + +func (p *flakyTokenProvider) Refreshable() bool { return true } + +// The SDK's token fetch errors, which carry the response's status only as text. +var ( + errOCIThrottled = errors.New("failed to renew security token: failed to get security token: failed to get a Workload Identity token. Status: 429 Too Many Requests, Message: . Please contact OKE team") + errOCIForbidden = errors.New("failed to renew security token: failed to get security token: please ensure the cluster type is enhanced: Status: 403 Forbidden, Message: ") +) + +// noOCITokenBackoff has a token fetch retried at once. +func noOCITokenBackoff(t *testing.T) { + backoff := ociTokenBackoff + ociTokenBackoff = 0 + t.Cleanup(func() { ociTokenBackoff = backoff }) +} + +// A workload identity token fetch that is throttled is made again, so a client +// is built where the SDK alone fails on the first refusal. +func TestOCIRetriesTokenFetch(t *testing.T) { + noOCITokenBackoff(t) + + t.Run("a client is built over throttled fetches", func(t *testing.T) { + _, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(flakyProvider(t, errOCIThrottled)) + require.Error(t, err, "the SDK's one fetch, which this package repeats") + + p := flakyProvider(t, errOCIThrottled, errOCIThrottled) + client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(ociTokenProvider{p}) + require.NoError(t, err) + require.Empty(t, p.failures) + require.True(t, client.IsRefreshableAuthType()) + }) + + t.Run("gives up after its attempts", func(t *testing.T) { + p := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, ociTokenAttempts+1)...) + _, err := ociTokenProvider{p}.KeyID() + require.ErrorIs(t, err, errOCIThrottled) + require.EqualValues(t, ociTokenAttempts, p.fetches.Load()) + }) + + t.Run("a cluster without workload identity fails at once", func(t *testing.T) { + p := flakyProvider(t, errOCIForbidden, errOCIForbidden) + _, err := ociTokenProvider{p}.GetClaim("res_tenant") + require.ErrorIs(t, err, errOCIForbidden) + require.EqualValues(t, 1, p.fetches.Load()) + }) +} + +// With workload identity the first call builds the client, not NewOCI: a +// build that fails is that call's failure, and the next call builds again. +func TestOCIBuildsClientOnFirstCall(t *testing.T) { + noOCITokenBackoff(t) + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + requests.Add(1) + })) + t.Cleanup(srv.Close) + + // The store NewOCI makes with workload identity, over provider. + lazy := func(provider *flakyTokenProvider) *ociStorage { + conf := ociTestConfig(t, srv.URL) + return &ociStorage{conf: conf, provider: ociTokenProvider{provider}, namespace: conf.Namespace} + } + ctx := context.Background() + + t.Run("a failed build is tried again", func(t *testing.T) { + requests.Store(0) + s := lazy(flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, ociTokenAttempts)...)) + + err := s.PutObject(ctx, "k", []byte("hello"), PutOptions{}) + require.ErrorContains(t, err, "429 Too Many Requests") + require.Zero(t, requests.Load()) + + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("a call is released when its context ends", func(t *testing.T) { + requests.Store(0) + provider := flakyProvider(t) + provider.gate = make(chan struct{}) + s := lazy(provider) + + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + require.ErrorIs(t, s.PutObject(short, "k", []byte("hello"), PutOptions{}), context.DeadlineExceeded) + + // The build the first call began goes on, and serves the next. + close(provider.gate) + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{})) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("a delete checks once for all its paths", func(t *testing.T) { + provider := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, 10*ociTokenAttempts)...) + s := lazy(provider) + + paths := make([]string, 40) + for i := range paths { + paths[i] = strconv.Itoa(i) + } + var failed *DeleteError + require.ErrorAs(t, s.DeleteObjects(ctx, paths), &failed) + require.Len(t, failed.Failed, len(paths)) + require.EqualValues(t, ociTokenAttempts, provider.fetches.Load()) + }) + + t.Run("calls that leave start no build of their own", func(t *testing.T) { + provider := flakyProvider(t, slices.Repeat([]error{errOCIThrottled}, 10*ociTokenAttempts)...) + provider.gate = make(chan struct{}) + s := lazy(provider) + + errs := make([]error, 5) + var wg sync.WaitGroup + for i := range errs { + wg.Go(func() { + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + errs[i] = s.PutObject(short, "k", []byte("hello"), PutOptions{}) + }) + } + wg.Wait() + for _, err := range errs { + require.ErrorIs(t, err, context.DeadlineExceeded) + } + + // The one build they shared fails after its attempts, and none follows. + close(provider.gate) + require.Eventually(t, func() bool { return provider.fetches.Load() == ociTokenAttempts }, 5*time.Second, 5*time.Millisecond) + require.Never(t, func() bool { return provider.fetches.Load() > ociTokenAttempts }, 200*time.Millisecond, 10*time.Millisecond) + }) +} + +// A namespace the config leaves out is looked up by the first call, so NewOCI +// makes no request. +func TestOCILooksUpNamespaceOnFirstCall(t *testing.T) { + var mu sync.Mutex + var seen []string + requests := func() []string { + mu.Lock() + defer mu.Unlock() + return slices.Clone(seen) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + path := strings.TrimSuffix(r.URL.Path, "/") + mu.Lock() + seen = append(seen, r.Method+" "+path) + mu.Unlock() + if path == "/n" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `"ns"`) + } + })) + t.Cleanup(srv.Close) + + conf := ociTestConfig(t, srv.URL) + conf.Namespace = "" + s, err := NewOCI(conf) + require.NoError(t, err) + require.Empty(t, requests()) + + require.NoError(t, s.PutObject(context.Background(), "k", []byte("hello"), PutOptions{})) + require.Equal(t, []string{"GET /n", "PUT /n/ns/b/bucket/o/k"}, requests()) +} + +// hangUp reads the request, then closes its connection with no answer. +func hangUp(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if c, _, err := w.(http.Hijacker).Hijack(); err == nil { + _ = c.Close() + } +} + +// The policy the SDK is configured with stays in effect: retries switched off +// stay off, and a global policy keeps its attempts, with a closed connection +// among what it retries. +func TestOCIKeepsConfiguredRetries(t *testing.T) { + t.Setenv(common.UsingExpectHeaderEnvVar, "false") + + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + hangUp(w, r) + })) + t.Cleanup(srv.Close) + ctx := context.Background() + + t.Run("retries switched off", func(t *testing.T) { + t.Setenv("OCI_SDK_DEFAULT_RETRY_ENABLED", "false") + requests.Store(0) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.ErrorIs(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{}), io.EOF) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("a global policy", func(t *testing.T) { + twice := common.NewRetryPolicy(2, common.DefaultShouldRetryOperation, func(common.OCIOperationResponse) time.Duration { return 0 }) + common.GlobalRetry = &twice + t.Cleanup(func() { common.GlobalRetry = nil }) + requests.Store(0) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + require.ErrorIs(t, s.PutObject(ctx, "k", []byte("hello"), PutOptions{}), io.EOF) + require.EqualValues(t, 2, requests.Load()) + }) +} + +// A pre-authenticated request that got no answer may have been created, so +// it is not sent again; one the server refused is. +func TestOCIPresignedURLRetriesOnlyAnswered(t *testing.T) { + var requests atomic.Int32 + var answer atomic.Value // func(http.ResponseWriter, *http.Request, int32) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + answer.Load().(func(http.ResponseWriter, *http.Request, int32))(w, r, requests.Add(1)) + })) + t.Cleanup(srv.Close) + + s, err := NewOCI(ociTestConfig(t, srv.URL)) + require.NoError(t, err) + ctx := context.Background() + + t.Run("no answer", func(t *testing.T) { + requests.Store(0) + answer.Store(func(w http.ResponseWriter, r *http.Request, _ int32) { hangUp(w, r) }) + + _, err := s.PresignedURL(ctx, "k", time.Hour) + require.ErrorIs(t, err, io.EOF) + require.EqualValues(t, 1, requests.Load()) + }) + + t.Run("refused, then created", func(t *testing.T) { + requests.Store(0) + answer.Store(func(w http.ResponseWriter, r *http.Request, n int32) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"code":"ServiceUnavailable","message":"busy"}`) + return + } + _, _ = io.WriteString(w, `{"accessUri":"/p/token/n/ns/b/bucket/o/k"}`) + }) + + u, err := s.PresignedURL(ctx, "k", time.Hour) + require.NoError(t, err) + require.Equal(t, srv.URL+"/p/token/n/ns/b/bucket/o/k", u) + require.EqualValues(t, 2, requests.Load()) + }) +}