From 03d23f67eb0e0d50893dca984d7d2c3b7262a830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Mon, 16 Nov 2026 23:10:05 -0800 Subject: [PATCH] OCI: a multipart upload whose body fails commits nothing --- oci.go | 22 +++++++++++-- oci_test.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/oci.go b/oci.go index 398bfb5..0033784 100644 --- a/oci.go +++ b/oci.go @@ -533,8 +533,7 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload } parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag}) - n, err = io.ReadFull(body, part) - if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + if n, err = fillPart(body, part); err != nil { return err } } @@ -550,6 +549,25 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload return wrapOCIObjectError(err) } +// fillPart reads body into part until part is full or body ends with io.EOF. +// Any other error fails the upload, io.ErrUnexpectedEOF included: it is the +// body's failure, not its end, and committing what was read would publish a +// prefix as the whole object. +func fillPart(body io.Reader, part []byte) (int, error) { + n := 0 + for n < len(part) { + m, err := body.Read(part[n:]) + n += m + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + } + return n, nil +} + func (s *ociStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { return getObject(ctx, s, storagePath) } diff --git a/oci_test.go b/oci_test.go index 6510d25..451c79b 100644 --- a/oci_test.go +++ b/oci_test.go @@ -15,12 +15,14 @@ package storage import ( + "bytes" "context" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -32,6 +34,7 @@ import ( "sync/atomic" "syscall" "testing" + "testing/iotest" "time" "github.com/oracle/oci-go-sdk/v65/common" @@ -482,3 +485,90 @@ func TestOCIPresignedURLRetriesOnlyAnswered(t *testing.T) { require.EqualValues(t, 2, requests.Load()) }) } + +// ociMultipartServer answers OCI's multipart upload calls, keeping the bytes +// of each part and of each committed object. +type ociMultipartServer struct { + mu sync.Mutex + parts map[int][]byte + committed map[string][]byte +} + +func newOCIMultipartServer(t *testing.T) (*ociMultipartServer, string) { + m := &ociMultipartServer{parts: map[int][]byte{}, committed: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + object, upload := strings.CutPrefix(r.URL.Path, "/n/ns/b/bucket/u/") + m.mu.Lock() + defer m.mu.Unlock() + switch { + case r.Method == http.MethodPost && r.URL.Path == "/n/ns/b/bucket/u": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"uploadId":"up","namespace":"ns","bucket":"bucket","object":"k"}`) + case r.Method == http.MethodPut && upload: + num, _ := strconv.Atoi(r.URL.Query().Get("uploadPartNum")) + m.parts[num] = body + w.Header().Set("ETag", "etag-"+strconv.Itoa(num)) + case r.Method == http.MethodPost && upload: + var whole []byte + for num := 1; num <= len(m.parts); num++ { + whole = append(whole, m.parts[num]...) + } + m.committed[object] = whole + case r.Method == http.MethodDelete && upload: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotImplemented) + } + })) + t.Cleanup(srv.Close) + return m, srv.URL +} + +// failingAfter yields n bytes of data, then fails with err. +type failingAfter struct { + n int + err error +} + +func (r *failingAfter) Read(p []byte) (int, error) { + if r.n == 0 { + return 0, r.err + } + n := min(len(p), r.n) + r.n -= n + return n, nil +} + +// A body that fails partway through a later part fails the upload, nothing +// committed, also with an error wrapping io.ErrUnexpectedEOF, as a body +// streamed from a download cut short does: only io.EOF ends a body. +func TestOCIMultipartBodyError(t *testing.T) { + t.Run("fails, nothing committed", func(t *testing.T) { + m, endpoint := newOCIMultipartServer(t) + s, err := NewOCI(ociTestConfig(t, endpoint)) + require.NoError(t, err) + + cut := fmt.Errorf("download base: %w", io.ErrUnexpectedEOF) + body := &failingAfter{n: ociUploadPartSize + 1<<20, err: cut} + err = s.UploadObject(context.Background(), "k", body, PutOptions{}) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + m.mu.Lock() + defer m.mu.Unlock() + require.Empty(t, m.committed) + }) + + t.Run("a short last part read in pieces commits the whole body", func(t *testing.T) { + m, endpoint := newOCIMultipartServer(t) + s, err := NewOCI(ociTestConfig(t, endpoint)) + require.NoError(t, err) + + data := make([]byte, ociUploadPartSize+3<<20) + _, _ = rand.Read(data) + err = s.UploadObject(context.Background(), "k", iotest.HalfReader(bytes.NewReader(data)), PutOptions{}) + require.NoError(t, err) + m.mu.Lock() + defer m.mu.Unlock() + require.Equal(t, data, m.committed["k"]) + }) +}