Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -533,8 +533,7 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload
}
parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag})

n, err = io.ReadFull(body, part)
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
if n, err = fillPart(body, part); err != nil {
return err
}
}
Expand All @@ -550,6 +549,25 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload
return wrapOCIObjectError(err)
}

// fillPart reads body into part until part is full or body ends with io.EOF.
// Any other error fails the upload, io.ErrUnexpectedEOF included: it is the
// body's failure, not its end, and committing what was read would publish a
// prefix as the whole object.
func fillPart(body io.Reader, part []byte) (int, error) {
n := 0
for n < len(part) {
m, err := body.Read(part[n:])
n += m
if err == io.EOF {
return n, nil
}
if err != nil {
return n, err
}
}
return n, nil
}

func (s *ociStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) {
return getObject(ctx, s, storagePath)
}
Expand Down
90 changes: 90 additions & 0 deletions oci_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
package storage

import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
Expand All @@ -32,6 +34,7 @@ import (
"sync/atomic"
"syscall"
"testing"
"testing/iotest"
"time"

"github.com/oracle/oci-go-sdk/v65/common"
Expand Down Expand Up @@ -482,3 +485,90 @@ func TestOCIPresignedURLRetriesOnlyAnswered(t *testing.T) {
require.EqualValues(t, 2, requests.Load())
})
}

// ociMultipartServer answers OCI's multipart upload calls, keeping the bytes
// of each part and of each committed object.
type ociMultipartServer struct {
mu sync.Mutex
parts map[int][]byte
committed map[string][]byte
}

func newOCIMultipartServer(t *testing.T) (*ociMultipartServer, string) {
m := &ociMultipartServer{parts: map[int][]byte{}, committed: map[string][]byte{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
object, upload := strings.CutPrefix(r.URL.Path, "/n/ns/b/bucket/u/")
m.mu.Lock()
defer m.mu.Unlock()
switch {
case r.Method == http.MethodPost && r.URL.Path == "/n/ns/b/bucket/u":
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"uploadId":"up","namespace":"ns","bucket":"bucket","object":"k"}`)
case r.Method == http.MethodPut && upload:
num, _ := strconv.Atoi(r.URL.Query().Get("uploadPartNum"))
m.parts[num] = body
w.Header().Set("ETag", "etag-"+strconv.Itoa(num))
case r.Method == http.MethodPost && upload:
var whole []byte
for num := 1; num <= len(m.parts); num++ {
whole = append(whole, m.parts[num]...)
}
m.committed[object] = whole
case r.Method == http.MethodDelete && upload:
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotImplemented)
}
}))
t.Cleanup(srv.Close)
return m, srv.URL
}

// failingAfter yields n bytes of data, then fails with err.
type failingAfter struct {
n int
err error
}

func (r *failingAfter) Read(p []byte) (int, error) {
if r.n == 0 {
return 0, r.err
}
n := min(len(p), r.n)
r.n -= n
return n, nil
}

// A body that fails partway through a later part fails the upload, nothing
// committed, also with an error wrapping io.ErrUnexpectedEOF, as a body
// streamed from a download cut short does: only io.EOF ends a body.
func TestOCIMultipartBodyError(t *testing.T) {
t.Run("fails, nothing committed", func(t *testing.T) {
m, endpoint := newOCIMultipartServer(t)
s, err := NewOCI(ociTestConfig(t, endpoint))
require.NoError(t, err)

cut := fmt.Errorf("download base: %w", io.ErrUnexpectedEOF)
body := &failingAfter{n: ociUploadPartSize + 1<<20, err: cut}
err = s.UploadObject(context.Background(), "k", body, PutOptions{})
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
m.mu.Lock()
defer m.mu.Unlock()
require.Empty(t, m.committed)
})

t.Run("a short last part read in pieces commits the whole body", func(t *testing.T) {
m, endpoint := newOCIMultipartServer(t)
s, err := NewOCI(ociTestConfig(t, endpoint))
require.NoError(t, err)

data := make([]byte, ociUploadPartSize+3<<20)
_, _ = rand.Read(data)
err = s.UploadObject(context.Background(), "k", iotest.HalfReader(bytes.NewReader(data)), PutOptions{})
require.NoError(t, err)
m.mu.Lock()
defer m.mu.Unlock()
require.Equal(t, data, m.committed["k"])
})
}
Loading