diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go index 328c601d..12440b3b 100644 --- a/http/handler/api/filesystems.go +++ b/http/handler/api/filesystems.go @@ -8,6 +8,7 @@ import ( "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/handler" "github.com/datarhei/core/v16/http/handler/util" + "github.com/datarhei/core/v16/io/fs" "github.com/fujiwara/shapeio" "github.com/labstack/echo/v4" @@ -244,6 +245,11 @@ func (h *FSHandler) FileOperation(c echo.Context) error { defer fromFile.Close() + fromFileStat, err := fromFile.Stat() + if err != nil { + return api.Err(http.StatusBadRequest, "Source files with unknown size", "%s", fromFSName) + } + var reader io.Reader = fromFile if operation.RateLimit != 0 { @@ -254,7 +260,10 @@ func (h *FSHandler) FileOperation(c echo.Context) error { reader = shapedReader } - _, _, err := toFS.Handler.FS.Filesystem.WriteFileReader(toPath, reader) + // In case the target is S3, allow it to determine the size of the file + sizer := fs.NewReadSizer(reader, fromFileStat.Size()) + + _, _, err = toFS.Handler.FS.Filesystem.WriteFileReader(toPath, sizer) if err != nil { toFS.Handler.FS.Filesystem.Remove(toPath) return api.Err(http.StatusBadRequest, "Writing target file failed", "%s", err) diff --git a/io/fs/mem_test.go b/io/fs/mem_test.go index 431d23b4..c3050541 100644 --- a/io/fs/mem_test.go +++ b/io/fs/mem_test.go @@ -26,5 +26,6 @@ func TestMemFromDir(t *testing.T) { "/s3.go", "/sized_test.go", "/sized.go", + "/sizer.go", }, names) } diff --git a/io/fs/s3.go b/io/fs/s3.go index 41000088..20c64599 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -296,7 +296,13 @@ func (fs *s3Filesystem) write(path string, r io.Reader) (int64, bool, error) { overwrite = true } - info, err := fs.client.PutObject(ctx, fs.bucket, path, r, -1, minio.PutObjectOptions{ + var size int64 = -1 + sizer, ok := r.(Sizer) + if ok { + size = sizer.Size() + } + + info, err := fs.client.PutObject(ctx, fs.bucket, path, r, size, minio.PutObjectOptions{ UserMetadata: map[string]string{}, UserTags: map[string]string{}, Progress: nil, @@ -337,11 +343,13 @@ func (fs *s3Filesystem) WriteFileReader(path string, r io.Reader) (int64, bool, } func (fs *s3Filesystem) WriteFile(path string, data []byte) (int64, bool, error) { - return fs.WriteFileReader(path, bytes.NewBuffer(data)) + rs := NewReadSizer(bytes.NewBuffer(data), int64(len(data))) + return fs.WriteFileReader(path, rs) } func (fs *s3Filesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) { - return fs.WriteFileReader(path, bytes.NewBuffer(data)) + rs := NewReadSizer(bytes.NewBuffer(data), int64(len(data))) + return fs.WriteFileReader(path, rs) } func (fs *s3Filesystem) Rename(src, dst string) error { diff --git a/io/fs/sizer.go b/io/fs/sizer.go new file mode 100644 index 00000000..92f74687 --- /dev/null +++ b/io/fs/sizer.go @@ -0,0 +1,36 @@ +package fs + +import "io" + +// Sizer interface can decorate a Reader +type Sizer interface { + // Size returns the size of the object + Size() int64 +} + +type ReadSizer interface { + io.Reader + Sizer +} + +type readSizer struct { + r io.Reader + size int64 +} + +func NewReadSizer(r io.Reader, size int64) ReadSizer { + rs := &readSizer{ + r: r, + size: size, + } + + return rs +} + +func (rs *readSizer) Read(p []byte) (int, error) { + return rs.r.Read(p) +} + +func (rs *readSizer) Size() int64 { + return rs.size +}