Fix high memory consumption when copying files to S3

http/handler/api/filesystems.go

@@ -8,6 +8,7 @@ import (
 	"github.com/datarhei/core/v16/http/api"
 	"github.com/datarhei/core/v16/http/handler"
 	"github.com/datarhei/core/v16/http/handler/util"
+	"github.com/datarhei/core/v16/io/fs"
 
 	"github.com/fujiwara/shapeio"
 	"github.com/labstack/echo/v4"
@@ -244,6 +245,11 @@ func (h *FSHandler) FileOperation(c echo.Context) error {
 
 	defer fromFile.Close()
 
+	fromFileStat, err := fromFile.Stat()
+	if err != nil {
+		return api.Err(http.StatusBadRequest, "Source files with unknown size", "%s", fromFSName)
+	}
+
 	var reader io.Reader = fromFile
 
 	if operation.RateLimit != 0 {
@@ -254,7 +260,10 @@ func (h *FSHandler) FileOperation(c echo.Context) error {
 		reader = shapedReader
 	}
 
-	_, _, err := toFS.Handler.FS.Filesystem.WriteFileReader(toPath, reader)
+	// In case the target is S3, allow it to determine the size of the file
+	sizer := fs.NewReadSizer(reader, fromFileStat.Size())
+
+	_, _, err = toFS.Handler.FS.Filesystem.WriteFileReader(toPath, sizer)
 	if err != nil {
 		toFS.Handler.FS.Filesystem.Remove(toPath)
 		return api.Err(http.StatusBadRequest, "Writing target file failed", "%s", err)

io/fs/mem_test.go

@@ -26,5 +26,6 @@ func TestMemFromDir(t *testing.T) {
 		"/s3.go",
 		"/sized_test.go",
 		"/sized.go",
+		"/sizer.go",
 	}, names)
 }

io/fs/s3.go

@@ -296,7 +296,13 @@ func (fs *s3Filesystem) write(path string, r io.Reader) (int64, bool, error) {
 		overwrite = true
 	}
 
-	info, err := fs.client.PutObject(ctx, fs.bucket, path, r, -1, minio.PutObjectOptions{
+	var size int64 = -1
+	sizer, ok := r.(Sizer)
+	if ok {
+		size = sizer.Size()
+	}
+
+	info, err := fs.client.PutObject(ctx, fs.bucket, path, r, size, minio.PutObjectOptions{
 		UserMetadata:            map[string]string{},
 		UserTags:                map[string]string{},
 		Progress:                nil,
@@ -337,11 +343,13 @@ func (fs *s3Filesystem) WriteFileReader(path string, r io.Reader) (int64, bool,
 }
 
 func (fs *s3Filesystem) WriteFile(path string, data []byte) (int64, bool, error) {
-	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+	rs := NewReadSizer(bytes.NewBuffer(data), int64(len(data)))
+	return fs.WriteFileReader(path, rs)
 }
 
 func (fs *s3Filesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
-	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+	rs := NewReadSizer(bytes.NewBuffer(data), int64(len(data)))
+	return fs.WriteFileReader(path, rs)
 }
 
 func (fs *s3Filesystem) Rename(src, dst string) error {

io/fs/sizer.go

@@ -0,0 +1,36 @@
+package fs
+
+import "io"
+
+// Sizer interface can decorate a Reader
+type Sizer interface {
+	// Size returns the size of the object
+	Size() int64
+}
+
+type ReadSizer interface {
+	io.Reader
+	Sizer
+}
+
+type readSizer struct {
+	r    io.Reader
+	size int64
+}
+
+func NewReadSizer(r io.Reader, size int64) ReadSizer {
+	rs := &readSizer{
+		r:    r,
+		size: size,
+	}
+
+	return rs
+}
+
+func (rs *readSizer) Read(p []byte) (int, error) {
+	return rs.r.Read(p)
+}
+
+func (rs *readSizer) Size() int64 {
+	return rs.size
+}