apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-09-26 20:11:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use abstract filesystem for stores

Browse Source
This commit is contained in:
Ingo Oppermann
2023-02-01 16:09:20 +01:00
parent 49b16f44a8
commit 2a3288ffd0
316 changed files with 13512 additions and 17601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

415
io/fs/disk.go
View File

@@ -1,28 +1,30 @@
package fs
import (
"bytes"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/datarhei/core/v16/glob"
"github.com/datarhei/core/v16/log"
)
// DiskConfig is the config required to create a new disk
// filesystem.
// DiskConfig is the config required to create a new disk filesystem.
type DiskConfig struct {
// Namee is the name of the filesystem
Name string
// For logging, optional
Logger log.Logger
}
// Dir is the path to the directory to observe
Dir string
// Size of the filesystem in bytes
Size int64
// RootedDiskConfig is the config required to create a new rooted disk filesystem.
type RootedDiskConfig struct {
// Root is the path this filesystem is rooted to
Root string
// For logging, optional
Logger log.Logger
@@ -30,8 +32,9 @@ type DiskConfig struct {
// diskFileInfo implements the FileInfo interface
type diskFileInfo struct {
dir string
root string
name string
mode os.FileMode
finfo os.FileInfo
}
@@ -40,31 +43,37 @@ func (fi *diskFileInfo) Name() string {
}
func (fi *diskFileInfo) Size() int64 {
if fi.finfo.IsDir() {
return 0
}
return fi.finfo.Size()
}
func (fi *diskFileInfo) Mode() fs.FileMode {
return fi.mode
}
func (fi *diskFileInfo) ModTime() time.Time {
return fi.finfo.ModTime()
}
func (fi *diskFileInfo) IsLink() (string, bool) {
mode := fi.finfo.Mode()
if mode&os.ModeSymlink == 0 {
if fi.mode&os.ModeSymlink == 0 {
return fi.name, false
}
path, err := os.Readlink(filepath.Join(fi.dir, fi.name))
path, err := os.Readlink(filepath.Join(fi.root, fi.name))
if err != nil {
return fi.name, false
}
path = filepath.Join(fi.dir, path)
if !strings.HasPrefix(path, fi.dir) {
if !strings.HasPrefix(path, fi.root) {
return fi.name, false
}
name := strings.TrimPrefix(path, fi.dir)
name := strings.TrimPrefix(path, fi.root)
if name[0] != os.PathSeparator {
name = string(os.PathSeparator) + name
}
@@ -78,8 +87,9 @@ func (fi *diskFileInfo) IsDir() bool {
// diskFile implements the File interface
type diskFile struct {
dir string
root string
name string
mode os.FileMode
file *os.File
}
@@ -94,8 +104,9 @@ func (f *diskFile) Stat() (FileInfo, error) {
}
dif := &diskFileInfo{
dir: f.dir,
root: f.root,
name: f.name,
mode: f.mode,
finfo: finfo,
}
@@ -112,12 +123,11 @@ func (f *diskFile) Read(p []byte) (int, error) {
// diskFilesystem implements the Filesystem interface
type diskFilesystem struct {
name string
dir string
metadata map[string]string
lock sync.RWMutex
// Max. size of the filesystem in bytes as
// given by the config
maxSize int64
root string
cwd string
// Current size of the filesystem in bytes
currentSize int64
@@ -127,67 +137,102 @@ type diskFilesystem struct {
logger log.Logger
}
// NewDiskFilesystem returns a new filesystem that is backed by a disk
// that implements the Filesystem interface
// NewDiskFilesystem returns a new filesystem that is backed by the disk filesystem.
// The root is / and the working directory is whatever is returned by os.Getwd(). The value
// of Root in the config will be ignored.
func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
fs := &diskFilesystem{
name: config.Name,
maxSize: config.Size,
logger: config.Logger,
metadata: make(map[string]string),
root: "/",
cwd: "/",
logger: config.Logger,
}
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
fs.cwd = cwd
if len(fs.cwd) == 0 {
fs.cwd = "/"
}
fs.cwd = filepath.Clean(fs.cwd)
if !filepath.IsAbs(fs.cwd) {
return nil, fmt.Errorf("the current working directory must be an absolute path")
}
if fs.logger == nil {
fs.logger = log.New("")
}
fs.logger = fs.logger.WithFields(log.Fields{
"name": fs.name,
"type": "disk",
})
return fs, nil
}
if err := fs.Rebase(config.Dir); err != nil {
// NewRootedDiskFilesystem returns a filesystem that is backed by the disk filesystem. The
// root of the filesystem is defined by DiskConfig.Root. The working directory is "/". Root
// must be directory. If it doesn't exist, it will be created
func NewRootedDiskFilesystem(config RootedDiskConfig) (Filesystem, error) {
fs := &diskFilesystem{
metadata: make(map[string]string),
root: config.Root,
cwd: "/",
logger: config.Logger,
}
if len(fs.root) == 0 {
fs.root = "/"
}
if root, err := filepath.Abs(fs.root); err != nil {
return nil, err
} else {
fs.root = root
}
err := os.MkdirAll(fs.root, 0700)
if err != nil {
return nil, err
}
info, err := os.Stat(fs.root)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("root is not a directory")
}
if fs.logger == nil {
fs.logger = log.New("")
}
return fs, nil
}
func (fs *diskFilesystem) Name() string {
return fs.name
}
func (fs *diskFilesystem) Base() string {
return fs.dir
}
func (fs *diskFilesystem) Rebase(base string) error {
if len(base) == 0 {
return fmt.Errorf("invalid base path provided")
}
dir, err := filepath.Abs(base)
if err != nil {
return err
}
base = dir
finfo, err := os.Stat(base)
if err != nil {
return fmt.Errorf("the provided base path '%s' doesn't exist", fs.dir)
}
if !finfo.IsDir() {
return fmt.Errorf("the provided base path '%s' must be a directory", fs.dir)
}
fs.dir = base
return nil
return "disk"
}
func (fs *diskFilesystem) Type() string {
return "diskfs"
return "disk"
}
func (fs *diskFilesystem) Metadata(key string) string {
fs.lock.RLock()
defer fs.lock.RUnlock()
return fs.metadata[key]
}
func (fs *diskFilesystem) SetMetadata(key, data string) {
fs.lock.Lock()
defer fs.lock.Unlock()
fs.metadata[key] = data
}
func (fs *diskFilesystem) Size() (int64, int64) {
@@ -196,7 +241,11 @@ func (fs *diskFilesystem) Size() (int64, int64) {
if time.Since(fs.lastSizeCheck) >= 10*time.Second {
var size int64 = 0
fs.walk(func(path string, info os.FileInfo) {
fs.walk(fs.root, func(path string, info os.FileInfo) {
if info.IsDir() {
return
}
size += info.Size()
})
@@ -205,17 +254,21 @@ func (fs *diskFilesystem) Size() (int64, int64) {
fs.lastSizeCheck = time.Now()
}
return fs.currentSize, fs.maxSize
return fs.currentSize, -1
}
func (fs *diskFilesystem) Resize(size int64) {
fs.maxSize = size
func (fs *diskFilesystem) Purge(size int64) int64 {
return 0
}
func (fs *diskFilesystem) Files() int64 {
var nfiles int64 = 0
fs.walk(func(path string, info os.FileInfo) {
fs.walk(fs.root, func(path string, info os.FileInfo) {
if info.IsDir() {
return
}
nfiles++
})
@@ -223,38 +276,58 @@ func (fs *diskFilesystem) Files() int64 {
}
func (fs *diskFilesystem) Symlink(oldname, newname string) error {
oldname = filepath.Join(fs.dir, filepath.Clean("/"+oldname))
oldname = fs.cleanPath(oldname)
newname = fs.cleanPath(newname)
if !filepath.IsAbs(newname) {
return nil
info, err := os.Lstat(oldname)
if err != nil {
return err
}
newname = filepath.Join(fs.dir, filepath.Clean("/"+newname))
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("%s can't link to another link (%s)", newname, oldname)
}
err := os.Symlink(oldname, newname)
if info.IsDir() {
return fmt.Errorf("can't symlink directories")
}
return err
return os.Symlink(oldname, newname)
}
func (fs *diskFilesystem) Open(path string) File {
path = filepath.Join(fs.dir, filepath.Clean("/"+path))
path = fs.cleanPath(path)
df := &diskFile{
root: fs.root,
name: strings.TrimPrefix(path, fs.root),
}
info, err := os.Lstat(path)
if err != nil {
return nil
}
df.mode = info.Mode()
f, err := os.Open(path)
if err != nil {
return nil
}
df := &diskFile{
dir: fs.dir,
name: path,
file: f,
}
df.file = f
return df
}
func (fs *diskFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
path = filepath.Join(fs.dir, filepath.Clean("/"+path))
func (fs *diskFilesystem) ReadFile(path string) ([]byte, error) {
path = fs.cleanPath(path)
return os.ReadFile(path)
}
func (fs *diskFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
path = fs.cleanPath(path)
replace := true
@@ -276,16 +349,155 @@ func (fs *diskFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
replace = false
}
defer f.Close()
size, err := f.ReadFrom(r)
if err != nil {
return -1, false, fmt.Errorf("reading data failed: %w", err)
}
fs.lastSizeCheck = time.Time{}
return size, !replace, nil
}
func (fs *diskFilesystem) Delete(path string) int64 {
path = filepath.Join(fs.dir, filepath.Clean("/"+path))
func (fs *diskFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
return fs.WriteFileReader(path, bytes.NewBuffer(data))
}
func (fs *diskFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
path = fs.cleanPath(path)
dir, filename := filepath.Split(path)
tmpfile, err := os.CreateTemp(dir, filename)
if err != nil {
return -1, false, err
}
defer os.Remove(tmpfile.Name())
size, err := tmpfile.Write(data)
if err != nil {
return -1, false, err
}
if err := tmpfile.Close(); err != nil {
return -1, false, err
}
replace := false
if _, err := fs.Stat(path); err == nil {
replace = true
}
if err := fs.rename(tmpfile.Name(), path); err != nil {
return -1, false, err
}
fs.lastSizeCheck = time.Time{}
return int64(size), !replace, nil
}
func (fs *diskFilesystem) Rename(src, dst string) error {
src = fs.cleanPath(src)
dst = fs.cleanPath(dst)
return fs.rename(src, dst)
}
func (fs *diskFilesystem) rename(src, dst string) error {
if src == dst {
return nil
}
// First try to rename the file
if err := os.Rename(src, dst); err == nil {
return nil
}
// If renaming the file fails, copy the data
if err := fs.copy(src, dst); err != nil {
os.Remove(dst)
return fmt.Errorf("failed to copy files: %w", err)
}
if err := os.Remove(src); err != nil {
os.Remove(dst)
return fmt.Errorf("failed to remove source file: %w", err)
}
return nil
}
func (fs *diskFilesystem) Copy(src, dst string) error {
src = fs.cleanPath(src)
dst = fs.cleanPath(dst)
return fs.copy(src, dst)
}
func (fs *diskFilesystem) copy(src, dst string) error {
source, err := os.Open(src)
if err != nil {
return fmt.Errorf("failed to open source file: %w", err)
}
destination, err := os.Create(dst)
if err != nil {
source.Close()
return fmt.Errorf("failed to create destination file: %w", err)
}
defer destination.Close()
if _, err := io.Copy(destination, source); err != nil {
source.Close()
os.Remove(dst)
return fmt.Errorf("failed to copy data from source to destination: %w", err)
}
source.Close()
fs.lastSizeCheck = time.Time{}
return nil
}
func (fs *diskFilesystem) MkdirAll(path string, perm os.FileMode) error {
path = fs.cleanPath(path)
return os.MkdirAll(path, perm)
}
func (fs *diskFilesystem) Stat(path string) (FileInfo, error) {
path = fs.cleanPath(path)
dif := &diskFileInfo{
root: fs.root,
name: strings.TrimPrefix(path, fs.root),
}
info, err := os.Lstat(path)
if err != nil {
return nil, err
}
dif.mode = info.Mode()
if info.Mode()&os.ModeSymlink != 0 {
info, err = os.Stat(path)
if err != nil {
return nil, err
}
}
dif.finfo = info
return dif, nil
}
func (fs *diskFilesystem) Remove(path string) int64 {
path = fs.cleanPath(path)
finfo, err := os.Stat(path)
if err != nil {
@@ -298,28 +510,31 @@ func (fs *diskFilesystem) Delete(path string) int64 {
return -1
}
fs.lastSizeCheck = time.Time{}
return size
}
func (fs *diskFilesystem) DeleteAll() int64 {
func (fs *diskFilesystem) RemoveAll() int64 {
return 0
}
func (fs *diskFilesystem) List(pattern string) []FileInfo {
func (fs *diskFilesystem) List(path, pattern string) []FileInfo {
path = fs.cleanPath(path)
files := []FileInfo{}
fs.walk(func(path string, info os.FileInfo) {
if path == fs.dir {
fs.walk(path, func(path string, info os.FileInfo) {
if path == fs.root {
return
}
name := strings.TrimPrefix(path, fs.dir)
name := strings.TrimPrefix(path, fs.root)
if name[0] != os.PathSeparator {
name = string(os.PathSeparator) + name
}
if info.IsDir() {
name += "/"
return
}
if len(pattern) != 0 {
@@ -329,7 +544,7 @@ func (fs *diskFilesystem) List(pattern string) []FileInfo {
}
files = append(files, &diskFileInfo{
dir: fs.dir,
root: fs.root,
name: name,
finfo: info,
})
@@ -338,8 +553,8 @@ func (fs *diskFilesystem) List(pattern string) []FileInfo {
return files
}
func (fs *diskFilesystem) walk(walkfn func(path string, info os.FileInfo)) {
filepath.Walk(fs.dir, func(path string, info os.FileInfo, err error) error {
func (fs *diskFilesystem) walk(path string, walkfn func(path string, info os.FileInfo)) {
filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
@@ -359,3 +574,11 @@ func (fs *diskFilesystem) walk(walkfn func(path string, info os.FileInfo)) {
return nil
})
}
func (fs *diskFilesystem) cleanPath(path string) string {
if !filepath.IsAbs(path) {
path = filepath.Join(fs.cwd, path)
}
return filepath.Join(fs.root, filepath.Clean(path))
}

48
io/fs/dummy.go
View File

@@ -1,48 +0,0 @@
package fs
import (
"io"
"time"
)
type dummyFileInfo struct{}
func (d *dummyFileInfo) Name() string { return "" }
func (d *dummyFileInfo) Size() int64 { return 0 }
func (d *dummyFileInfo) ModTime() time.Time { return time.Date(2000, 1, 1, 0, 0, 0, 0, nil) }
func (d *dummyFileInfo) IsLink() (string, bool) { return "", false }
func (d *dummyFileInfo) IsDir() bool { return false }
type dummyFile struct{}
func (d *dummyFile) Read(p []byte) (int, error) { return 0, io.EOF }
func (d *dummyFile) Close() error { return nil }
func (d *dummyFile) Name() string { return "" }
func (d *dummyFile) Stat() (FileInfo, error) { return &dummyFileInfo{}, nil }
type dummyFilesystem struct {
name string
typ string
}
func (d *dummyFilesystem) Name() string { return d.name }
func (d *dummyFilesystem) Base() string { return "/" }
func (d *dummyFilesystem) Rebase(string) error { return nil }
func (d *dummyFilesystem) Type() string { return d.typ }
func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 }
func (d *dummyFilesystem) Resize(int64) {}
func (d *dummyFilesystem) Files() int64 { return 0 }
func (d *dummyFilesystem) Symlink(string, string) error { return nil }
func (d *dummyFilesystem) Open(string) File { return &dummyFile{} }
func (d *dummyFilesystem) Store(string, io.Reader) (int64, bool, error) { return 0, true, nil }
func (d *dummyFilesystem) Delete(string) int64 { return 0 }
func (d *dummyFilesystem) DeleteAll() int64 { return 0 }
func (d *dummyFilesystem) List(string) []FileInfo { return []FileInfo{} }
// NewDummyFilesystem return a dummy filesystem
func NewDummyFilesystem(name, typ string) Filesystem {
return &dummyFilesystem{
name: name,
typ: typ,
}
}

120
io/fs/fs.go
View File

@@ -3,24 +3,29 @@ package fs
import (
"io"
"io/fs"
"os"
"time"
)
// FileInfo describes a file and is returned by Stat.
type FileInfo interface {
// Name returns the full name of the file
// Name returns the full name of the file.
Name() string
// Size reports the size of the file in bytes
// Size reports the size of the file in bytes.
Size() int64
// ModTime returns the time of last modification
// Mode returns the file mode.
Mode() fs.FileMode
// ModTime returns the time of last modification.
ModTime() time.Time
// IsLink returns the path this file is linking to and true. Otherwise an empty string and false.
IsLink() (string, bool)
// IsDir returns whether the file represents a directory
// IsDir returns whether the file represents a directory.
IsDir() bool
}
@@ -28,58 +33,95 @@ type FileInfo interface {
type File interface {
io.ReadCloser
// Name returns the Name of the file
// Name returns the Name of the file.
Name() string
// Stat returns the FileInfo to this file. In case of an error
// FileInfo is nil and the error is non-nil.
// Stat returns the FileInfo to this file. In case of an error FileInfo is nil
// and the error is non-nil. If the file is a symlink, the info reports the name and mode
// of the link itself, but the modification time and size of the linked file.
Stat() (FileInfo, error)
}
// Filesystem is an interface that provides access to a filesystem.
type Filesystem interface {
// Name returns the name of this filesystem
Name() string
// Base returns the base path of this filesystem
Base() string
// Rebase sets a new base path for this filesystem
Rebase(string) error
// Type returns the type of this filesystem
Type() string
type ReadFilesystem interface {
// Size returns the consumed size and capacity of the filesystem in bytes. The
// capacity is negative if the filesystem can consume as much space as it can.
// capacity is negative if the filesystem can consume as much space as it wants.
Size() (int64, int64)
// Resize resizes the filesystem to the new size. Files may need to be deleted.
Resize(size int64)
// Files returns the current number of files in the filesystem.
Files() int64
// Open returns the file stored at the given path. It returns nil if the
// file doesn't exist. If the file is a symlink, the name is the name of
// the link, but it will read the contents of the linked file.
Open(path string) File
// ReadFile reads the content of the file at the given path into the writer. Returns
// the number of bytes read or an error.
ReadFile(path string) ([]byte, error)
// Stat returns info about the file at path. If the file doesn't exist, an error
// will be returned. If the file is a symlink, the info reports the name and mode
// of the link itself, but the modification time and size are of the linked file.
Stat(path string) (FileInfo, error)
// List lists all files that are currently on the filesystem.
List(path, pattern string) []FileInfo
}
type WriteFilesystem interface {
// Symlink creates newname as a symbolic link to oldname.
Symlink(oldname, newname string) error
// Open returns the file stored at the given path. It returns nil if the
// file doesn't exist.
Open(path string) File
// Store adds a file to the filesystem. Returns the size of the data that has been
// WriteFileReader adds a file to the filesystem. Returns the size of the data that has been
// stored in bytes and whether the file is new. The size is negative if there was
// an error adding the file and error is not nil.
Store(path string, r io.Reader) (int64, bool, error)
WriteFileReader(path string, r io.Reader) (int64, bool, error)
// Delete removes a file at the given path from the filesystem. Returns the size of
// the removed file in bytes. The size is negative if the file doesn't exist.
Delete(path string) int64
// WriteFile adds a file to the filesystem. Returns the size of the data that has been
// stored in bytes and whether the file is new. The size is negative if there was
// an error adding the file and error is not nil.
WriteFile(path string, data []byte) (int64, bool, error)
// DeleteAll removes all files from the filesystem. Returns the size of the
// WriteFileSafe adds a file to the filesystem by first writing it to a tempfile and then
// renaming it to the actual path. Returns the size of the data that has been
// stored in bytes and whether the file is new. The size is negative if there was
// an error adding the file and error is not nil.
WriteFileSafe(path string, data []byte) (int64, bool, error)
// MkdirAll creates a directory named path, along with any necessary parents, and returns nil,
// or else returns an error. The permission bits perm (before umask) are used for all directories
// that MkdirAll creates. If path is already a directory, MkdirAll does nothing and returns nil.
MkdirAll(path string, perm os.FileMode) error
// Rename renames the file from src to dst. If src and dst can't be renamed
// regularly, the data is copied from src to dst. dst will be overwritten
// if it already exists. src will be removed after all data has been copied
// successfully. Both files exist during copying.
Rename(src, dst string) error
// Copy copies a file from src to dst.
Copy(src, dst string) error
// Remove removes a file at the given path from the filesystem. Returns the size of
// the remove file in bytes. The size is negative if the file doesn't exist.
Remove(path string) int64
// RemoveAll removes all files from the filesystem. Returns the size of the
// removed files in bytes.
DeleteAll() int64
// List lists all files that are currently on the filesystem.
List(pattern string) []FileInfo
RemoveAll() int64
}
// Filesystem is an interface that provides access to a filesystem.
type Filesystem interface {
ReadFilesystem
WriteFilesystem
// Name returns the name of the filesystem.
Name() string
// Type returns the type of the filesystem, e.g. disk, mem, s3
Type() string
Metadata(key string) string
SetMetadata(key string, data string)
}

742
io/fs/fs_test.go Normal file
View File

@@ -0,0 +1,742 @@
package fs
import (
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
var ErrNoMinio = errors.New("minio binary not found")
func startMinio(t *testing.T, path string) (*exec.Cmd, error) {
err := os.MkdirAll(path, 0700)
require.NoError(t, err)
minio, err := exec.LookPath("minio")
if err != nil {
return nil, ErrNoMinio
}
proc := exec.Command(minio, "server", path, "--address", "127.0.0.1:9000")
proc.Stderr = os.Stderr
proc.Stdout = os.Stdout
err = proc.Start()
require.NoError(t, err)
time.Sleep(5 * time.Second)
return proc, nil
}
func stopMinio(t *testing.T, proc *exec.Cmd) {
err := proc.Process.Signal(os.Interrupt)
require.NoError(t, err)
proc.Wait()
}
func TestFilesystem(t *testing.T) {
miniopath, err := filepath.Abs("./minio")
require.NoError(t, err)
err = os.RemoveAll(miniopath)
require.NoError(t, err)
minio, err := startMinio(t, miniopath)
if err != nil {
if err != ErrNoMinio {
require.NoError(t, err)
}
}
os.RemoveAll("./testing/")
filesystems := map[string]func(string) (Filesystem, error){
"memfs": func(name string) (Filesystem, error) {
return NewMemFilesystem(MemConfig{})
},
"diskfs": func(name string) (Filesystem, error) {
return NewRootedDiskFilesystem(RootedDiskConfig{
Root: "./testing/" + name,
})
},
"s3fs": func(name string) (Filesystem, error) {
return NewS3Filesystem(S3Config{
Name: name,
Endpoint: "127.0.0.1:9000",
AccessKeyID: "minioadmin",
SecretAccessKey: "minioadmin",
Region: "",
Bucket: strings.ToLower(name),
UseSSL: false,
Logger: nil,
})
},
}
tests := map[string]func(*testing.T, Filesystem){
"new": testNew,
"metadata": testMetadata,
"writeFile": testWriteFile,
"writeFileSafe": testWriteFileSafe,
"writeFileReader": testWriteFileReader,
"delete": testDelete,
"files": testFiles,
"replace": testReplace,
"list": testList,
"listGlob": testListGlob,
"deleteAll": testDeleteAll,
"data": testData,
"statDir": testStatDir,
"mkdirAll": testMkdirAll,
"rename": testRename,
"renameOverwrite": testRenameOverwrite,
"copy": testCopy,
"symlink": testSymlink,
"stat": testStat,
"copyOverwrite": testCopyOverwrite,
"symlinkErrors": testSymlinkErrors,
"symlinkOpenStat": testSymlinkOpenStat,
"open": testOpen,
}
for fsname, fs := range filesystems {
for name, test := range tests {
t.Run(fsname+"-"+name, func(t *testing.T) {
if fsname == "s3fs" && minio == nil {
t.Skip("minio server not available")
}
filesystem, err := fs(name)
require.NoError(t, err)
test(t, filesystem)
})
}
}
os.RemoveAll("./testing/")
if minio != nil {
stopMinio(t, minio)
}
os.RemoveAll(miniopath)
}
func testNew(t *testing.T, fs Filesystem) {
cur, max := fs.Size()
require.Equal(t, int64(0), cur, "current size")
require.Equal(t, int64(-1), max, "max size")
cur = fs.Files()
require.Equal(t, int64(0), cur, "number of files")
}
func testMetadata(t *testing.T, fs Filesystem) {
fs.SetMetadata("foo", "bar")
require.Equal(t, "bar", fs.Metadata("foo"))
}
func testWriteFile(t *testing.T, fs Filesystem) {
size, created, err := fs.WriteFile("/foobar", []byte("xxxxx"))
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testWriteFileSafe(t *testing.T, fs Filesystem) {
size, created, err := fs.WriteFileSafe("/foobar", []byte("xxxxx"))
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testWriteFileReader(t *testing.T, fs Filesystem) {
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testOpen(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("xxxxx"))
require.NoError(t, err)
file = fs.Open("/foobar")
require.NotNil(t, file)
require.Equal(t, "/foobar", file.Name())
stat, err := file.Stat()
require.NoError(t, err)
require.Equal(t, "/foobar", stat.Name())
require.Equal(t, int64(5), stat.Size())
require.Equal(t, false, stat.IsDir())
}
func testDelete(t *testing.T, fs Filesystem) {
size := fs.Remove("/foobar")
require.Equal(t, int64(-1), size)
data := strings.NewReader("xxxxx")
fs.WriteFileReader("/foobar", data)
size = fs.Remove("/foobar")
require.Equal(t, int64(5), size)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(0), cur)
}
func testFiles(t *testing.T, fs Filesystem) {
require.Equal(t, int64(0), fs.Files())
fs.WriteFileReader("/foobar.txt", strings.NewReader("bar"))
require.Equal(t, int64(1), fs.Files())
fs.MkdirAll("/path/to/foo", 0777)
require.Equal(t, int64(1), fs.Files())
fs.Remove("/foobar.txt")
require.Equal(t, int64(0), fs.Files())
}
func testReplace(t *testing.T, fs Filesystem) {
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(3), size)
require.Equal(t, false, created)
cur, max = fs.Size()
require.Equal(t, int64(3), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testList(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("a"))
fs.WriteFileReader("/foobar2", strings.NewReader("bb"))
fs.WriteFileReader("/foobar3", strings.NewReader("ccc"))
fs.WriteFileReader("/foobar4", strings.NewReader("dddd"))
fs.WriteFileReader("/path/foobar3", strings.NewReader("ccc"))
fs.WriteFileReader("/path/to/foobar4", strings.NewReader("dddd"))
cur, max := fs.Size()
require.Equal(t, int64(17), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(6), cur)
getNames := func(files []FileInfo) []string {
names := []string{}
for _, f := range files {
names = append(names, f.Name())
}
return names
}
files := fs.List("/", "")
require.Equal(t, 6, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar2", "/foobar3", "/foobar4", "/path/foobar3", "/path/to/foobar4"}, getNames(files))
files = fs.List("/path", "")
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar3", "/path/to/foobar4"}, getNames(files))
}
func testListGlob(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("a"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("a"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("a"))
fs.WriteFileReader("/foobar4", strings.NewReader("a"))
cur := fs.Files()
require.Equal(t, int64(4), cur)
getNames := func(files []FileInfo) []string {
names := []string{}
for _, f := range files {
names = append(names, f.Name())
}
return names
}
files := getNames(fs.List("/", "/foo*"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
files = getNames(fs.List("/", "/*bar?"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
files = getNames(fs.List("/", "/path/*"))
require.Equal(t, 1, len(files))
require.ElementsMatch(t, []string{"/path/foobar2"}, files)
files = getNames(fs.List("/", "/path/**"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
files = getNames(fs.List("/path", "/**"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
}
func testDeleteAll(t *testing.T, fs Filesystem) {
if _, ok := fs.(*diskFilesystem); ok {
return
}
fs.WriteFileReader("/foobar1", strings.NewReader("abc"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("abc"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc"))
fs.WriteFileReader("/foobar4", strings.NewReader("abc"))
cur := fs.Files()
require.Equal(t, int64(4), cur)
size := fs.RemoveAll()
require.Equal(t, int64(12), size)
cur = fs.Files()
require.Equal(t, int64(0), cur)
}
func testData(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)
_, err := fs.ReadFile("/foobar")
require.Error(t, err)
data := "gduwotoxqb"
data1 := strings.NewReader(data)
_, _, err = fs.WriteFileReader("/foobar", data1)
require.NoError(t, err)
file = fs.Open("/foobar")
require.NotNil(t, file)
data2 := make([]byte, len(data)+1)
n, err := file.Read(data2)
if err != nil {
if err != io.EOF {
require.NoError(t, err)
}
}
require.Equal(t, len(data), n)
require.Equal(t, []byte(data), data2[:n])
data3, err := fs.ReadFile("/foobar")
require.NoError(t, err)
require.Equal(t, []byte(data), data3)
}
func testStatDir(t *testing.T, fs Filesystem) {
info, err := fs.Stat("/")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, true, info.IsDir())
data := strings.NewReader("gduwotoxqb")
fs.WriteFileReader("/these/are/some/directories/foobar", data)
info, err = fs.Stat("/foobar")
require.Error(t, err)
require.Nil(t, info)
info, err = fs.Stat("/these/are/some/directories/foobar")
require.NoError(t, err)
require.Equal(t, "/these/are/some/directories/foobar", info.Name())
require.Equal(t, int64(10), info.Size())
require.Equal(t, false, info.IsDir())
info, err = fs.Stat("/these")
require.NoError(t, err)
require.Equal(t, "/these", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/")
require.NoError(t, err)
require.Equal(t, "/these/are", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/some")
require.NoError(t, err)
require.Equal(t, "/these/are/some", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/some/directories")
require.NoError(t, err)
require.Equal(t, "/these/are/some/directories", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
}
func testMkdirAll(t *testing.T, fs Filesystem) {
info, err := fs.Stat("/foo/bar/dir")
require.Error(t, err)
require.Nil(t, info)
err = fs.MkdirAll("/foo/bar/dir", 0755)
require.NoError(t, err)
err = fs.MkdirAll("/foo/bar", 0755)
require.NoError(t, err)
info, err = fs.Stat("/foo/bar/dir")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/foo")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/foo/bar")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
err = fs.MkdirAll("/foobar", 0755)
require.Error(t, err)
}
func testRename(t *testing.T, fs Filesystem) {
err := fs.Rename("/foobar", "/foobaz")
require.Error(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
err = fs.Rename("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
}
func testRenameOverwrite(t *testing.T, fs Filesystem) {
_, err := fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
err = fs.Rename("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
data, err := fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlink(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
err := fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.NoError(t, err)
file := fs.Open("/foobaz")
require.NotNil(t, file)
require.Equal(t, "/foobaz", file.Name())
data := make([]byte, 10)
n, err := file.Read(data)
if err != nil {
if err != io.EOF {
require.NoError(t, err)
}
}
require.NoError(t, err)
require.Equal(t, 6, n)
require.Equal(t, "foobar", string(data[:n]))
stat, err := fs.Stat("/foobaz")
require.NoError(t, err)
require.Equal(t, "/foobaz", stat.Name())
require.Equal(t, int64(6), stat.Size())
require.NotEqual(t, 0, int(stat.Mode()&os.ModeSymlink))
link, ok := stat.IsLink()
require.Equal(t, "/foobar", link)
require.Equal(t, true, ok)
data, err = fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlinkOpenStat(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.NoError(t, err)
file := fs.Open("/foobaz")
require.NotNil(t, file)
require.Equal(t, "/foobaz", file.Name())
fstat, err := file.Stat()
require.NoError(t, err)
stat, err := fs.Stat("/foobaz")
require.NoError(t, err)
require.Equal(t, "/foobaz", fstat.Name())
require.Equal(t, fstat.Name(), stat.Name())
require.Equal(t, int64(6), fstat.Size())
require.Equal(t, fstat.Size(), stat.Size())
require.NotEqual(t, 0, int(fstat.Mode()&os.ModeSymlink))
require.Equal(t, fstat.Mode(), stat.Mode())
}
func testStat(t *testing.T, fs Filesystem) {
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
file := fs.Open("/foobar")
require.NotNil(t, file)
stat1, err := fs.Stat("/foobar")
require.NoError(t, err)
stat2, err := file.Stat()
require.NoError(t, err)
require.Equal(t, stat1, stat2)
}
func testCopy(t *testing.T, fs Filesystem) {
err := fs.Rename("/foobar", "/foobaz")
require.Error(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
err = fs.Copy("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
}
func testCopyOverwrite(t *testing.T, fs Filesystem) {
_, err := fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
err = fs.Copy("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
data, err := fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlinkErrors(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
err := fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
err = fs.Symlink("/foobar", "/bazfoo")
require.NoError(t, err)
err = fs.Symlink("/bazfoo", "/barfoo")
require.Error(t, err)
}

556
io/fs/mem.go
View File

@@ -4,7 +4,11 @@ import (
"bytes"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
@@ -15,28 +19,15 @@ import (
// MemConfig is the config that is required for creating
// a new memory filesystem.
type MemConfig struct {
// Namee is the name of the filesystem
Name string
// Base is the base path to be reported for this filesystem
Base string
// Size is the capacity of the filesystem in bytes
Size int64
// Set true to automatically delete the oldest files until there's
// enough space to store a new file
Purge bool
// For logging, optional
Logger log.Logger
Logger log.Logger // For logging, optional
}
type memFileInfo struct {
name string
size int64
lastMod time.Time
linkTo string
name string // Full name of the file (including path)
size int64 // The size of the file in bytes
dir bool // Whether this file represents a directory
lastMod time.Time // The time of the last modification of the file
linkTo string // Where the file links to, empty if it's not a link
}
func (f *memFileInfo) Name() string {
@@ -47,6 +38,20 @@ func (f *memFileInfo) Size() int64 {
return f.size
}
func (f *memFileInfo) Mode() fs.FileMode {
mode := fs.FileMode(fs.ModePerm)
if f.dir {
mode |= fs.ModeDir
}
if len(f.linkTo) != 0 {
mode |= fs.ModeSymlink
}
return mode
}
func (f *memFileInfo) ModTime() time.Time {
return f.lastMod
}
@@ -56,24 +61,12 @@ func (f *memFileInfo) IsLink() (string, bool) {
}
func (f *memFileInfo) IsDir() bool {
return false
return f.dir
}
type memFile struct {
// Name of the file
name string
// Size of the file in bytes
size int64
// Last modification of the file as a UNIX timestamp
lastMod time.Time
// Contents of the file
data *bytes.Buffer
// Link to another file
linkTo string
memFileInfo
data *bytes.Buffer // Contents of the file
}
func (f *memFile) Name() string {
@@ -84,6 +77,7 @@ func (f *memFile) Stat() (FileInfo, error) {
info := &memFileInfo{
name: f.name,
size: f.size,
dir: f.dir,
lastMod: f.lastMod,
linkTo: f.linkTo,
}
@@ -110,8 +104,8 @@ func (f *memFile) Close() error {
}
type memFilesystem struct {
name string
base string
metadata map[string]string
metaLock sync.RWMutex
// Mapping of path to file
files map[string]*memFile
@@ -122,29 +116,19 @@ type memFilesystem struct {
// Pool for the storage of the contents of files
dataPool sync.Pool
// Max. size of the filesystem in bytes as
// given by the config
maxSize int64
// Current size of the filesystem in bytes
currentSize int64
// Purge setting from the config
purge bool
// Logger from the config
logger log.Logger
}
// NewMemFilesystem creates a new filesystem in memory that implements
// the Filesystem interface.
func NewMemFilesystem(config MemConfig) Filesystem {
func NewMemFilesystem(config MemConfig) (Filesystem, error) {
fs := &memFilesystem{
name: config.Name,
base: config.Base,
maxSize: config.Size,
purge: config.Purge,
logger: config.Logger,
metadata: make(map[string]string),
logger: config.Logger,
}
if fs.logger == nil {
@@ -161,70 +145,105 @@ func NewMemFilesystem(config MemConfig) Filesystem {
},
}
fs.logger.WithFields(log.Fields{
"name": fs.name,
"size_bytes": fs.maxSize,
"purge": fs.purge,
}).Debug().Log("Created")
fs.logger.Debug().Log("Created")
return fs
return fs, nil
}
func NewMemFilesystemFromDir(dir string, config MemConfig) (Filesystem, error) {
mem, err := NewMemFilesystem(config)
if err != nil {
return nil, err
}
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if info.IsDir() {
return nil
}
mode := info.Mode()
if !mode.IsRegular() {
return nil
}
if mode&os.ModeSymlink != 0 {
return nil
}
file, err := os.Open(path)
if err != nil {
return nil
}
defer file.Close()
_, _, err = mem.WriteFileReader(path, file)
if err != nil {
return fmt.Errorf("can't copy %s", path)
}
return nil
})
if err != nil {
return nil, err
}
return mem, nil
}
func (fs *memFilesystem) Name() string {
return fs.name
}
func (fs *memFilesystem) Base() string {
return fs.base
}
func (fs *memFilesystem) Rebase(base string) error {
fs.base = base
return nil
return "mem"
}
func (fs *memFilesystem) Type() string {
return "memfs"
return "mem"
}
func (fs *memFilesystem) Metadata(key string) string {
fs.metaLock.RLock()
defer fs.metaLock.RUnlock()
return fs.metadata[key]
}
func (fs *memFilesystem) SetMetadata(key, data string) {
fs.metaLock.Lock()
defer fs.metaLock.Unlock()
fs.metadata[key] = data
}
func (fs *memFilesystem) Size() (int64, int64) {
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()
return fs.currentSize, fs.maxSize
}
func (fs *memFilesystem) Resize(size int64) {
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
diffSize := fs.maxSize - size
if diffSize == 0 {
return
}
if diffSize > 0 {
fs.free(diffSize)
}
fs.logger.WithFields(log.Fields{
"from_bytes": fs.maxSize,
"to_bytes": size,
}).Debug().Log("Resizing")
fs.maxSize = size
return fs.currentSize, -1
}
func (fs *memFilesystem) Files() int64 {
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()
return int64(len(fs.files))
nfiles := int64(0)
for _, f := range fs.files {
if f.dir {
continue
}
nfiles++
}
return nfiles
}
func (fs *memFilesystem) Open(path string) File {
path = fs.cleanPath(path)
fs.filesLock.RLock()
file, ok := fs.files[path]
fs.filesLock.RUnlock()
@@ -234,29 +253,68 @@ func (fs *memFilesystem) Open(path string) File {
}
newFile := &memFile{
name: file.name,
size: file.size,
lastMod: file.lastMod,
linkTo: file.linkTo,
memFileInfo: memFileInfo{
name: file.name,
size: file.size,
lastMod: file.lastMod,
linkTo: file.linkTo,
},
}
if len(file.linkTo) != 0 {
file, ok = fs.files[file.linkTo]
if !ok {
return nil
}
}
if file.data != nil {
newFile.lastMod = file.lastMod
newFile.data = bytes.NewBuffer(file.data.Bytes())
newFile.size = int64(newFile.data.Len())
}
return newFile
}
func (fs *memFilesystem) ReadFile(path string) ([]byte, error) {
path = fs.cleanPath(path)
fs.filesLock.RLock()
file, ok := fs.files[path]
fs.filesLock.RUnlock()
if !ok {
return nil, os.ErrNotExist
}
if len(file.linkTo) != 0 {
file, ok = fs.files[file.linkTo]
if !ok {
return nil, os.ErrNotExist
}
}
if file.data != nil {
return file.data.Bytes(), nil
}
return nil, nil
}
func (fs *memFilesystem) Symlink(oldname, newname string) error {
oldname = fs.cleanPath(oldname)
newname = fs.cleanPath(newname)
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
if _, ok := fs.files[newname]; ok {
return fmt.Errorf("%s already exist", newname)
if _, ok := fs.files[oldname]; !ok {
return os.ErrNotExist
}
if oldname[0] != '/' {
oldname = "/" + oldname
if _, ok := fs.files[newname]; ok {
return os.ErrExist
}
if file, ok := fs.files[oldname]; ok {
@@ -266,11 +324,14 @@ func (fs *memFilesystem) Symlink(oldname, newname string) error {
}
newFile := &memFile{
name: newname,
size: 0,
lastMod: time.Now(),
data: nil,
linkTo: oldname,
memFileInfo: memFileInfo{
name: newname,
dir: false,
size: 0,
lastMod: time.Now(),
linkTo: oldname,
},
data: nil,
}
fs.files[newname] = newFile
@@ -278,18 +339,21 @@ func (fs *memFilesystem) Symlink(oldname, newname string) error {
return nil
}
func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
func (fs *memFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
path = fs.cleanPath(path)
newFile := &memFile{
name: path,
size: 0,
lastMod: time.Now(),
data: nil,
memFileInfo: memFileInfo{
name: path,
dir: false,
size: 0,
lastMod: time.Now(),
},
data: fs.dataPool.Get().(*bytes.Buffer),
}
data := fs.dataPool.Get().(*bytes.Buffer)
data.Reset()
size, err := data.ReadFrom(r)
newFile.data.Reset()
size, err := newFile.data.ReadFrom(r)
if err != nil {
fs.logger.WithFields(log.Fields{
"path": path,
@@ -297,55 +361,26 @@ func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
"error": err,
}).Warn().Log("Incomplete file")
}
newFile.size = size
newFile.data = data
// reject if the new file is larger than the available space
if fs.maxSize > 0 && newFile.size > fs.maxSize {
fs.dataPool.Put(data)
return -1, false, fmt.Errorf("File is too big")
}
newFile.size = size
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
// calculate the new size of the filesystem
newSize := fs.currentSize + newFile.size
file, replace := fs.files[path]
if replace {
newSize -= file.size
delete(fs.files, path)
fs.currentSize -= file.size
fs.dataPool.Put(file.data)
file.data = nil
}
if fs.maxSize > 0 {
if newSize > fs.maxSize {
if !fs.purge {
fs.dataPool.Put(data)
return -1, false, fmt.Errorf("not enough space on device")
}
if replace {
delete(fs.files, path)
fs.currentSize -= file.size
fs.dataPool.Put(file.data)
file.data = nil
}
newSize -= fs.free(fs.currentSize + newFile.size - fs.maxSize)
}
} else {
if replace {
delete(fs.files, path)
fs.dataPool.Put(file.data)
file.data = nil
}
}
fs.currentSize = newSize
fs.files[path] = newFile
fs.currentSize += newFile.size
logger := fs.logger.WithFields(log.Fields{
"path": newFile.name,
"filesize_bytes": newFile.size,
@@ -361,7 +396,18 @@ func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
return newFile.size, !replace, nil
}
func (fs *memFilesystem) free(size int64) int64 {
func (fs *memFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
return fs.WriteFileReader(path, bytes.NewBuffer(data))
}
func (fs *memFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
return fs.WriteFileReader(path, bytes.NewBuffer(data))
}
func (fs *memFilesystem) Purge(size int64) int64 {
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
files := []*memFile{}
for _, f := range fs.files {
@@ -399,7 +445,190 @@ func (fs *memFilesystem) free(size int64) int64 {
return freed
}
func (fs *memFilesystem) Delete(path string) int64 {
func (fs *memFilesystem) MkdirAll(path string, perm os.FileMode) error {
path = fs.cleanPath(path)
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
info, err := fs.stat(path)
if err == nil {
if info.IsDir() {
return nil
}
return os.ErrExist
}
f := &memFile{
memFileInfo: memFileInfo{
name: path,
size: 0,
dir: true,
lastMod: time.Now(),
},
data: nil,
}
fs.files[path] = f
return nil
}
func (fs *memFilesystem) Rename(src, dst string) error {
src = filepath.Join("/", filepath.Clean(src))
dst = filepath.Join("/", filepath.Clean(dst))
if src == dst {
return nil
}
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
srcFile, ok := fs.files[src]
if !ok {
return os.ErrNotExist
}
dstFile, ok := fs.files[dst]
if ok {
fs.currentSize -= dstFile.size
fs.dataPool.Put(dstFile.data)
dstFile.data = nil
}
fs.files[dst] = srcFile
delete(fs.files, src)
return nil
}
func (fs *memFilesystem) Copy(src, dst string) error {
src = filepath.Join("/", filepath.Clean(src))
dst = filepath.Join("/", filepath.Clean(dst))
if src == dst {
return nil
}
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
srcFile, ok := fs.files[src]
if !ok {
return os.ErrNotExist
}
if srcFile.dir {
return os.ErrNotExist
}
if fs.isDir(dst) {
return os.ErrInvalid
}
dstFile, ok := fs.files[dst]
if ok {
fs.currentSize -= dstFile.size
} else {
dstFile = &memFile{
memFileInfo: memFileInfo{
name: dst,
dir: false,
size: srcFile.size,
lastMod: time.Now(),
},
data: fs.dataPool.Get().(*bytes.Buffer),
}
}
dstFile.data.Reset()
dstFile.data.Write(srcFile.data.Bytes())
fs.currentSize += dstFile.size
fs.files[dst] = dstFile
return nil
}
func (fs *memFilesystem) Stat(path string) (FileInfo, error) {
path = fs.cleanPath(path)
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()
return fs.stat(path)
}
func (fs *memFilesystem) stat(path string) (FileInfo, error) {
file, ok := fs.files[path]
if ok {
f := &memFileInfo{
name: file.name,
size: file.size,
dir: file.dir,
lastMod: file.lastMod,
linkTo: file.linkTo,
}
if len(f.linkTo) != 0 {
file, ok := fs.files[f.linkTo]
if !ok {
return nil, os.ErrNotExist
}
f.lastMod = file.lastMod
f.size = file.size
}
return f, nil
}
// Check for directories
if !fs.isDir(path) {
return nil, os.ErrNotExist
}
f := &memFileInfo{
name: path,
size: 0,
dir: true,
lastMod: time.Now(),
linkTo: "",
}
return f, nil
}
func (fs *memFilesystem) isDir(path string) bool {
file, ok := fs.files[path]
if ok {
return file.dir
}
if !strings.HasSuffix(path, "/") {
path = path + "/"
}
if path == "/" {
return true
}
for k := range fs.files {
if strings.HasPrefix(k, path) {
return true
}
}
return false
}
func (fs *memFilesystem) Remove(path string) int64 {
path = fs.cleanPath(path)
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
@@ -423,7 +652,7 @@ func (fs *memFilesystem) Delete(path string) int64 {
return file.size
}
func (fs *memFilesystem) DeleteAll() int64 {
func (fs *memFilesystem) RemoveAll() int64 {
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
@@ -435,19 +664,28 @@ func (fs *memFilesystem) DeleteAll() int64 {
return size
}
func (fs *memFilesystem) List(pattern string) []FileInfo {
func (fs *memFilesystem) List(path, pattern string) []FileInfo {
path = fs.cleanPath(path)
files := []FileInfo{}
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()
for _, file := range fs.files {
if !strings.HasPrefix(file.name, path) {
continue
}
if len(pattern) != 0 {
if ok, _ := glob.Match(pattern, file.name, '/'); !ok {
continue
}
}
if file.dir {
continue
}
files = append(files, &memFileInfo{
name: file.name,
size: file.size,
@@ -458,3 +696,11 @@ func (fs *memFilesystem) List(pattern string) []FileInfo {
return files
}
func (fs *memFilesystem) cleanPath(path string) string {
if !filepath.IsAbs(path) {
path = filepath.Join("/", path)
}
return filepath.Join("/", filepath.Clean(path))
}

416
io/fs/mem_test.go
View File

@@ -1,406 +1,30 @@
package fs
import (
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNew(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
func TestMemFromDir(t *testing.T) {
mem, err := NewMemFilesystemFromDir(".", MemConfig{})
require.NoError(t, err)
cur, max := mem.Size()
names := []string{}
for _, f := range mem.List("/", "/*.go") {
names = append(names, f.Name())
}
assert.Equal(t, int64(0), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(0), cur)
}
func TestSimplePutNoPurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data := strings.NewReader("xxxxx")
size, created, err := mem.Store("/foobar", data)
assert.Nil(t, err)
assert.Equal(t, int64(5), size)
assert.Equal(t, true, created)
cur, max := mem.Size()
assert.Equal(t, int64(5), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(1), cur)
}
func TestSimpleDelete(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
size := mem.Delete("/foobar")
assert.Equal(t, int64(-1), size)
data := strings.NewReader("xxxxx")
mem.Store("/foobar", data)
size = mem.Delete("/foobar")
assert.Equal(t, int64(5), size)
cur, max := mem.Size()
assert.Equal(t, int64(0), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(0), cur)
}
func TestReplaceNoPurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data := strings.NewReader("xxxxx")
size, created, err := mem.Store("/foobar", data)
assert.Nil(t, err)
assert.Equal(t, int64(5), size)
assert.Equal(t, true, created)
cur, max := mem.Size()
assert.Equal(t, int64(5), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = mem.Store("/foobar", data)
assert.Nil(t, err)
assert.Equal(t, int64(3), size)
assert.Equal(t, false, created)
cur, max = mem.Size()
assert.Equal(t, int64(3), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(1), cur)
}
func TestReplacePurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: true,
})
data1 := strings.NewReader("xxx")
data2 := strings.NewReader("yyy")
data3 := strings.NewReader("zzz")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
mem.Store("/foobar3", data3)
cur, max := mem.Size()
assert.Equal(t, int64(9), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(3), cur)
data4 := strings.NewReader("zzzzz")
size, _, _ := mem.Store("/foobar1", data4)
assert.Equal(t, int64(5), size)
cur, max = mem.Size()
assert.Equal(t, int64(8), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(2), cur)
}
func TestReplaceUnlimited(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 0,
Purge: false,
})
data := strings.NewReader("xxxxx")
size, created, err := mem.Store("/foobar", data)
assert.Nil(t, err)
assert.Equal(t, int64(5), size)
assert.Equal(t, true, created)
cur, max := mem.Size()
assert.Equal(t, int64(5), cur)
assert.Equal(t, int64(0), max)
cur = mem.Files()
assert.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = mem.Store("/foobar", data)
assert.Nil(t, err)
assert.Equal(t, int64(3), size)
assert.Equal(t, false, created)
cur, max = mem.Size()
assert.Equal(t, int64(3), cur)
assert.Equal(t, int64(0), max)
cur = mem.Files()
assert.Equal(t, int64(1), cur)
}
func TestTooBigNoPurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data := strings.NewReader("xxxxxyyyyyz")
size, _, _ := mem.Store("/foobar", data)
assert.Equal(t, int64(-1), size)
}
func TestTooBigPurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: true,
})
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
data := strings.NewReader("xxxxxyyyyyz")
size, _, _ := mem.Store("/foobar", data)
assert.Equal(t, int64(-1), size)
}
func TestFullSpaceNoPurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
cur, max := mem.Size()
assert.Equal(t, int64(10), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(2), cur)
data3 := strings.NewReader("zzzzz")
size, _, _ := mem.Store("/foobar3", data3)
assert.Equal(t, int64(-1), size)
}
func TestFullSpacePurge(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: true,
})
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
cur, max := mem.Size()
assert.Equal(t, int64(10), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(2), cur)
data3 := strings.NewReader("zzzzz")
size, _, _ := mem.Store("/foobar3", data3)
assert.Equal(t, int64(5), size)
cur, max = mem.Size()
assert.Equal(t, int64(10), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(2), cur)
}
func TestFullSpacePurgeMulti(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: true,
})
data1 := strings.NewReader("xxx")
data2 := strings.NewReader("yyy")
data3 := strings.NewReader("zzz")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
mem.Store("/foobar3", data3)
cur, max := mem.Size()
assert.Equal(t, int64(9), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(3), cur)
data4 := strings.NewReader("zzzzz")
size, _, _ := mem.Store("/foobar4", data4)
assert.Equal(t, int64(5), size)
cur, max = mem.Size()
assert.Equal(t, int64(8), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(2), cur)
}
func TestPurgeOrder(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: true,
})
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
data3 := strings.NewReader("zzzzz")
mem.Store("/foobar1", data1)
time.Sleep(1 * time.Second)
mem.Store("/foobar2", data2)
time.Sleep(1 * time.Second)
mem.Store("/foobar3", data3)
file := mem.Open("/foobar1")
assert.Nil(t, file)
}
func TestList(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data1 := strings.NewReader("a")
data2 := strings.NewReader("bb")
data3 := strings.NewReader("ccc")
data4 := strings.NewReader("dddd")
mem.Store("/foobar1", data1)
mem.Store("/foobar2", data2)
mem.Store("/foobar3", data3)
mem.Store("/foobar4", data4)
cur, max := mem.Size()
assert.Equal(t, int64(10), cur)
assert.Equal(t, int64(10), max)
cur = mem.Files()
assert.Equal(t, int64(4), cur)
files := mem.List("")
assert.Equal(t, 4, len(files))
}
func TestData(t *testing.T) {
mem := NewMemFilesystem(MemConfig{
Size: 10,
Purge: false,
})
data := "gduwotoxqb"
data1 := strings.NewReader(data)
mem.Store("/foobar", data1)
file := mem.Open("/foobar")
data2 := make([]byte, len(data)+1)
n, _ := file.Read(data2)
assert.Equal(t, len(data), n)
assert.Equal(t, []byte(data), data2[:n])
require.ElementsMatch(t, []string{
"/disk.go",
"/fs_test.go",
"/fs.go",
"/mem_test.go",
"/mem.go",
"/readonly_test.go",
"/readonly.go",
"/s3.go",
"/sized_test.go",
"/sized.go",
}, names)
}

54
io/fs/readonly.go Normal file
View File

@@ -0,0 +1,54 @@
package fs
import (
"io"
"os"
)
type readOnlyFilesystem struct {
Filesystem
}
func NewReadOnlyFilesystem(fs Filesystem) (Filesystem, error) {
r := &readOnlyFilesystem{
Filesystem: fs,
}
return r, nil
}
func (r *readOnlyFilesystem) Symlink(oldname, newname string) error {
return os.ErrPermission
}
func (r *readOnlyFilesystem) WriteFileReader(path string, rd io.Reader) (int64, bool, error) {
return -1, false, os.ErrPermission
}
func (r *readOnlyFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
return -1, false, os.ErrPermission
}
func (r *readOnlyFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
return -1, false, os.ErrPermission
}
func (r *readOnlyFilesystem) MkdirAll(path string, perm os.FileMode) error {
return os.ErrPermission
}
func (r *readOnlyFilesystem) Remove(path string) int64 {
return -1
}
func (r *readOnlyFilesystem) RemoveAll() int64 {
return 0
}
func (r *readOnlyFilesystem) Purge(size int64) int64 {
return 0
}
func (r *readOnlyFilesystem) Resize(size int64) error {
return os.ErrPermission
}

50
io/fs/readonly_test.go Normal file
View File

@@ -0,0 +1,50 @@
package fs
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestReadOnly(t *testing.T) {
mem, err := NewMemFilesystemFromDir(".", MemConfig{})
require.NoError(t, err)
ro, err := NewReadOnlyFilesystem(mem)
require.NoError(t, err)
err = ro.Symlink("/readonly.go", "/foobar.go")
require.Error(t, err)
_, _, err = ro.WriteFile("/readonly.go", []byte("foobar"))
require.Error(t, err)
_, _, err = ro.WriteFileReader("/readonly.go", strings.NewReader("foobar"))
require.Error(t, err)
_, _, err = ro.WriteFileSafe("/readonly.go", []byte("foobar"))
require.Error(t, err)
err = ro.MkdirAll("/foobar/baz", 0700)
require.Error(t, err)
res := ro.Remove("/readonly.go")
require.Equal(t, int64(-1), res)
res = ro.RemoveAll()
require.Equal(t, int64(0), res)
rop, ok := ro.(PurgeFilesystem)
require.True(t, ok, "must implement PurgeFilesystem")
size, _ := ro.Size()
res = rop.Purge(size)
require.Equal(t, int64(0), res)
ros, ok := ro.(SizedFilesystem)
require.True(t, ok, "must implement SizedFilesystem")
err = ros.Resize(100)
require.Error(t, err)
}

293
io/fs/s3.go
View File

@@ -1,9 +1,15 @@
package fs
import (
"bytes"
"context"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/datarhei/core/v16/glob"
@@ -15,7 +21,6 @@ import (
type S3Config struct {
// Namee is the name of the filesystem
Name string
Base string
Endpoint string
AccessKeyID string
SecretAccessKey string
@@ -26,9 +31,11 @@ type S3Config struct {
Logger log.Logger
}
type s3fs struct {
type s3Filesystem struct {
metadata map[string]string
metaLock sync.RWMutex
name string
base string
endpoint string
accessKeyID string
@@ -42,10 +49,12 @@ type s3fs struct {
logger log.Logger
}
var fakeDirEntry = "..."
func NewS3Filesystem(config S3Config) (Filesystem, error) {
fs := &s3fs{
fs := &s3Filesystem{
metadata: make(map[string]string),
name: config.Name,
base: config.Base,
endpoint: config.Endpoint,
accessKeyID: config.AccessKeyID,
secretAccessKey: config.SecretAccessKey,
@@ -106,28 +115,32 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
return fs, nil
}
func (fs *s3fs) Name() string {
func (fs *s3Filesystem) Name() string {
return fs.name
}
func (fs *s3fs) Base() string {
return fs.base
func (fs *s3Filesystem) Type() string {
return "s3"
}
func (fs *s3fs) Rebase(base string) error {
fs.base = base
func (fs *s3Filesystem) Metadata(key string) string {
fs.metaLock.RLock()
defer fs.metaLock.RUnlock()
return nil
return fs.metadata[key]
}
func (fs *s3fs) Type() string {
return "s3fs"
func (fs *s3Filesystem) SetMetadata(key, data string) {
fs.metaLock.Lock()
defer fs.metaLock.Unlock()
fs.metadata[key] = data
}
func (fs *s3fs) Size() (int64, int64) {
func (fs *s3Filesystem) Size() (int64, int64) {
size := int64(0)
files := fs.List("")
files := fs.List("/", "")
for _, file := range files {
size += file.Size()
@@ -136,14 +149,18 @@ func (fs *s3fs) Size() (int64, int64) {
return size, -1
}
func (fs *s3fs) Resize(size int64) {}
func (fs *s3fs) Files() int64 {
func (fs *s3Filesystem) Files() int64 {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
Recursive: true,
WithVersions: false,
WithMetadata: false,
Prefix: "",
Recursive: true,
MaxKeys: 0,
StartAfter: "",
UseV1: false,
})
nfiles := int64(0)
@@ -152,19 +169,77 @@ func (fs *s3fs) Files() int64 {
if object.Err != nil {
fs.logger.WithError(object.Err).Log("Listing object failed")
}
if strings.HasSuffix("/"+object.Key, "/"+fakeDirEntry) {
// Skip fake entries (see MkdirAll)
continue
}
nfiles++
}
return nfiles
}
func (fs *s3fs) Symlink(oldname, newname string) error {
func (fs *s3Filesystem) Symlink(oldname, newname string) error {
return fmt.Errorf("not implemented")
}
func (fs *s3fs) Open(path string) File {
//ctx, cancel := context.WithCancel(context.Background())
//defer cancel()
func (fs *s3Filesystem) Stat(path string) (FileInfo, error) {
path = fs.cleanPath(path)
if len(path) == 0 {
return &s3FileInfo{
name: "/",
size: 0,
dir: true,
lastModified: time.Now(),
}, nil
}
ctx := context.Background()
object, err := fs.client.GetObject(ctx, fs.bucket, path, minio.GetObjectOptions{})
if err != nil {
if fs.isDir(path) {
return &s3FileInfo{
name: "/" + path,
size: 0,
dir: true,
lastModified: time.Now(),
}, nil
}
fs.logger.Debug().WithField("key", path).WithError(err).Log("Not found")
return nil, err
}
defer object.Close()
stat, err := object.Stat()
if err != nil {
if fs.isDir(path) {
return &s3FileInfo{
name: "/" + path,
size: 0,
dir: true,
lastModified: time.Now(),
}, nil
}
fs.logger.Debug().WithField("key", path).WithError(err).Log("Stat failed")
return nil, err
}
return &s3FileInfo{
name: "/" + stat.Key,
size: stat.Size,
lastModified: stat.LastModified,
}, nil
}
func (fs *s3Filesystem) Open(path string) File {
path = fs.cleanPath(path)
ctx := context.Background()
object, err := fs.client.GetObject(ctx, fs.bucket, path, minio.GetObjectOptions{})
@@ -181,7 +256,7 @@ func (fs *s3fs) Open(path string) File {
file := &s3File{
data: object,
name: stat.Key,
name: "/" + stat.Key,
size: stat.Size,
lastModified: stat.LastModified,
}
@@ -191,7 +266,26 @@ func (fs *s3fs) Open(path string) File {
return file
}
func (fs *s3fs) Store(path string, r io.Reader) (int64, bool, error) {
func (fs *s3Filesystem) ReadFile(path string) ([]byte, error) {
path = fs.cleanPath(path)
file := fs.Open(path)
if file == nil {
return nil, os.ErrNotExist
}
defer file.Close()
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(file)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (fs *s3Filesystem) write(path string, r io.Reader) (int64, bool, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -234,10 +328,84 @@ func (fs *s3fs) Store(path string, r io.Reader) (int64, bool, error) {
"overwrite": overwrite,
}).Log("Stored")
return info.Size, overwrite, nil
return info.Size, !overwrite, nil
}
func (fs *s3fs) Delete(path string) int64 {
func (fs *s3Filesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
path = fs.cleanPath(path)
return fs.write(path, r)
}
func (fs *s3Filesystem) WriteFile(path string, data []byte) (int64, bool, error) {
return fs.WriteFileReader(path, bytes.NewBuffer(data))
}
func (fs *s3Filesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
return fs.WriteFileReader(path, bytes.NewBuffer(data))
}
func (fs *s3Filesystem) Rename(src, dst string) error {
src = fs.cleanPath(src)
dst = fs.cleanPath(dst)
err := fs.Copy(src, dst)
if err != nil {
return err
}
res := fs.Remove(src)
if res == -1 {
return fmt.Errorf("failed to remove source file: %s", src)
}
return nil
}
func (fs *s3Filesystem) Copy(src, dst string) error {
src = fs.cleanPath(src)
dst = fs.cleanPath(dst)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_, err := fs.client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: fs.bucket,
Object: dst,
}, minio.CopySrcOptions{
Bucket: fs.bucket,
Object: src,
})
return err
}
func (fs *s3Filesystem) MkdirAll(path string, perm os.FileMode) error {
if path == "/" {
return nil
}
info, err := fs.Stat(path)
if err == nil {
if !info.IsDir() {
return os.ErrExist
}
return nil
}
path = filepath.Join(path, fakeDirEntry)
_, _, err = fs.write(path, strings.NewReader(""))
if err != nil {
return fmt.Errorf("can't create directory")
}
return nil
}
func (fs *s3Filesystem) Remove(path string) int64 {
path = fs.cleanPath(path)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -260,7 +428,7 @@ func (fs *s3fs) Delete(path string) int64 {
return stat.Size
}
func (fs *s3fs) DeleteAll() int64 {
func (fs *s3Filesystem) RemoveAll() int64 {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -295,14 +463,16 @@ func (fs *s3fs) DeleteAll() int64 {
return totalSize
}
func (fs *s3fs) List(pattern string) []FileInfo {
func (fs *s3Filesystem) List(path, pattern string) []FileInfo {
path = fs.cleanPath(path)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
WithVersions: false,
WithMetadata: false,
Prefix: "",
Prefix: path,
Recursive: true,
MaxKeys: 0,
StartAfter: "",
@@ -317,14 +487,20 @@ func (fs *s3fs) List(pattern string) []FileInfo {
continue
}
key := "/" + object.Key
if strings.HasSuffix(key, "/"+fakeDirEntry) {
// filter out fake directory entries (see MkdirAll)
continue
}
if len(pattern) != 0 {
if ok, _ := glob.Match(pattern, object.Key, '/'); !ok {
if ok, _ := glob.Match(pattern, key, '/'); !ok {
continue
}
}
f := &s3FileInfo{
name: object.Key,
name: key,
size: object.Size,
lastModified: object.LastModified,
}
@@ -335,9 +511,56 @@ func (fs *s3fs) List(pattern string) []FileInfo {
return files
}
func (fs *s3Filesystem) isDir(path string) bool {
if !strings.HasSuffix(path, "/") {
path = path + "/"
}
if path == "/" {
return true
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
WithVersions: false,
WithMetadata: false,
Prefix: path,
Recursive: true,
MaxKeys: 1,
StartAfter: "",
UseV1: false,
})
files := uint64(0)
for object := range ch {
if object.Err != nil {
fs.logger.WithError(object.Err).Log("Listing object failed")
continue
}
files++
}
return files > 0
}
func (fs *s3Filesystem) cleanPath(path string) string {
if !filepath.IsAbs(path) {
path = filepath.Join("/", path)
}
path = strings.TrimSuffix(path, "/"+fakeDirEntry)
return filepath.Join("/", filepath.Clean(path))[1:]
}
type s3FileInfo struct {
name string
size int64
dir bool
lastModified time.Time
}
@@ -349,6 +572,10 @@ func (f *s3FileInfo) Size() int64 {
return f.size
}
func (f *s3FileInfo) Mode() os.FileMode {
return fs.FileMode(fs.ModePerm)
}
func (f *s3FileInfo) ModTime() time.Time {
return f.lastModified
}
@@ -358,7 +585,7 @@ func (f *s3FileInfo) IsLink() (string, bool) {
}
func (f *s3FileInfo) IsDir() bool {
return false
return f.dir
}
type s3File struct {

168
io/fs/sized.go Normal file
View File

@@ -0,0 +1,168 @@
package fs
import (
"bytes"
"fmt"
"io"
)
type SizedFilesystem interface {
Filesystem
// Resize resizes the filesystem to the new size. Files may need to be deleted.
Resize(size int64) error
}
type PurgeFilesystem interface {
// Purge will free up at least size number of bytes and returns the actual
// freed space in bytes.
Purge(size int64) int64
}
type sizedFilesystem struct {
Filesystem
// Siez is the capacity of the filesystem in bytes
maxSize int64
// Set true to automatically delete the oldest files until there's
// enough space to store a new file
purge bool
}
var _ PurgeFilesystem = &sizedFilesystem{}
func NewSizedFilesystem(fs Filesystem, maxSize int64, purge bool) (SizedFilesystem, error) {
r := &sizedFilesystem{
Filesystem: fs,
maxSize: maxSize,
purge: purge,
}
return r, nil
}
func (r *sizedFilesystem) Size() (int64, int64) {
currentSize, _ := r.Filesystem.Size()
return currentSize, r.maxSize
}
func (r *sizedFilesystem) Resize(size int64) error {
currentSize, _ := r.Size()
if size >= currentSize {
// If the new size is the same or larger than the current size,
// nothing to do.
r.maxSize = size
return nil
}
// If the new size is less than the current size, purge some files.
r.Purge(currentSize - size)
r.maxSize = size
return nil
}
func (r *sizedFilesystem) WriteFileReader(path string, rd io.Reader) (int64, bool, error) {
currentSize, maxSize := r.Size()
if maxSize < 0 {
return r.Filesystem.WriteFileReader(path, rd)
}
data := bytes.Buffer{}
size, err := data.ReadFrom(rd)
if err != nil {
return -1, false, err
}
// reject if the new file is larger than the available space
if size > maxSize {
return -1, false, fmt.Errorf("File is too big")
}
// Calculate the new size of the filesystem
newSize := currentSize + size
// If the the new size is larger than the allowed size, we have to free
// some space.
if newSize > maxSize {
if !r.purge {
return -1, false, fmt.Errorf("not enough space on device")
}
if r.Purge(size) < size {
return -1, false, fmt.Errorf("not enough space on device")
}
}
return r.Filesystem.WriteFileReader(path, &data)
}
func (r *sizedFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
return r.WriteFileReader(path, bytes.NewBuffer(data))
}
func (r *sizedFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
currentSize, maxSize := r.Size()
if maxSize < 0 {
return r.Filesystem.WriteFile(path, data)
}
size := int64(len(data))
// reject if the new file is larger than the available space
if size > maxSize {
return -1, false, fmt.Errorf("File is too big")
}
// Calculate the new size of the filesystem
newSize := currentSize + size
// If the the new size is larger than the allowed size, we have to free
// some space.
if newSize > maxSize {
if !r.purge {
return -1, false, fmt.Errorf("not enough space on device")
}
if r.Purge(size) < size {
return -1, false, fmt.Errorf("not enough space on device")
}
}
return r.Filesystem.WriteFileSafe(path, data)
}
func (r *sizedFilesystem) Purge(size int64) int64 {
if purger, ok := r.Filesystem.(PurgeFilesystem); ok {
return purger.Purge(size)
}
return 0
/*
files := r.Filesystem.List("/", "")
sort.Slice(files, func(i, j int) bool {
return files[i].ModTime().Before(files[j].ModTime())
})
var freed int64 = 0
for _, f := range files {
r.Filesystem.Remove(f.Name())
size -= f.Size()
freed += f.Size()
r.currentSize -= f.Size()
if size <= 0 {
break
}
}
files = nil
return freed
*/
}

350
io/fs/sized_test.go Normal file
View File

@@ -0,0 +1,350 @@
package fs
import (
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func newMemFS() Filesystem {
mem, _ := NewMemFilesystem(MemConfig{})
return mem
}
func TestNewSized(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(0), cur)
}
func TestSizedResize(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(10), max)
err := fs.Resize(20)
require.NoError(t, err)
cur, max = fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(20), max)
}
func TestSizedResizePurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(10), max)
fs.WriteFileReader("/foobar", strings.NewReader("xxxxxxxxxx"))
cur, max = fs.Size()
require.Equal(t, int64(10), cur)
require.Equal(t, int64(10), max)
err := fs.Resize(5)
require.NoError(t, err)
cur, max = fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(5), max)
}
func TestSizedWrite(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(10), max)
size, created, err := fs.WriteFileReader("/foobar", strings.NewReader("xxxxx"))
require.NoError(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max = fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(10), max)
_, _, err = fs.WriteFile("/foobaz", []byte("xxxxxx"))
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("xxxxxx"))
require.Error(t, err)
_, _, err = fs.WriteFileSafe("/foobaz", []byte("xxxxxx"))
require.Error(t, err)
}
func TestSizedReplaceNoPurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(3), size)
require.Equal(t, false, created)
cur, max = fs.Size()
require.Equal(t, int64(3), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func TestSizedReplacePurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
data1 := strings.NewReader("xxx")
data2 := strings.NewReader("yyy")
data3 := strings.NewReader("zzz")
fs.WriteFileReader("/foobar1", data1)
fs.WriteFileReader("/foobar2", data2)
fs.WriteFileReader("/foobar3", data3)
cur, max := fs.Size()
require.Equal(t, int64(9), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(3), cur)
data4 := strings.NewReader("zzzzz")
size, _, _ := fs.WriteFileReader("/foobar1", data4)
require.Equal(t, int64(5), size)
cur, max = fs.Size()
require.Equal(t, int64(8), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(2), cur)
}
func TestSizedReplaceUnlimited(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), -1, false)
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(3), size)
require.Equal(t, false, created)
cur, max = fs.Size()
require.Equal(t, int64(3), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func TestSizedTooBigNoPurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
data := strings.NewReader("xxxxxyyyyyz")
size, _, err := fs.WriteFileReader("/foobar", data)
require.Error(t, err)
require.Equal(t, int64(-1), size)
}
func TestSizedTooBigPurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
fs.WriteFileReader("/foobar1", data1)
fs.WriteFileReader("/foobar2", data2)
data := strings.NewReader("xxxxxyyyyyz")
size, _, err := fs.WriteFileReader("/foobar", data)
require.Error(t, err)
require.Equal(t, int64(-1), size)
require.Equal(t, int64(2), fs.Files())
}
func TestSizedFullSpaceNoPurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
fs.WriteFileReader("/foobar1", data1)
fs.WriteFileReader("/foobar2", data2)
cur, max := fs.Size()
require.Equal(t, int64(10), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(2), cur)
data3 := strings.NewReader("zzzzz")
size, _, err := fs.WriteFileReader("/foobar3", data3)
require.Error(t, err)
require.Equal(t, int64(-1), size)
}
func TestSizedFullSpacePurge(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
fs.WriteFileReader("/foobar1", data1)
fs.WriteFileReader("/foobar2", data2)
cur, max := fs.Size()
require.Equal(t, int64(10), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(2), cur)
data3 := strings.NewReader("zzzzz")
size, _, _ := fs.WriteFileReader("/foobar3", data3)
require.Equal(t, int64(5), size)
cur, max = fs.Size()
require.Equal(t, int64(10), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(2), cur)
}
func TestSizedFullSpacePurgeMulti(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
data1 := strings.NewReader("xxx")
data2 := strings.NewReader("yyy")
data3 := strings.NewReader("zzz")
fs.WriteFileReader("/foobar1", data1)
fs.WriteFileReader("/foobar2", data2)
fs.WriteFileReader("/foobar3", data3)
cur, max := fs.Size()
require.Equal(t, int64(9), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(3), cur)
data4 := strings.NewReader("zzzzz")
size, _, _ := fs.WriteFileReader("/foobar4", data4)
require.Equal(t, int64(5), size)
cur, max = fs.Size()
require.Equal(t, int64(8), cur)
require.Equal(t, int64(10), max)
cur = fs.Files()
require.Equal(t, int64(2), cur)
}
func TestSizedPurgeOrder(t *testing.T) {
fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
data1 := strings.NewReader("xxxxx")
data2 := strings.NewReader("yyyyy")
data3 := strings.NewReader("zzzzz")
fs.WriteFileReader("/foobar1", data1)
time.Sleep(1 * time.Second)
fs.WriteFileReader("/foobar2", data2)
time.Sleep(1 * time.Second)
fs.WriteFileReader("/foobar3", data3)
file := fs.Open("/foobar1")
require.Nil(t, file)
}