Files
core/io/fs/fs_test.go
2023-02-01 16:09:20 +01:00

743 lines
17 KiB
Go

package fs
import (
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
var ErrNoMinio = errors.New("minio binary not found")
func startMinio(t *testing.T, path string) (*exec.Cmd, error) {
err := os.MkdirAll(path, 0700)
require.NoError(t, err)
minio, err := exec.LookPath("minio")
if err != nil {
return nil, ErrNoMinio
}
proc := exec.Command(minio, "server", path, "--address", "127.0.0.1:9000")
proc.Stderr = os.Stderr
proc.Stdout = os.Stdout
err = proc.Start()
require.NoError(t, err)
time.Sleep(5 * time.Second)
return proc, nil
}
func stopMinio(t *testing.T, proc *exec.Cmd) {
err := proc.Process.Signal(os.Interrupt)
require.NoError(t, err)
proc.Wait()
}
func TestFilesystem(t *testing.T) {
miniopath, err := filepath.Abs("./minio")
require.NoError(t, err)
err = os.RemoveAll(miniopath)
require.NoError(t, err)
minio, err := startMinio(t, miniopath)
if err != nil {
if err != ErrNoMinio {
require.NoError(t, err)
}
}
os.RemoveAll("./testing/")
filesystems := map[string]func(string) (Filesystem, error){
"memfs": func(name string) (Filesystem, error) {
return NewMemFilesystem(MemConfig{})
},
"diskfs": func(name string) (Filesystem, error) {
return NewRootedDiskFilesystem(RootedDiskConfig{
Root: "./testing/" + name,
})
},
"s3fs": func(name string) (Filesystem, error) {
return NewS3Filesystem(S3Config{
Name: name,
Endpoint: "127.0.0.1:9000",
AccessKeyID: "minioadmin",
SecretAccessKey: "minioadmin",
Region: "",
Bucket: strings.ToLower(name),
UseSSL: false,
Logger: nil,
})
},
}
tests := map[string]func(*testing.T, Filesystem){
"new": testNew,
"metadata": testMetadata,
"writeFile": testWriteFile,
"writeFileSafe": testWriteFileSafe,
"writeFileReader": testWriteFileReader,
"delete": testDelete,
"files": testFiles,
"replace": testReplace,
"list": testList,
"listGlob": testListGlob,
"deleteAll": testDeleteAll,
"data": testData,
"statDir": testStatDir,
"mkdirAll": testMkdirAll,
"rename": testRename,
"renameOverwrite": testRenameOverwrite,
"copy": testCopy,
"symlink": testSymlink,
"stat": testStat,
"copyOverwrite": testCopyOverwrite,
"symlinkErrors": testSymlinkErrors,
"symlinkOpenStat": testSymlinkOpenStat,
"open": testOpen,
}
for fsname, fs := range filesystems {
for name, test := range tests {
t.Run(fsname+"-"+name, func(t *testing.T) {
if fsname == "s3fs" && minio == nil {
t.Skip("minio server not available")
}
filesystem, err := fs(name)
require.NoError(t, err)
test(t, filesystem)
})
}
}
os.RemoveAll("./testing/")
if minio != nil {
stopMinio(t, minio)
}
os.RemoveAll(miniopath)
}
func testNew(t *testing.T, fs Filesystem) {
cur, max := fs.Size()
require.Equal(t, int64(0), cur, "current size")
require.Equal(t, int64(-1), max, "max size")
cur = fs.Files()
require.Equal(t, int64(0), cur, "number of files")
}
func testMetadata(t *testing.T, fs Filesystem) {
fs.SetMetadata("foo", "bar")
require.Equal(t, "bar", fs.Metadata("foo"))
}
func testWriteFile(t *testing.T, fs Filesystem) {
size, created, err := fs.WriteFile("/foobar", []byte("xxxxx"))
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testWriteFileSafe(t *testing.T, fs Filesystem) {
size, created, err := fs.WriteFileSafe("/foobar", []byte("xxxxx"))
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testWriteFileReader(t *testing.T, fs Filesystem) {
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testOpen(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("xxxxx"))
require.NoError(t, err)
file = fs.Open("/foobar")
require.NotNil(t, file)
require.Equal(t, "/foobar", file.Name())
stat, err := file.Stat()
require.NoError(t, err)
require.Equal(t, "/foobar", stat.Name())
require.Equal(t, int64(5), stat.Size())
require.Equal(t, false, stat.IsDir())
}
func testDelete(t *testing.T, fs Filesystem) {
size := fs.Remove("/foobar")
require.Equal(t, int64(-1), size)
data := strings.NewReader("xxxxx")
fs.WriteFileReader("/foobar", data)
size = fs.Remove("/foobar")
require.Equal(t, int64(5), size)
cur, max := fs.Size()
require.Equal(t, int64(0), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(0), cur)
}
func testFiles(t *testing.T, fs Filesystem) {
require.Equal(t, int64(0), fs.Files())
fs.WriteFileReader("/foobar.txt", strings.NewReader("bar"))
require.Equal(t, int64(1), fs.Files())
fs.MkdirAll("/path/to/foo", 0777)
require.Equal(t, int64(1), fs.Files())
fs.Remove("/foobar.txt")
require.Equal(t, int64(0), fs.Files())
}
func testReplace(t *testing.T, fs Filesystem) {
data := strings.NewReader("xxxxx")
size, created, err := fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(5), size)
require.Equal(t, true, created)
cur, max := fs.Size()
require.Equal(t, int64(5), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
data = strings.NewReader("yyy")
size, created, err = fs.WriteFileReader("/foobar", data)
require.Nil(t, err)
require.Equal(t, int64(3), size)
require.Equal(t, false, created)
cur, max = fs.Size()
require.Equal(t, int64(3), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(1), cur)
}
func testList(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("a"))
fs.WriteFileReader("/foobar2", strings.NewReader("bb"))
fs.WriteFileReader("/foobar3", strings.NewReader("ccc"))
fs.WriteFileReader("/foobar4", strings.NewReader("dddd"))
fs.WriteFileReader("/path/foobar3", strings.NewReader("ccc"))
fs.WriteFileReader("/path/to/foobar4", strings.NewReader("dddd"))
cur, max := fs.Size()
require.Equal(t, int64(17), cur)
require.Equal(t, int64(-1), max)
cur = fs.Files()
require.Equal(t, int64(6), cur)
getNames := func(files []FileInfo) []string {
names := []string{}
for _, f := range files {
names = append(names, f.Name())
}
return names
}
files := fs.List("/", "")
require.Equal(t, 6, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar2", "/foobar3", "/foobar4", "/path/foobar3", "/path/to/foobar4"}, getNames(files))
files = fs.List("/path", "")
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar3", "/path/to/foobar4"}, getNames(files))
}
func testListGlob(t *testing.T, fs Filesystem) {
fs.WriteFileReader("/foobar1", strings.NewReader("a"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("a"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("a"))
fs.WriteFileReader("/foobar4", strings.NewReader("a"))
cur := fs.Files()
require.Equal(t, int64(4), cur)
getNames := func(files []FileInfo) []string {
names := []string{}
for _, f := range files {
names = append(names, f.Name())
}
return names
}
files := getNames(fs.List("/", "/foo*"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
files = getNames(fs.List("/", "/*bar?"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
files = getNames(fs.List("/", "/path/*"))
require.Equal(t, 1, len(files))
require.ElementsMatch(t, []string{"/path/foobar2"}, files)
files = getNames(fs.List("/", "/path/**"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
files = getNames(fs.List("/path", "/**"))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
}
func testDeleteAll(t *testing.T, fs Filesystem) {
if _, ok := fs.(*diskFilesystem); ok {
return
}
fs.WriteFileReader("/foobar1", strings.NewReader("abc"))
fs.WriteFileReader("/path/foobar2", strings.NewReader("abc"))
fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc"))
fs.WriteFileReader("/foobar4", strings.NewReader("abc"))
cur := fs.Files()
require.Equal(t, int64(4), cur)
size := fs.RemoveAll()
require.Equal(t, int64(12), size)
cur = fs.Files()
require.Equal(t, int64(0), cur)
}
func testData(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)
_, err := fs.ReadFile("/foobar")
require.Error(t, err)
data := "gduwotoxqb"
data1 := strings.NewReader(data)
_, _, err = fs.WriteFileReader("/foobar", data1)
require.NoError(t, err)
file = fs.Open("/foobar")
require.NotNil(t, file)
data2 := make([]byte, len(data)+1)
n, err := file.Read(data2)
if err != nil {
if err != io.EOF {
require.NoError(t, err)
}
}
require.Equal(t, len(data), n)
require.Equal(t, []byte(data), data2[:n])
data3, err := fs.ReadFile("/foobar")
require.NoError(t, err)
require.Equal(t, []byte(data), data3)
}
func testStatDir(t *testing.T, fs Filesystem) {
info, err := fs.Stat("/")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, true, info.IsDir())
data := strings.NewReader("gduwotoxqb")
fs.WriteFileReader("/these/are/some/directories/foobar", data)
info, err = fs.Stat("/foobar")
require.Error(t, err)
require.Nil(t, info)
info, err = fs.Stat("/these/are/some/directories/foobar")
require.NoError(t, err)
require.Equal(t, "/these/are/some/directories/foobar", info.Name())
require.Equal(t, int64(10), info.Size())
require.Equal(t, false, info.IsDir())
info, err = fs.Stat("/these")
require.NoError(t, err)
require.Equal(t, "/these", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/")
require.NoError(t, err)
require.Equal(t, "/these/are", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/some")
require.NoError(t, err)
require.Equal(t, "/these/are/some", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/these/are/some/directories")
require.NoError(t, err)
require.Equal(t, "/these/are/some/directories", info.Name())
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
}
func testMkdirAll(t *testing.T, fs Filesystem) {
info, err := fs.Stat("/foo/bar/dir")
require.Error(t, err)
require.Nil(t, info)
err = fs.MkdirAll("/foo/bar/dir", 0755)
require.NoError(t, err)
err = fs.MkdirAll("/foo/bar", 0755)
require.NoError(t, err)
info, err = fs.Stat("/foo/bar/dir")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/foo")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
info, err = fs.Stat("/foo/bar")
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, int64(0), info.Size())
require.Equal(t, true, info.IsDir())
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
err = fs.MkdirAll("/foobar", 0755)
require.Error(t, err)
}
func testRename(t *testing.T, fs Filesystem) {
err := fs.Rename("/foobar", "/foobaz")
require.Error(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
err = fs.Rename("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
}
func testRenameOverwrite(t *testing.T, fs Filesystem) {
_, err := fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
err = fs.Rename("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
data, err := fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlink(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
err := fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.NoError(t, err)
file := fs.Open("/foobaz")
require.NotNil(t, file)
require.Equal(t, "/foobaz", file.Name())
data := make([]byte, 10)
n, err := file.Read(data)
if err != nil {
if err != io.EOF {
require.NoError(t, err)
}
}
require.NoError(t, err)
require.Equal(t, 6, n)
require.Equal(t, "foobar", string(data[:n]))
stat, err := fs.Stat("/foobaz")
require.NoError(t, err)
require.Equal(t, "/foobaz", stat.Name())
require.Equal(t, int64(6), stat.Size())
require.NotEqual(t, 0, int(stat.Mode()&os.ModeSymlink))
link, ok := stat.IsLink()
require.Equal(t, "/foobar", link)
require.Equal(t, true, ok)
data, err = fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlinkOpenStat(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.NoError(t, err)
file := fs.Open("/foobaz")
require.NotNil(t, file)
require.Equal(t, "/foobaz", file.Name())
fstat, err := file.Stat()
require.NoError(t, err)
stat, err := fs.Stat("/foobaz")
require.NoError(t, err)
require.Equal(t, "/foobaz", fstat.Name())
require.Equal(t, fstat.Name(), stat.Name())
require.Equal(t, int64(6), fstat.Size())
require.Equal(t, fstat.Size(), stat.Size())
require.NotEqual(t, 0, int(fstat.Mode()&os.ModeSymlink))
require.Equal(t, fstat.Mode(), stat.Mode())
}
func testStat(t *testing.T, fs Filesystem) {
_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
file := fs.Open("/foobar")
require.NotNil(t, file)
stat1, err := fs.Stat("/foobar")
require.NoError(t, err)
stat2, err := file.Stat()
require.NoError(t, err)
require.Equal(t, stat1, stat2)
}
func testCopy(t *testing.T, fs Filesystem) {
err := fs.Rename("/foobar", "/foobaz")
require.Error(t, err)
_, err = fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
err = fs.Copy("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
}
func testCopyOverwrite(t *testing.T, fs Filesystem) {
_, err := fs.Stat("/foobar")
require.Error(t, err)
_, err = fs.Stat("/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
err = fs.Copy("/foobar", "/foobaz")
require.NoError(t, err)
_, err = fs.Stat("/foobar")
require.NoError(t, err)
_, err = fs.Stat("/foobaz")
require.NoError(t, err)
data, err := fs.ReadFile("/foobaz")
require.NoError(t, err)
require.Equal(t, "foobar", string(data))
}
func testSymlinkErrors(t *testing.T, fs Filesystem) {
if _, ok := fs.(*s3Filesystem); ok {
return
}
err := fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
require.NoError(t, err)
_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
require.NoError(t, err)
err = fs.Symlink("/foobar", "/foobaz")
require.Error(t, err)
err = fs.Symlink("/foobar", "/bazfoo")
require.NoError(t, err)
err = fs.Symlink("/bazfoo", "/barfoo")
require.Error(t, err)
}