apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-10-05 07:57:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add DELETE /api/v3/fs/:storage endpoint

Browse Source 
Works similar to the GET endpoint. It has the same query parameter (without
"sort" and "order"). A glob pattern has to be provided. To select all
files, use the glob pattern "/**".
This commit is contained in:
Ingo Oppermann
2023-03-17 17:05:56 +01:00
parent 7e7aadc6cb
commit a692f88b69
14 changed files with 426 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
docs/docs.go
View File

@@ -467,6 +467,72 @@ const docTemplate = `{
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Delete all files on a filesystem based on patterns",
"produces": [
"application/json"
],
"tags": [
"v16.?.?"
],
"summary": "Delete all files on a filesystem based on patterns",
"operationId": "filesystem-3-delete-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "integer",
"description": "minimal size of files",
"name": "size_min",
"in": "query"
},
{
"type": "integer",
"description": "maximal size of files",
"name": "size_max",
"in": "query"
},
{
"type": "integer",
"description": "minimal last modification time",
"name": "lastmod_start",
"in": "query"
},
{
"type": "integer",
"description": "maximal last modification time",
"name": "lastmod_end",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
},
"/api/v3/fs/{storage}/{filepath}": {
@@ -605,7 +671,7 @@ const docTemplate = `{
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"operationId": "filesystem-3-delete-files",
"parameters": [
{
"type": "string",

68
docs/swagger.json
View File

@@ -460,6 +460,72 @@
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Delete all files on a filesystem based on patterns",
"produces": [
"application/json"
],
"tags": [
"v16.?.?"
],
"summary": "Delete all files on a filesystem based on patterns",
"operationId": "filesystem-3-delete-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "storage",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "integer",
"description": "minimal size of files",
"name": "size_min",
"in": "query"
},
{
"type": "integer",
"description": "maximal size of files",
"name": "size_max",
"in": "query"
},
{
"type": "integer",
"description": "minimal last modification time",
"name": "lastmod_start",
"in": "query"
},
{
"type": "integer",
"description": "maximal last modification time",
"name": "lastmod_end",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
},
"/api/v3/fs/{storage}/{filepath}": {
@@ -598,7 +664,7 @@
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"operationId": "filesystem-3-delete-files",
"parameters": [
{
"type": "string",

45
docs/swagger.yaml
View File

@@ -2205,6 +2205,49 @@ paths:
tags:
- v16.?.?
/api/v3/fs/{storage}:
delete:
description: Delete all files on a filesystem based on patterns
operationId: filesystem-3-delete-files
parameters:
- description: Name of the filesystem
in: path
name: storage
required: true
type: string
- description: glob pattern for file names
in: query
name: glob
type: string
- description: minimal size of files
in: query
name: size_min
type: integer
- description: maximal size of files
in: query
name: size_max
type: integer
- description: minimal last modification time
in: query
name: lastmod_start
type: integer
- description: maximal last modification time
in: query
name: lastmod_end
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
type: string
type: array
security:
- ApiKeyAuth: []
summary: Delete all files on a filesystem based on patterns
tags:
- v16.?.?
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
@@ -2260,7 +2303,7 @@ paths:
/api/v3/fs/{storage}/{filepath}:
delete:
description: Remove a file from a filesystem
operationId: filesystem-3-delete-file
operationId: filesystem-3-delete-files
parameters:
- description: Name of the filesystem
in: path

42
http/handler/api/filesystems.go
View File

@@ -29,7 +29,7 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler {
}
}
// GetFileAPI returns the file at the given path
// GetFile returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @Tags v16.7.2
@@ -44,7 +44,7 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler {
// @Security ApiKeyAuth
// @Router /api/v3/fs/{storage}/{filepath} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
name := util.PathParam(c, "storage")
config, ok := h.filesystems[name]
if !ok {
@@ -54,7 +54,7 @@ func (h *FSHandler) GetFile(c echo.Context) error {
return config.Handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @Tags v16.7.2
@@ -71,7 +71,7 @@ func (h *FSHandler) GetFile(c echo.Context) error {
// @Security ApiKeyAuth
// @Router /api/v3/fs/{storage}/{filepath} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
name := util.PathParam(c, "storage")
config, ok := h.filesystems[name]
if !ok {
@@ -81,11 +81,11 @@ func (h *FSHandler) PutFile(c echo.Context) error {
return config.Handler.PutFile(c)
}
// DeleteFileAPI removes a file from a filesystem
// DeleteFile removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @Tags v16.7.2
// @ID filesystem-3-delete-file
// @ID filesystem-3-delete-files
// @Produce text/plain
// @Param storage path string true "Name of the filesystem"
// @Param filepath path string true "Path to file"
@@ -94,7 +94,7 @@ func (h *FSHandler) PutFile(c echo.Context) error {
// @Security ApiKeyAuth
// @Router /api/v3/fs/{storage}/{filepath} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
name := util.PathParam(c, "storage")
config, ok := h.filesystems[name]
if !ok {
@@ -104,6 +104,32 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
return config.Handler.DeleteFile(c)
}
// DeleteFiles deletes all files on a filesystem based on patterns
// @Summary Delete all files on a filesystem based on patterns
// @Description Delete all files on a filesystem based on patterns
// @Tags v16.?.?
// @ID filesystem-3-delete-files
// @Produce json
// @Param storage path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param size_min query int64 false "minimal size of files"
// @Param size_max query int64 false "maximal size of files"
// @Param lastmod_start query int64 false "minimal last modification time"
// @Param lastmod_end query int64 false "maximal last modification time"
// @Success 200 {array} string
// @Security ApiKeyAuth
// @Router /api/v3/fs/{storage} [delete]
func (h *FSHandler) DeleteFiles(c echo.Context) error {
name := util.PathParam(c, "storage")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.DeleteFiles(c)
}
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
@@ -122,7 +148,7 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
// @Security ApiKeyAuth
// @Router /api/v3/fs/{storage} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
name := util.PathParam(c, "storage")
config, ok := h.filesystems[name]
if !ok {

138
http/handler/api/filesystems_test.go
View File

@@ -43,10 +43,11 @@ func getDummyFilesystemsRouter(filesystems []httpfs.FS) (*echo.Echo, error) {
return nil, err
}
router.GET("/:name/*", handler.GetFile)
router.PUT("/:name/*", handler.PutFile)
router.DELETE("/:name/*", handler.DeleteFile)
router.GET("/:name", handler.ListFiles)
router.GET("/:storage/*", handler.GetFile)
router.PUT("/:storage/*", handler.PutFile)
router.DELETE("/:storage", handler.DeleteFiles)
router.DELETE("/:storage/*", handler.DeleteFile)
router.GET("/:storage", handler.ListFiles)
router.GET("/", handler.List)
router.PUT("/", handler.FileOperation)
@@ -206,11 +207,11 @@ func TestFilesystemsListLastmod(t *testing.T) {
require.NoError(t, err)
memfs.WriteFileReader("/a", strings.NewReader("a"))
time.Sleep(500 * time.Millisecond)
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/b", strings.NewReader("b"))
time.Sleep(500 * time.Millisecond)
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/c", strings.NewReader("c"))
time.Sleep(500 * time.Millisecond)
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/d", strings.NewReader("d"))
var a, b, c, d time.Time
@@ -268,6 +269,129 @@ func TestFilesystemsListLastmod(t *testing.T) {
require.ElementsMatch(t, []string{"/b", "/c"}, files)
}
func TestFilesystemsDeleteFiles(t *testing.T) {
memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
memfs.WriteFileReader("/a", strings.NewReader("a"))
memfs.WriteFileReader("/aa", strings.NewReader("aa"))
memfs.WriteFileReader("/aaa", strings.NewReader("aaa"))
memfs.WriteFileReader("/aaaa", strings.NewReader("aaaa"))
filesystems := []httpfs.FS{
{
Name: "foo",
Mountpoint: "/foo",
AllowWrite: true,
Filesystem: memfs,
},
}
router, err := getDummyFilesystemsRouter(filesystems)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "DELETE", "/foo", nil)
getNames := func(r *mock.Response) []string {
files := []string{}
err := json.Unmarshal(r.Raw, &files)
require.NoError(t, err)
return files
}
files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**", nil))
require.Equal(t, 4, len(files))
require.ElementsMatch(t, []string{"/a", "/aa", "/aaa", "/aaaa"}, files)
require.Equal(t, int64(0), memfs.Files())
}
func TestFilesystemsDeleteFilesSize(t *testing.T) {
memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
memfs.WriteFileReader("/a", strings.NewReader("a"))
memfs.WriteFileReader("/aa", strings.NewReader("aa"))
memfs.WriteFileReader("/aaa", strings.NewReader("aaa"))
memfs.WriteFileReader("/aaaa", strings.NewReader("aaaa"))
filesystems := []httpfs.FS{
{
Name: "foo",
Mountpoint: "/foo",
AllowWrite: true,
Filesystem: memfs,
},
}
router, err := getDummyFilesystemsRouter(filesystems)
require.NoError(t, err)
getNames := func(r *mock.Response) []string {
files := []string{}
err := json.Unmarshal(r.Raw, &files)
require.NoError(t, err)
return files
}
files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**&size_min=2&size_max=3", nil))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/aa", "/aaa"}, files)
require.Equal(t, int64(2), memfs.Files())
}
func TestFilesystemsDeleteFilesLastmod(t *testing.T) {
memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
memfs.WriteFileReader("/a", strings.NewReader("a"))
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/b", strings.NewReader("b"))
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/c", strings.NewReader("c"))
time.Sleep(1 * time.Second)
memfs.WriteFileReader("/d", strings.NewReader("d"))
var b, c time.Time
for _, f := range memfs.List("/", fs.ListOptions{}) {
if f.Name() == "/b" {
b = f.ModTime()
} else if f.Name() == "/c" {
c = f.ModTime()
}
}
filesystems := []httpfs.FS{
{
Name: "foo",
Mountpoint: "/foo",
AllowWrite: true,
Filesystem: memfs,
},
}
router, err := getDummyFilesystemsRouter(filesystems)
require.NoError(t, err)
getNames := func(r *mock.Response) []string {
files := []string{}
err := json.Unmarshal(r.Raw, &files)
require.NoError(t, err)
return files
}
files := getNames(mock.Request(t, http.StatusOK, router, "DELETE", "/foo?glob=/**&lastmod_start="+strconv.FormatInt(b.Unix(), 10)+"&lastmod_end="+strconv.FormatInt(c.Unix(), 10), nil))
require.Equal(t, 2, len(files))
require.ElementsMatch(t, []string{"/b", "/c"}, files)
require.Equal(t, int64(2), memfs.Files())
}
func TestFileOperation(t *testing.T) {
memfs1, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)

56
http/handler/filesystem.go
View File

@@ -170,6 +170,62 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
return c.String(http.StatusOK, "Deleted: "+path)
}
func (h *FSHandler) DeleteFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sizeMin := util.DefaultQuery(c, "size_min", "0")
sizeMax := util.DefaultQuery(c, "size_max", "0")
modifiedStart := util.DefaultQuery(c, "lastmod_start", "")
modifiedEnd := util.DefaultQuery(c, "lastmod_end", "")
if len(pattern) == 0 {
return api.Err(http.StatusBadRequest, "Bad request", "A glob pattern is required")
}
options := fs.ListOptions{
Pattern: pattern,
}
if x, err := strconv.ParseInt(sizeMin, 10, 64); err != nil {
return api.Err(http.StatusBadRequest, "Bad request", "%s", err)
} else {
options.SizeMin = x
}
if x, err := strconv.ParseInt(sizeMax, 10, 64); err != nil {
return api.Err(http.StatusBadRequest, "Bad request", "%s", err)
} else {
options.SizeMax = x
}
if len(modifiedStart) != 0 {
if x, err := strconv.ParseInt(modifiedStart, 10, 64); err != nil {
return api.Err(http.StatusBadRequest, "Bad request", "%s", err)
} else {
t := time.Unix(x, 0)
options.ModifiedStart = &t
}
}
if len(modifiedEnd) != 0 {
if x, err := strconv.ParseInt(modifiedEnd, 10, 64); err != nil {
return api.Err(http.StatusBadRequest, "Bad request", "%s", err)
} else {
t := time.Unix(x+1, 0)
options.ModifiedEnd = &t
}
}
paths, _ := h.FS.Filesystem.RemoveList("/", options)
if h.FS.Cache != nil {
for _, path := range paths {
h.FS.Cache.Delete(path)
}
}
return c.JSON(http.StatusOK, paths)
}
func (h *FSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sizeMin := util.DefaultQuery(c, "size_min", "0")

11
http/server.go
View File

@@ -607,19 +607,20 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
v3.GET("/fs", handler.List)
v3.PUT("/fs", handler.FileOperation)
v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.GET("/fs/:storage", handler.ListFiles)
v3.GET("/fs/:storage/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.HEAD("/fs/:storage/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
if !s.readOnly {
v3.PUT("/fs/:name/*", handler.PutFile)
v3.DELETE("/fs/:name/*", handler.DeleteFile)
v3.PUT("/fs/:storage/*", handler.PutFile)
v3.DELETE("/fs/:storage", handler.DeleteFiles)
v3.DELETE("/fs/:storage/*", handler.DeleteFile)
}
// v3 RTMP

6
io/fs/disk.go
View File

@@ -519,10 +519,11 @@ func (fs *diskFilesystem) Remove(path string) int64 {
return size
}
func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 {
func (fs *diskFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) {
path = fs.cleanPath(path)
var size int64 = 0
files := []string{}
fs.walk(path, func(path string, info os.FileInfo) {
if path == fs.root {
@@ -569,11 +570,12 @@ func (fs *diskFilesystem) RemoveList(path string, options ListOptions) int64 {
}
if err := os.Remove(path); err == nil {
files = append(files, name)
size += info.Size()
}
})
return size
return files, size
}
func (fs *diskFilesystem) List(path string, options ListOptions) []FileInfo {

2
io/fs/fs.go
View File

@@ -122,7 +122,7 @@ type WriteFilesystem interface {
// RemoveList removes all files from the filesystem. Returns the size of the
// removed files in bytes.
RemoveList(path string, options ListOptions) int64
RemoveList(path string, options ListOptions) ([]string, int64)
}
// Filesystem is an interface that provides access to a filesystem.

4
io/fs/fs_test.go
View File

@@ -472,7 +472,7 @@ func testRemoveAll(t *testing.T, fs Filesystem) {
require.Equal(t, int64(4), cur)
size := fs.RemoveList("/", ListOptions{
_, size := fs.RemoveList("/", ListOptions{
Pattern: "",
})
require.Equal(t, int64(12), size)
@@ -492,7 +492,7 @@ func testRemoveList(t *testing.T, fs Filesystem) {
require.Equal(t, int64(4), cur)
size := fs.RemoveList("/", ListOptions{
_, size := fs.RemoveList("/", ListOptions{
Pattern: "/path/**",
})
require.Equal(t, int64(6), size)

7
io/fs/mem.go
View File

@@ -675,13 +675,14 @@ func (fs *memFilesystem) remove(path string) int64 {
return file.size
}
func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 {
func (fs *memFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) {
path = fs.cleanPath(path)
fs.filesLock.Lock()
defer fs.filesLock.Unlock()
var size int64 = 0
files := []string{}
for _, file := range fs.files {
if !strings.HasPrefix(file.name, path) {
@@ -723,9 +724,11 @@ func (fs *memFilesystem) RemoveList(path string, options ListOptions) int64 {
}
size += fs.remove(file.name)
files = append(files, file.name)
}
return size
return files, size
}
func (fs *memFilesystem) List(path string, options ListOptions) []FileInfo {

4
io/fs/readonly.go
View File

@@ -41,8 +41,8 @@ func (r *readOnlyFilesystem) Remove(path string) int64 {
return -1
}
func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) int64 {
return 0
func (r *readOnlyFilesystem) RemoveList(path string, options ListOptions) ([]string, int64) {
return nil, 0
}
func (r *readOnlyFilesystem) Purge(size int64) int64 {

2
io/fs/readonly_test.go
View File

@@ -32,7 +32,7 @@ func TestReadOnly(t *testing.T) {
res := ro.Remove("/readonly.go")
require.Equal(t, int64(-1), res)
res = ro.RemoveList("/", ListOptions{})
_, res = ro.RemoveList("/", ListOptions{})
require.Equal(t, int64(0), res)
rop, ok := ro.(PurgeFilesystem)

9
io/fs/s3.go
View File

@@ -428,13 +428,14 @@ func (fs *s3Filesystem) Remove(path string) int64 {
return stat.Size
}
func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 {
func (fs *s3Filesystem) RemoveList(path string, options ListOptions) ([]string, int64) {
path = fs.cleanPath(path)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
totalSize := int64(0)
var totalSize int64 = 0
files := []string{}
objectsCh := make(chan minio.ObjectInfo)
@@ -493,6 +494,8 @@ func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 {
totalSize += object.Size
objectsCh <- object
files = append(files, key)
}
}()
@@ -504,7 +507,7 @@ func (fs *s3Filesystem) RemoveList(path string, options ListOptions) int64 {
fs.logger.Debug().Log("Deleted all files")
return totalSize
return files, totalSize
}
func (fs *s3Filesystem) List(path string, options ListOptions) []FileInfo {