apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-10-04 23:53:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add PUT /api/v3/fs endpoint for file operations

Browse Source
This commit is contained in:
Ingo Oppermann
2023-03-03 14:26:17 +01:00
parent eb3f396793
commit a3ff16ef30
10 changed files with 486 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
docs/docs.go
View File

@@ -321,6 +321,9 @@ const docTemplate = `{
"produces": [
"application/json"
],
"tags": [
"v16.12.0"
],
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
@@ -334,6 +337,56 @@ const docTemplate = `{
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Execute file operations (copy or move) between registered filesystems",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"v16.?.?"
],
"summary": "File operations between filesystems",
"operationId": "filesystem-3-file-operation",
"parameters": [
{
"description": "Filesystem operation",
"name": "config",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.FilesystemOperation"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/fs/{name}": {
@@ -347,6 +400,9 @@ const docTemplate = `{
"produces": [
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
@@ -401,6 +457,9 @@ const docTemplate = `{
"application/data",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
@@ -454,6 +513,9 @@ const docTemplate = `{
"text/plain",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
@@ -515,6 +577,9 @@ const docTemplate = `{
"produces": [
"text/plain"
],
"tags": [
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
@@ -2785,6 +2850,27 @@ const docTemplate = `{
}
}
},
"api.FilesystemOperation": {
"type": "object",
"required": [
"operation"
],
"properties": {
"from": {
"type": "string"
},
"operation": {
"type": "string",
"enum": [
"copy",
"move"
]
},
"to": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {

86
docs/swagger.json
View File

@@ -314,6 +314,9 @@
"produces": [
"application/json"
],
"tags": [
"v16.12.0"
],
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
@@ -327,6 +330,56 @@
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Execute file operations (copy or move) between registered filesystems",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"v16.?.?"
],
"summary": "File operations between filesystems",
"operationId": "filesystem-3-file-operation",
"parameters": [
{
"description": "Filesystem operation",
"name": "config",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.FilesystemOperation"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/fs/{name}": {
@@ -340,6 +393,9 @@
"produces": [
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
@@ -394,6 +450,9 @@
"application/data",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
@@ -447,6 +506,9 @@
"text/plain",
"application/json"
],
"tags": [
"v16.7.2"
],
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
@@ -508,6 +570,9 @@
"produces": [
"text/plain"
],
"tags": [
"v16.7.2"
],
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
@@ -2778,6 +2843,27 @@
}
}
},
"api.FilesystemOperation": {
"type": "object",
"required": [
"operation"
],
"properties": {
"from": {
"type": "string"
},
"operation": {
"type": "string",
"enum": [
"copy",
"move"
]
},
"to": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {

56
docs/swagger.yaml
View File

@@ -456,6 +456,20 @@ definitions:
type:
type: string
type: object
api.FilesystemOperation:
properties:
from:
type: string
operation:
enum:
- copy
- move
type: string
to:
type: string
required:
- operation
type: object
api.GraphQuery:
properties:
query:
@@ -2142,6 +2156,40 @@ paths:
security:
- ApiKeyAuth: []
summary: List all registered filesystems
tags:
- v16.12.0
put:
consumes:
- application/json
description: Execute file operations (copy or move) between registered filesystems
operationId: filesystem-3-file-operation
parameters:
- description: Filesystem operation
in: body
name: config
required: true
schema:
$ref: '#/definitions/api.FilesystemOperation'
produces:
- application/json
responses:
"200":
description: OK
schema:
type: string
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.Error'
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: File operations between filesystems
tags:
- v16.?.?
/api/v3/fs/{name}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
@@ -2177,6 +2225,8 @@ paths:
security:
- ApiKeyAuth: []
summary: List all files on a filesystem
tags:
- v16.7.2
/api/v3/fs/{name}/{path}:
delete:
description: Remove a file from a filesystem
@@ -2206,6 +2256,8 @@ paths:
security:
- ApiKeyAuth: []
summary: Remove a file from a filesystem
tags:
- v16.7.2
get:
description: Fetch a file from a filesystem
operationId: filesystem-3-get-file
@@ -2239,6 +2291,8 @@ paths:
security:
- ApiKeyAuth: []
summary: Fetch a file from a filesystem
tags:
- v16.7.2
put:
consumes:
- application/data
@@ -2282,6 +2336,8 @@ paths:
security:
- ApiKeyAuth: []
summary: Add a file to a filesystem
tags:
- v16.7.2
/api/v3/log:
get:
description: Get the last log lines of the Restreamer application

9
http/api/fs.go
View File

@@ -1,6 +1,6 @@
package api
// FileInfo represents informatiion about a file on a filesystem
// FileInfo represents information about a file on a filesystem
type FileInfo struct {
Name string `json:"name" jsonschema:"minLength=1"`
Size int64 `json:"size_bytes" jsonschema:"minimum=0" format:"int64"`
@@ -13,3 +13,10 @@ type FilesystemInfo struct {
Type string `json:"type"`
Mount string `json:"mount"`
}
// FilesystemOperation represents a file operation on one or more filesystems
type FilesystemOperation struct {
Operation string `json:"operation" validate:"required" enums:"copy,move" jsonschema:"enum=copy,enum=move"`
From string `json:"from"`
To string `json:"to"`
}

77
http/handler/api/filesystems.go
View File

@@ -2,6 +2,8 @@ package api
import (
"net/http"
"path/filepath"
"strings"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
@@ -32,6 +34,7 @@ func NewFS(filesystems map[string]FSConfig) *FSHandler {
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @ID filesystem-3-get-file
// @Tags v16.7.2
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
@@ -56,6 +59,7 @@ func (h *FSHandler) GetFile(c echo.Context) error {
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @ID filesystem-3-put-file
// @Tags v16.7.2
// @Accept application/data
// @Produce text/plain
// @Produce json
@@ -82,6 +86,7 @@ func (h *FSHandler) PutFile(c echo.Context) error {
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @ID filesystem-3-delete-file
// @Tags v16.7.2
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
@@ -104,6 +109,7 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID filesystem-3-list-files
// @Tags v16.7.2
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
@@ -127,6 +133,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error {
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @ID filesystem-3-list
// @Tags v16.12.0
// @Produce json
// @Success 200 {array} api.FilesystemInfo
// @Security ApiKeyAuth
@@ -144,3 +151,73 @@ func (h *FSHandler) List(c echo.Context) error {
return c.JSON(http.StatusOK, fss)
}
// FileOperation executes file operations between filesystems
// @Summary File operations between filesystems
// @Description Execute file operations (copy or move) between registered filesystems
// @ID filesystem-3-file-operation
// @Tags v16.?.?
// @Accept json
// @Produce json
// @Param config body api.FilesystemOperation true "Filesystem operation"
// @Success 200 {string} string
// @Failure 400 {object} api.Error
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs [put]
func (h *FSHandler) FileOperation(c echo.Context) error {
operation := api.FilesystemOperation{}
if err := util.ShouldBindJSON(c, &operation); err != nil {
return api.Err(http.StatusBadRequest, "Invalid JSON", "%s", err)
}
if operation.Operation != "copy" && operation.Operation != "move" {
return api.Err(http.StatusBadRequest, "Invalid operation", "%s", operation.Operation)
}
from := strings.Split(filepath.Join("/", operation.From), "/")
if len(from) < 2 {
return api.Err(http.StatusBadRequest, "Invalid source path", "%s", operation.From)
}
fromFSName := from[1]
fromPath := strings.Join(from[2:], "/")
fromFS, ok := h.filesystems[fromFSName]
if !ok {
return api.Err(http.StatusBadRequest, "Source filesystem not found", "%s", fromFSName)
}
if operation.From == operation.To {
return c.JSON(http.StatusOK, "OK")
}
to := strings.Split(filepath.Join("/", operation.To), "/")
if len(to) < 2 {
return api.Err(http.StatusBadRequest, "Invalid target path", "%s", operation.To)
}
toFSName := to[1]
toPath := strings.Join(to[2:], "/")
toFS, ok := h.filesystems[toFSName]
if !ok {
return api.Err(http.StatusBadRequest, "Target filesystem not found", "%s", toFSName)
}
fromFile := fromFS.Handler.FS.Filesystem.Open(fromPath)
if fromFile == nil {
return api.Err(http.StatusNotFound, "File not found", "%s:%s", fromFSName, fromPath)
}
defer fromFile.Close()
_, _, err := toFS.Handler.FS.Filesystem.WriteFileReader(toPath, fromFile)
if err != nil {
toFS.Handler.FS.Filesystem.Remove(toPath)
return api.Err(http.StatusBadRequest, "Writing target file failed", "%s", err)
}
if operation.Operation == "move" {
fromFS.Handler.FS.Filesystem.Remove(fromPath)
}
return c.JSON(http.StatusOK, "OK")
}

172
http/handler/api/filesystems_test.go
View File

@@ -1,6 +1,7 @@
package api
import (
"bytes"
"encoding/json"
"io"
"net/http"
@@ -16,36 +17,25 @@ import (
"github.com/labstack/echo/v4"
)
func getDummyFilesystemsHandler(filesystem httpfs.FS) (*FSHandler, error) {
handler := NewFS(map[string]FSConfig{
filesystem.Name: {
Type: filesystem.Filesystem.Type(),
Mountpoint: filesystem.Mountpoint,
Handler: handler.NewFS(filesystem),
},
})
func getDummyFilesystemsHandler(filesystems []httpfs.FS) (*FSHandler, error) {
config := map[string]FSConfig{}
for _, fs := range filesystems {
config[fs.Name] = FSConfig{
Type: fs.Filesystem.Type(),
Mountpoint: fs.Mountpoint,
Handler: handler.NewFS(fs),
}
}
handler := NewFS(config)
return handler, nil
}
func getDummyFilesystemsRouter(filesystem fs.Filesystem) (*echo.Echo, error) {
func getDummyFilesystemsRouter(filesystems []httpfs.FS) (*echo.Echo, error) {
router := mock.DummyEcho()
fs := httpfs.FS{
Name: "foo",
Mountpoint: "/",
AllowWrite: true,
EnableAuth: false,
Username: "",
Password: "",
DefaultFile: "",
DefaultContentType: "text/html",
Gzip: false,
Filesystem: filesystem,
Cache: nil,
}
handler, err := getDummyFilesystemsHandler(fs)
handler, err := getDummyFilesystemsHandler(filesystems)
if err != nil {
return nil, err
}
@@ -55,6 +45,7 @@ func getDummyFilesystemsRouter(filesystem fs.Filesystem) (*echo.Echo, error) {
router.DELETE("/:name/*", handler.DeleteFile)
router.GET("/:name", handler.ListFiles)
router.GET("/", handler.List)
router.PUT("/", handler.FileOperation)
return router, nil
}
@@ -63,7 +54,16 @@ func TestFilesystems(t *testing.T) {
memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
router, err := getDummyFilesystemsRouter(memfs)
filesystems := []httpfs.FS{
{
Name: "foo",
Mountpoint: "/",
AllowWrite: true,
Filesystem: memfs,
},
}
router, err := getDummyFilesystemsRouter(filesystems)
require.NoError(t, err)
response := mock.Request(t, http.StatusOK, router, "GET", "/", nil)
@@ -139,3 +139,125 @@ func TestFilesystems(t *testing.T) {
require.Equal(t, 0, len(l))
}
func TestFileOperation(t *testing.T) {
memfs1, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
memfs2, err := fs.NewMemFilesystem(fs.MemConfig{})
require.NoError(t, err)
filesystems := []httpfs.FS{
{
Name: "foo",
Mountpoint: "/foo",
AllowWrite: true,
Filesystem: memfs1,
},
{
Name: "bar",
Mountpoint: "/bar",
AllowWrite: true,
Filesystem: memfs2,
},
}
router, err := getDummyFilesystemsRouter(filesystems)
require.NoError(t, err)
mock.Request(t, http.StatusNotFound, router, "GET", "/foo/file", nil)
mock.Request(t, http.StatusNotFound, router, "GET", "/bar/file", nil)
data := mock.Read(t, "./fixtures/addProcess.json")
require.NotNil(t, data)
mock.Request(t, http.StatusCreated, router, "PUT", "/foo/file", data)
mock.Request(t, http.StatusOK, router, "GET", "/foo/file", nil)
mock.Request(t, http.StatusNotFound, router, "GET", "/bar/file", nil)
op := api.FilesystemOperation{}
jsondata, err := json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/elif",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/elif",
To: "/bar",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusNotFound, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/file",
To: "/bar",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusBadRequest, router, "PUT", "/", bytes.NewReader(jsondata))
op = api.FilesystemOperation{
Operation: "copy",
From: "foo/file",
To: "/bar/file",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusOK, router, "PUT", "/", bytes.NewReader(jsondata))
mock.Request(t, http.StatusOK, router, "GET", "/foo/file", nil)
response := mock.Request(t, http.StatusOK, router, "GET", "/bar/file", nil)
filedata, err := io.ReadAll(mock.Read(t, "./fixtures/addProcess.json"))
require.NoError(t, err)
require.Equal(t, filedata, response.Raw)
op = api.FilesystemOperation{
Operation: "move",
From: "foo/file",
To: "/bar/file",
}
jsondata, err = json.Marshal(op)
require.NoError(t, err)
mock.Request(t, http.StatusOK, router, "PUT", "/", bytes.NewReader(jsondata))
mock.Request(t, http.StatusNotFound, router, "GET", "/foo/file", nil)
response = mock.Request(t, http.StatusOK, router, "GET", "/bar/file", nil)
filedata, err = io.ReadAll(mock.Read(t, "./fixtures/addProcess.json"))
require.NoError(t, err)
require.Equal(t, filedata, response.Raw)
}

26
http/handler/filesystem.go
View File

@@ -14,13 +14,13 @@ import (
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
fs fs.FS
FS fs.FS
}
// NewFS return a new FSHandler type. You have to provide a filesystem to act on.
func NewFS(fs fs.FS) *FSHandler {
return &FSHandler{
fs: fs,
FS: fs,
}
}
@@ -30,20 +30,20 @@ func (h *FSHandler) GetFile(c echo.Context) error {
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.fs.Filesystem.Open(path)
file := h.FS.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if len(h.fs.DefaultFile) != 0 {
if len(h.FS.DefaultFile) != 0 {
if stat.IsDir() {
path = filepath.Join(path, h.fs.DefaultFile)
path = filepath.Join(path, h.FS.DefaultFile)
file.Close()
file = h.fs.Filesystem.Open(path)
file = h.FS.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
@@ -82,13 +82,13 @@ func (h *FSHandler) PutFile(c echo.Context) error {
req := c.Request()
_, created, err := h.fs.Filesystem.WriteFileReader(path, req.Body)
_, created, err := h.FS.Filesystem.WriteFileReader(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "Bad request", "%s", err)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
if h.FS.Cache != nil {
h.FS.Cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
@@ -105,14 +105,14 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
c.Response().Header().Del(echo.HeaderContentType)
size := h.fs.Filesystem.Remove(path)
size := h.FS.Filesystem.Remove(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
if h.FS.Cache != nil {
h.FS.Cache.Delete(path)
}
return c.String(http.StatusOK, "Deleted: "+path)
@@ -123,7 +123,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error {
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.fs.Filesystem.List("/", pattern)
files := h.FS.Filesystem.List("/", pattern)
var sortFunc func(i, j int) bool

1
http/server.go
View File

@@ -605,6 +605,7 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
handler := api.NewFS(fshandlers)
v3.GET("/fs", handler.List)
v3.PUT("/fs", handler.FileOperation)
v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{

6
io/fs/fs_test.go
View File

@@ -87,6 +87,7 @@ func TestFilesystem(t *testing.T) {
"writeFile": testWriteFile,
"writeFileSafe": testWriteFileSafe,
"writeFileReader": testWriteFileReader,
"writeFileDir": testWriteFileDir,
"delete": testDelete,
"files": testFiles,
"replace": testReplace,
@@ -198,6 +199,11 @@ func testWriteFileReader(t *testing.T, fs Filesystem) {
require.Equal(t, int64(1), cur)
}
func testWriteFileDir(t *testing.T, fs Filesystem) {
_, _, err := fs.WriteFile("/", []byte("xxxxx"))
require.Error(t, err)
}
func testOpen(t *testing.T, fs Filesystem) {
file := fs.Open("/foobar")
require.Nil(t, file)

6
io/fs/mem.go
View File

@@ -342,6 +342,10 @@ func (fs *memFilesystem) Symlink(oldname, newname string) error {
func (fs *memFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
path = fs.cleanPath(path)
if fs.isDir(path) {
return -1, false, fmt.Errorf("path not writeable")
}
newFile := &memFile{
memFileInfo: memFileInfo{
name: path,
@@ -360,6 +364,8 @@ func (fs *memFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool,
"filesize_bytes": size,
"error": err,
}).Warn().Log("Incomplete file")
return -1, false, fmt.Errorf("incomplete file")
}
newFile.size = size