Refactor internal filesystem handling

This commit is contained in:
Ingo Oppermann
2022-08-22 19:10:59 +03:00
parent 9a49e371e3
commit cb0bc494f9
18 changed files with 955 additions and 1981 deletions

View File

@@ -19,6 +19,7 @@ import (
"github.com/datarhei/core/v16/ffmpeg" "github.com/datarhei/core/v16/ffmpeg"
"github.com/datarhei/core/v16/http" "github.com/datarhei/core/v16/http"
"github.com/datarhei/core/v16/http/cache" "github.com/datarhei/core/v16/http/cache"
httpfs "github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/io/fs" "github.com/datarhei/core/v16/io/fs"
@@ -71,7 +72,6 @@ type api struct {
prom prometheus.Metrics prom prometheus.Metrics
service service.Service service service.Service
sessions session.Registry sessions session.Registry
sessionsLimiter net.IPLimiter
cache cache.Cacher cache cache.Cacher
mainserver *gohttp.Server mainserver *gohttp.Server
sidecarserver *gohttp.Server sidecarserver *gohttp.Server
@@ -372,7 +372,7 @@ func (a *api) start() error {
diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{ diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
Dir: cfg.Storage.Disk.Dir, Dir: cfg.Storage.Disk.Dir,
Size: cfg.Storage.Disk.Size * 1024 * 1024, Size: cfg.Storage.Disk.Size * 1024 * 1024,
Logger: a.log.logger.core.WithComponent("DiskFS"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
if err != nil { if err != nil {
return err return err
@@ -401,7 +401,7 @@ func (a *api) start() error {
Base: baseMemFS.String(), Base: baseMemFS.String(),
Size: cfg.Storage.Memory.Size * 1024 * 1024, Size: cfg.Storage.Memory.Size * 1024 * 1024,
Purge: cfg.Storage.Memory.Purge, Purge: cfg.Storage.Memory.Purge,
Logger: a.log.logger.core.WithComponent("MemFS"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
a.memfs = memfs a.memfs = memfs
@@ -435,7 +435,7 @@ func (a *api) start() error {
Region: cfg.Storage.S3.Region, Region: cfg.Storage.S3.Region,
Bucket: cfg.Storage.S3.Bucket, Bucket: cfg.Storage.S3.Bucket,
UseSSL: cfg.Storage.S3.UseSSL, UseSSL: cfg.Storage.S3.UseSSL,
Logger: a.log.logger.core.WithComponent("S3"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
if err != nil { if err != nil {
return err return err
@@ -665,7 +665,7 @@ func (a *api) start() error {
} }
if cfg.Storage.Disk.Cache.Enable { if cfg.Storage.Disk.Cache.Enable {
diskCache, err := cache.NewLRUCache(cache.LRUConfig{ cache, err := cache.NewLRUCache(cache.LRUConfig{
TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second,
MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024,
MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024,
@@ -675,10 +675,10 @@ func (a *api) start() error {
}) })
if err != nil { if err != nil {
return fmt.Errorf("unable to create disk cache: %w", err) return fmt.Errorf("unable to create cache: %w", err)
} }
a.cache = diskCache a.cache = cache
} }
var autocertManager *autocert.Manager var autocertManager *autocert.Manager
@@ -829,25 +829,50 @@ func (a *api) start() error {
a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address) a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address)
mainserverhandler, err := http.NewServer(http.Config{ serverConfig := http.Config{
Logger: a.log.logger.main, Logger: a.log.logger.main,
LogBuffer: a.log.buffer, LogBuffer: a.log.buffer,
Restream: a.restream, Restream: a.restream,
Metrics: a.metrics, Metrics: a.metrics,
Prometheus: a.prom, Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes, MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs, Filesystems: []httpfs.FS{
MemFS: http.MemFSConfig{ {
EnableAuth: cfg.Storage.Memory.Auth.Enable, Name: "diskfs",
Mountpoint: "/",
AllowWrite: false,
Username: "",
Password: "",
DefaultFile: "index.html",
DefaultContentType: "text/html",
Gzip: true,
Filesystem: diskfs,
Cache: a.cache,
},
{
Name: "memfs",
Mountpoint: "/memfs",
AllowWrite: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username, Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password, Password: cfg.Storage.Memory.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.memfs, Filesystem: a.memfs,
Cache: a.cache,
}, },
S3FS: http.MemFSConfig{ {
EnableAuth: cfg.Storage.S3.Auth.Enable, Name: "s3fs",
Mountpoint: "/s3",
AllowWrite: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username, Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password, Password: cfg.Storage.S3.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.s3fs, Filesystem: a.s3fs,
Cache: a.cache,
},
}, },
IPLimiter: iplimiter, IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling, Profiling: cfg.Debug.Profiling,
@@ -858,11 +883,12 @@ func (a *api) start() error {
SRT: a.srtserver, SRT: a.srtserver,
JWT: a.httpjwt, JWT: a.httpjwt,
Config: a.config.store, Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions, Sessions: a.sessions,
Router: router, Router: router,
ReadOnly: cfg.API.ReadOnly, ReadOnly: cfg.API.ReadOnly,
}) }
mainserverhandler, err := http.NewServer(serverConfig)
if err != nil { if err != nil {
return fmt.Errorf("unable to create server: %w", err) return fmt.Errorf("unable to create server: %w", err)
@@ -897,40 +923,10 @@ func (a *api) start() error {
a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address) a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address)
sidecarserverhandler, err := http.NewServer(http.Config{ serverConfig.Logger = a.log.logger.sidecar
Logger: a.log.logger.sidecar, serverConfig.IPLimiter = iplimiter
LogBuffer: a.log.buffer,
Restream: a.restream, sidecarserverhandler, err := http.NewServer(serverConfig)
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
Filesystem: a.memfs,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
Filesystem: a.s3fs,
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
Cors: http.CorsConfig{
Origins: cfg.Storage.CORS.Origins,
},
RTMP: a.rtmpserver,
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
if err != nil { if err != nil {
return fmt.Errorf("unable to create sidecar HTTP server: %w", err) return fmt.Errorf("unable to create sidecar HTTP server: %w", err)

View File

@@ -302,20 +302,53 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/disk": { "/api/v3/fs": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", "description": "Listall registered filesystems",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"summary": "List all files on the filesystem", "summary": "List all registered filesystems",
"operationId": "diskfs-3-list-files", "operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "glob pattern for file names", "description": "glob pattern for file names",
@@ -348,21 +381,28 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/disk/{path}": { "/api/v3/fs/{name}/{path}": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Fetch a file from the filesystem. The contents of that file are returned.", "description": "Fetch a file from a filesystem",
"produces": [ "produces": [
"application/data", "application/data",
"application/json" "application/json"
], ],
"summary": "Fetch a file from the filesystem", "summary": "Fetch a file from a filesystem",
"operationId": "diskfs-3-get-file", "operationId": "filesystem-3-get-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -398,7 +438,7 @@ const docTemplate = `{
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Writes or overwrites a file on the filesystem", "description": "Writes or overwrites a file on a filesystem",
"consumes": [ "consumes": [
"application/data" "application/data"
], ],
@@ -406,9 +446,16 @@ const docTemplate = `{
"text/plain", "text/plain",
"application/json" "application/json"
], ],
"summary": "Add a file to the filesystem", "summary": "Add a file to a filesystem",
"operationId": "diskfs-3-put-file", "operationId": "filesystem-3-put-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -456,13 +503,20 @@ const docTemplate = `{
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Remove a file from the filesystem", "description": "Remove a file from a filesystem",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"summary": "Remove a file from the filesystem", "summary": "Remove a file from a filesystem",
"operationId": "diskfs-3-delete-file", "operationId": "filesystem-3-delete-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -487,240 +541,6 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": { "/api/v3/log": {
"get": { "get": {
"security": [ "security": [
@@ -1982,140 +1802,6 @@ const docTemplate = `{
} }
} }
}, },
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": { "/metrics": {
"get": { "get": {
"description": "Prometheus metrics", "description": "Prometheus metrics",
@@ -2175,46 +1861,6 @@ const docTemplate = `{
} }
} }
} }
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@@ -2798,6 +2444,46 @@ const docTemplate = `{
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },
@@ -2869,6 +2555,20 @@ const docTemplate = `{
} }
} }
}, },
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": { "api.GraphQuery": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -4406,6 +4106,46 @@ const docTemplate = `{
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },

View File

@@ -294,20 +294,53 @@
} }
} }
}, },
"/api/v3/fs/disk": { "/api/v3/fs": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", "description": "Listall registered filesystems",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"summary": "List all files on the filesystem", "summary": "List all registered filesystems",
"operationId": "diskfs-3-list-files", "operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "glob pattern for file names", "description": "glob pattern for file names",
@@ -340,21 +373,28 @@
} }
} }
}, },
"/api/v3/fs/disk/{path}": { "/api/v3/fs/{name}/{path}": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Fetch a file from the filesystem. The contents of that file are returned.", "description": "Fetch a file from a filesystem",
"produces": [ "produces": [
"application/data", "application/data",
"application/json" "application/json"
], ],
"summary": "Fetch a file from the filesystem", "summary": "Fetch a file from a filesystem",
"operationId": "diskfs-3-get-file", "operationId": "filesystem-3-get-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -390,7 +430,7 @@
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Writes or overwrites a file on the filesystem", "description": "Writes or overwrites a file on a filesystem",
"consumes": [ "consumes": [
"application/data" "application/data"
], ],
@@ -398,9 +438,16 @@
"text/plain", "text/plain",
"application/json" "application/json"
], ],
"summary": "Add a file to the filesystem", "summary": "Add a file to a filesystem",
"operationId": "diskfs-3-put-file", "operationId": "filesystem-3-put-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -448,13 +495,20 @@
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Remove a file from the filesystem", "description": "Remove a file from a filesystem",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"summary": "Remove a file from the filesystem", "summary": "Remove a file from a filesystem",
"operationId": "diskfs-3-delete-file", "operationId": "filesystem-3-delete-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -479,240 +533,6 @@
} }
} }
}, },
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": { "/api/v3/log": {
"get": { "get": {
"security": [ "security": [
@@ -1974,140 +1794,6 @@
} }
} }
}, },
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": { "/metrics": {
"get": { "get": {
"description": "Prometheus metrics", "description": "Prometheus metrics",
@@ -2167,46 +1853,6 @@
} }
} }
} }
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@@ -2790,6 +2436,46 @@
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },
@@ -2861,6 +2547,20 @@
} }
} }
}, },
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": { "api.GraphQuery": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -4398,6 +4098,46 @@
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },

View File

@@ -379,6 +379,32 @@ definitions:
type: object type: object
mimetypes_file: mimetypes_file:
type: string type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object type: object
tls: tls:
properties: properties:
@@ -424,6 +450,15 @@ definitions:
size_bytes: size_bytes:
type: integer type: integer
type: object type: object
api.FilesystemInfo:
properties:
mount:
type: string
name:
type: string
type:
type: string
type: object
api.GraphQuery: api.GraphQuery:
properties: properties:
query: query:
@@ -1491,6 +1526,32 @@ definitions:
type: object type: object
mimetypes_file: mimetypes_file:
type: string type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object type: object
tls: tls:
properties: properties:
@@ -1705,34 +1766,6 @@ info:
title: datarhei Core API title: datarhei Core API
version: "3.0" version: "3.0"
paths: paths:
/{path}:
get:
description: Fetch a file from the filesystem. If the file is a directory, a
index.html is returned, if it exists.
operationId: diskfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the filesystem
/api: /api:
get: get:
description: API version and build infos in case auth is valid or not required. description: API version and build infos in case auth is valid or not required.
@@ -1911,12 +1944,33 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Reload the currently active configuration summary: Reload the currently active configuration
/api/v3/fs/disk: /api/v3/fs:
get: get:
description: List all files on the filesystem. The listing can be ordered by description: Listall registered filesystems
name, size, or date of last modification in ascending or descending order. operationId: filesystem-3-list
operationId: diskfs-3-list-files produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FilesystemInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all registered filesystems
/api/v3/fs/{name}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
operationId: filesystem-3-list-files
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: glob pattern for file names - description: glob pattern for file names
in: query in: query
name: glob name: glob
@@ -1940,12 +1994,17 @@ paths:
type: array type: array
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: List all files on the filesystem summary: List all files on a filesystem
/api/v3/fs/disk/{path}: /api/v3/fs/{name}/{path}:
delete: delete:
description: Remove a file from the filesystem description: Remove a file from a filesystem
operationId: diskfs-3-delete-file operationId: filesystem-3-delete-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -1964,12 +2023,16 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Remove a file from the filesystem summary: Remove a file from a filesystem
get: get:
description: Fetch a file from the filesystem. The contents of that file are description: Fetch a file from a filesystem
returned. operationId: filesystem-3-get-file
operationId: diskfs-3-get-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -1993,13 +2056,18 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Fetch a file from the filesystem summary: Fetch a file from a filesystem
put: put:
consumes: consumes:
- application/data - application/data
description: Writes or overwrites a file on the filesystem description: Writes or overwrites a file on a filesystem
operationId: diskfs-3-put-file operationId: filesystem-3-put-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -2031,160 +2099,7 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Add a file to the filesystem summary: Add a file to a filesystem
/api/v3/fs/mem:
get:
description: List all files on the memory filesystem. The listing can be ordered
by name, size, or date of last modification in ascending or descending order.
operationId: memfs-3-list-files
parameters:
- description: glob pattern for file names
in: query
name: glob
type: string
- description: none, name, size, lastmod
in: query
name: sort
type: string
- description: asc, desc
in: query
name: order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FileInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all files on the memory filesystem
/api/v3/fs/mem/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-3-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-3-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the memory filesystem
patch:
consumes:
- application/data
description: Create a link to a file in the memory filesystem. The file linked
to has to exist.
operationId: memfs-3-patch
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: Path to the file to link to
in: body
name: url
required: true
schema:
type: string
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Create a link to a file in the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-3-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the memory filesystem
/api/v3/log: /api/v3/log:
get: get:
description: Get the last log lines of the Restreamer application description: Get the last log lines of the Restreamer application
@@ -3019,94 +2934,6 @@ paths:
schema: schema:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
summary: Fetch minimal statistics about a process summary: Fetch minimal statistics about a process
/memfs/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Add a file to the memory filesystem
/metrics: /metrics:
get: get:
description: Prometheus metrics description: Prometheus metrics

View File

@@ -6,3 +6,10 @@ type FileInfo struct {
Size int64 `json:"size_bytes" jsonschema:"minimum=0"` Size int64 `json:"size_bytes" jsonschema:"minimum=0"`
LastMod int64 `json:"last_modified" jsonschema:"minimum=0"` LastMod int64 `json:"last_modified" jsonschema:"minimum=0"`
} }
// FilesystemInfo represents information about a filesystem
type FilesystemInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Mount string `json:"mount"`
}

23
http/fs/fs.go Normal file
View File

@@ -0,0 +1,23 @@
package fs
import (
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/io/fs"
)
type FS struct {
Name string
Mountpoint string
AllowWrite bool
Username string
Password string
DefaultFile string
DefaultContentType string
Gzip bool
Filesystem fs.Filesystem
Cache cache.Cacher
}

View File

@@ -1,207 +0,0 @@
package api
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
handler *handler.DiskFSHandler
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
handler: handler.NewDiskFS(fs, cache),
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. The contents of that file are returned.
// @ID diskfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the filesystem
// @Description Writes or overwrites a file on the filesystem
// @ID diskfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [put]
func (h *DiskFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.cache != nil {
h.cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, path)
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the filesystem
// @Description Remove a file from the filesystem
// @ID diskfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [delete]
func (h *DiskFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.cache != nil {
h.cache.Delete(path)
}
return c.String(http.StatusOK, "OK")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the filesystem
// @Description List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID diskfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk [get]
func (h *DiskFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

View File

@@ -0,0 +1,146 @@
package api
import (
"net/http"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
type FSConfig struct {
Type string
Mountpoint string
Handler *handler.FSHandler
}
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
filesystems map[string]FSConfig
}
// NewFS return a new FSHanlder type. You have to provide a filesystem to act on.
func NewFS(filesystems map[string]FSConfig) *FSHandler {
return &FSHandler{
filesystems: filesystems,
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @ID filesystem-3-get-file
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @ID filesystem-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.PutFile(c)
}
// DeleteFileAPI removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @ID filesystem-3-delete-file
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.DeleteFile(c)
}
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID filesystem-3-list-files
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.ListFiles(c)
}
// List lists all registered filesystems
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @ID filesystem-3-list
// @Produce json
// @Success 200 {array} api.FilesystemInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs [get]
func (h *FSHandler) List(c echo.Context) error {
fss := []api.FilesystemInfo{}
for name, config := range h.filesystems {
fss = append(fss, api.FilesystemInfo{
Name: name,
Type: config.Type,
Mount: config.Mountpoint,
})
}
return c.JSON(http.StatusOK, fss)
}

View File

@@ -1,172 +0,0 @@
package api
import (
"io"
"net/http"
"net/url"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
handler *handler.MemFSHandler
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
handler: handler.NewMemFS(fs),
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
return h.handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
return h.handler.PutFile(c)
}
// DeleteFileAPI removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
return h.handler.DeleteFile(c)
}
// PatchFile creates a symbolic link to a file in the filesystem
// @Summary Create a link to a file in the memory filesystem
// @Description Create a link to a file in the memory filesystem. The file linked to has to exist.
// @ID memfs-3-patch
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param url body string true "Path to the file to link to"
// @Success 201 {string} string
// @Failure 400 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [patch]
func (h *MemFSHandler) PatchFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
body, err := io.ReadAll(req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "Failed reading request body", "%s", err)
}
u, err := url.Parse(string(body))
if err != nil {
return api.Err(http.StatusBadRequest, "Body doesn't contain a valid path", "%s", err)
}
if err := h.filesystem.Symlink(u.Path, path); err != nil {
return api.Err(http.StatusBadRequest, "Failed to create symlink", "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
return c.String(http.StatusCreated, "")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the memory filesystem
// @Description List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID memfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem [get]
func (h *MemFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

View File

@@ -1,88 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.
// @ID diskfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
path = filepath.Join(path, "index.html")
file.Close()
file = h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}

164
http/handler/filesystem.go Normal file
View File

@@ -0,0 +1,164 @@
package handler
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
fs fs.FS
}
// NewFS return a new FSHandler type. You have to provide a filesystem to act on.
func NewFS(fs fs.FS) *FSHandler {
return &FSHandler{
fs: fs,
}
}
func (h *FSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if len(h.fs.DefaultFile) != 0 {
if stat.IsDir() {
path = filepath.Join(path, h.fs.DefaultFile)
file.Close()
file = h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
func (h *FSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.fs.Filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
func (h *FSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.fs.Filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}
func (h *FSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.fs.Filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

View File

@@ -1,130 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /memfs/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
stat, _ := file.Stat()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}

View File

@@ -29,19 +29,19 @@
package http package http
import ( import (
"fmt"
"net/http" "net/http"
"strings" "strings"
"github.com/datarhei/core/v16/config" "github.com/datarhei/core/v16/config"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/errorhandler" "github.com/datarhei/core/v16/http/errorhandler"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/graph/resolver" "github.com/datarhei/core/v16/http/graph/resolver"
"github.com/datarhei/core/v16/http/handler" "github.com/datarhei/core/v16/http/handler"
api "github.com/datarhei/core/v16/http/handler/api" api "github.com/datarhei/core/v16/http/handler/api"
"github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/http/validator" "github.com/datarhei/core/v16/http/validator"
"github.com/datarhei/core/v16/io/fs"
"github.com/datarhei/core/v16/log" "github.com/datarhei/core/v16/log"
"github.com/datarhei/core/v16/monitor" "github.com/datarhei/core/v16/monitor"
"github.com/datarhei/core/v16/net" "github.com/datarhei/core/v16/net"
@@ -79,9 +79,7 @@ type Config struct {
Metrics monitor.HistoryReader Metrics monitor.HistoryReader
Prometheus prometheus.Reader Prometheus prometheus.Reader
MimeTypesFile string MimeTypesFile string
DiskFS fs.Filesystem Filesystems []fs.FS
MemFS MemFSConfig
S3FS MemFSConfig
IPLimiter net.IPLimiter IPLimiter net.IPLimiter
Profiling bool Profiling bool
Cors CorsConfig Cors CorsConfig
@@ -89,19 +87,11 @@ type Config struct {
SRT srt.Server SRT srt.Server
JWT jwt.JWT JWT jwt.JWT
Config config.Store Config config.Store
Cache cache.Cacher
Sessions session.RegistryReader Sessions session.RegistryReader
Router router.Router Router router.Router
ReadOnly bool ReadOnly bool
} }
type MemFSConfig struct {
EnableAuth bool
Username string
Password string
Filesystem fs.Filesystem
}
type CorsConfig struct { type CorsConfig struct {
Origins []string Origins []string
} }
@@ -115,9 +105,6 @@ type server struct {
handler struct { handler struct {
about *api.AboutHandler about *api.AboutHandler
memfs *handler.MemFSHandler
s3fs *handler.MemFSHandler
diskfs *handler.DiskFSHandler
prometheus *handler.PrometheusHandler prometheus *handler.PrometheusHandler
profiling *handler.ProfilingHandler profiling *handler.ProfilingHandler
ping *handler.PingHandler ping *handler.PingHandler
@@ -129,9 +116,6 @@ type server struct {
log *api.LogHandler log *api.LogHandler
restream *api.RestreamHandler restream *api.RestreamHandler
playout *api.PlayoutHandler playout *api.PlayoutHandler
memfs *api.MemFSHandler
s3fs *api.MemFSHandler
diskfs *api.DiskFSHandler
rtmp *api.RTMPHandler rtmp *api.RTMPHandler
srt *api.SRTHandler srt *api.SRTHandler
config *api.ConfigHandler config *api.ConfigHandler
@@ -150,18 +134,12 @@ type server struct {
session echo.MiddlewareFunc session echo.MiddlewareFunc
} }
memfs struct {
enableAuth bool
username string
password string
}
diskfs fs.Filesystem
gzip struct { gzip struct {
mimetypes []string mimetypes []string
} }
filesystems map[string]*filesystem
router *echo.Echo router *echo.Echo
mimeTypesFile string mimeTypesFile string
profiling bool profiling bool
@@ -169,28 +147,56 @@ type server struct {
readOnly bool readOnly bool
} }
type filesystem struct {
fs.FS
handler *handler.FSHandler
}
func NewServer(config Config) (Server, error) { func NewServer(config Config) (Server, error) {
s := &server{ s := &server{
logger: config.Logger, logger: config.Logger,
mimeTypesFile: config.MimeTypesFile, mimeTypesFile: config.MimeTypesFile,
profiling: config.Profiling, profiling: config.Profiling,
diskfs: config.DiskFS,
readOnly: config.ReadOnly, readOnly: config.ReadOnly,
} }
s.v3handler.diskfs = api.NewDiskFS( s.filesystems = map[string]*filesystem{}
config.DiskFS,
config.Cache,
)
s.handler.diskfs = handler.NewDiskFS( corsPrefixes := map[string][]string{
config.DiskFS, "/api": {"*"},
config.Cache, }
)
s.memfs.enableAuth = config.MemFS.EnableAuth for _, fs := range config.Filesystems {
s.memfs.username = config.MemFS.Username if _, ok := s.filesystems[fs.Name]; ok {
s.memfs.password = config.MemFS.Password return nil, fmt.Errorf("the filesystem name '%s' is already in use", fs.Name)
}
if !strings.HasPrefix(fs.Mountpoint, "/") {
fs.Mountpoint = "/" + fs.Mountpoint
}
if !strings.HasSuffix(fs.Mountpoint, "/") {
fs.Mountpoint = strings.TrimSuffix(fs.Mountpoint, "/")
}
if _, ok := corsPrefixes[fs.Mountpoint]; ok {
return nil, fmt.Errorf("the mount point '%s' is already in use (%s)", fs.Mountpoint, fs.Name)
}
corsPrefixes[fs.Mountpoint] = config.Cors.Origins
filesystem := &filesystem{
FS: fs,
handler: handler.NewFS(fs),
}
s.filesystems[filesystem.Name] = filesystem
}
if _, ok := corsPrefixes["/"]; !ok {
return nil, fmt.Errorf("one filesystem must be mounted at /")
}
if config.Logger == nil { if config.Logger == nil {
s.logger = log.New("HTTP") s.logger = log.New("HTTP")
@@ -222,26 +228,6 @@ func NewServer(config Config) (Server, error) {
) )
} }
if config.MemFS.Filesystem != nil {
s.v3handler.memfs = api.NewMemFS(
config.MemFS.Filesystem,
)
s.handler.memfs = handler.NewMemFS(
config.MemFS.Filesystem,
)
}
if config.S3FS.Filesystem != nil {
s.v3handler.s3fs = api.NewMemFS(
config.S3FS.Filesystem,
)
s.handler.s3fs = handler.NewMemFS(
config.S3FS.Filesystem,
)
}
if config.Prometheus != nil { if config.Prometheus != nil {
s.handler.prometheus = handler.NewPrometheus( s.handler.prometheus = handler.NewPrometheus(
config.Prometheus.HTTPHandler(), config.Prometheus.HTTPHandler(),
@@ -300,12 +286,6 @@ func NewServer(config Config) (Server, error) {
Logger: s.logger, Logger: s.logger,
}) })
if config.Cache != nil {
s.middleware.cache = mwcache.NewWithConfig(mwcache.Config{
Cache: config.Cache,
})
}
s.v3handler.widget = api.NewWidget(api.WidgetConfig{ s.v3handler.widget = api.NewWidget(api.WidgetConfig{
Restream: config.Restream, Restream: config.Restream,
Registry: config.Sessions, Registry: config.Sessions,
@@ -316,12 +296,7 @@ func NewServer(config Config) (Server, error) {
}) })
if middleware, err := mwcors.NewWithConfig(mwcors.Config{ if middleware, err := mwcors.NewWithConfig(mwcors.Config{
Prefixes: map[string][]string{ Prefixes: corsPrefixes,
"/": config.Cors.Origins,
"/api": {"*"},
"/memfs": config.Cors.Origins,
"/s3": config.Cors.Origins,
},
}); err != nil { }); err != nil {
return nil, err return nil, err
} else { } else {
@@ -447,105 +422,48 @@ func (s *server) setRoutes() {
doc.Use(gzipMiddleware) doc.Use(gzipMiddleware)
doc.GET("", echoSwagger.WrapHandler) doc.GET("", echoSwagger.WrapHandler)
// Serve static data // Mount filesystems
fs := s.router.Group("/*") for _, filesystem := range s.filesystems {
fs := s.router.Group(filesystem.Mountpoint + "/*")
fs.Use(mwmime.NewWithConfig(mwmime.Config{ fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "text/html", DefaultContentType: filesystem.DefaultContentType,
})) }))
if filesystem.Gzip {
fs.Use(mwgzip.NewWithConfig(mwgzip.Config{ fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed, Level: mwgzip.BestSpeed,
MinLength: 1000, MinLength: 1000,
ContentTypes: s.gzip.mimetypes, ContentTypes: s.gzip.mimetypes,
})) }))
if s.middleware.cache != nil {
fs.Use(s.middleware.cache)
} }
fs.GET("", s.handler.diskfs.GetFile) if filesystem.Cache != nil {
fs.HEAD("", s.handler.diskfs.GetFile) mwcache := mwcache.NewWithConfig(mwcache.Config{
Cache: filesystem.Cache,
// Memory FS })
if s.handler.memfs != nil { fs.Use(mwcache)
memfs := s.router.Group("/memfs/*")
memfs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
memfs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
memfs.Use(s.middleware.session)
} }
memfs.HEAD("", s.handler.memfs.GetFile) fs.GET("", filesystem.handler.GetFile)
memfs.GET("", s.handler.memfs.GetFile) fs.HEAD("", filesystem.handler.GetFile)
var authmw echo.MiddlewareFunc if len(filesystem.Username) != 0 || len(filesystem.Password) != 0 {
authmw := middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if s.memfs.enableAuth { if username == filesystem.Username && password == filesystem.Password {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
return true, nil return true, nil
} }
return false, nil return false, nil
}) })
memfs.POST("", s.handler.memfs.PutFile, authmw) fs.POST("", filesystem.handler.PutFile, authmw)
memfs.PUT("", s.handler.memfs.PutFile, authmw) fs.PUT("", filesystem.handler.PutFile, authmw)
memfs.DELETE("", s.handler.memfs.DeleteFile, authmw) fs.DELETE("", filesystem.handler.DeleteFile, authmw)
} else { } else {
memfs.POST("", s.handler.memfs.PutFile) fs.POST("", filesystem.handler.PutFile)
memfs.PUT("", s.handler.memfs.PutFile) fs.PUT("", filesystem.handler.PutFile)
memfs.DELETE("", s.handler.memfs.DeleteFile) fs.DELETE("", filesystem.handler.DeleteFile)
}
}
// S3 FS
if s.handler.s3fs != nil {
s3fs := s.router.Group("/s3/*")
s3fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
s3fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
s3fs.Use(s.middleware.session)
}
s3fs.HEAD("", s.handler.s3fs.GetFile)
s3fs.GET("", s.handler.s3fs.GetFile)
var authmw echo.MiddlewareFunc
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
return true, nil
}
return false, nil
})
s3fs.POST("", s.handler.s3fs.PutFile, authmw)
s3fs.PUT("", s.handler.s3fs.PutFile, authmw)
s3fs.DELETE("", s.handler.s3fs.DeleteFile, authmw)
} else {
s3fs.POST("", s.handler.s3fs.PutFile)
s3fs.PUT("", s.handler.s3fs.PutFile)
s3fs.DELETE("", s.handler.s3fs.DeleteFile)
}
if s.middleware.cache != nil {
s3fs.Use(s.middleware.cache)
} }
} }
@@ -643,44 +561,33 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
} }
} }
// v3 Memory FS // v3 Filesystems
if s.v3handler.memfs != nil { fshandlers := map[string]api.FSConfig{}
v3.GET("/fs/mem", s.v3handler.memfs.ListFiles) for _, fs := range s.filesystems {
v3.GET("/fs/mem/*", s.v3handler.memfs.GetFile) fshandlers[fs.Name] = api.FSConfig{
Type: fs.Filesystem.Type(),
if !s.readOnly { Mountpoint: fs.Mountpoint,
v3.DELETE("/fs/mem/*", s.v3handler.memfs.DeleteFile) Handler: fs.handler,
v3.PUT("/fs/mem/*", s.v3handler.memfs.PutFile)
v3.PATCH("/fs/mem/*", s.v3handler.memfs.PatchFile)
} }
} }
// v3 S3 FS handler := api.NewFS(fshandlers)
if s.v3handler.s3fs != nil {
v3.GET("/fs/s3", s.v3handler.s3fs.ListFiles)
v3.GET("/fs/s3/*", s.v3handler.s3fs.GetFile)
if !s.readOnly { v3.GET("/fs", handler.List)
v3.DELETE("/fs/s3/*", s.v3handler.s3fs.DeleteFile)
v3.PUT("/fs/s3/*", s.v3handler.s3fs.PutFile)
v3.PATCH("/fs/s3/*", s.v3handler.s3fs.PatchFile)
}
}
// v3 Disk FS v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/disk", s.v3handler.diskfs.ListFiles) v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.GET("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data", DefaultContentType: "application/data",
})) }))
v3.HEAD("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{ v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data", DefaultContentType: "application/data",
})) }))
if !s.readOnly { if !s.readOnly {
v3.PUT("/fs/disk/*", s.v3handler.diskfs.PutFile) v3.PUT("/fs/:name/*", handler.PutFile)
v3.DELETE("/fs/disk/*", s.v3handler.diskfs.DeleteFile) v3.DELETE("/fs/:name/*", handler.DeleteFile)
} }
// v3 RTMP // v3 RTMP

View File

@@ -135,6 +135,8 @@ func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
fs.logger = log.New("") fs.logger = log.New("")
} }
fs.logger = fs.logger.WithField("type", "disk")
if err := fs.Rebase(config.Dir); err != nil { if err := fs.Rebase(config.Dir); err != nil {
return nil, err return nil, err
} }
@@ -172,6 +174,10 @@ func (fs *diskFilesystem) Rebase(base string) error {
return nil return nil
} }
func (fs *diskFilesystem) Type() string {
return "diskfs"
}
func (fs *diskFilesystem) Size() (int64, int64) { func (fs *diskFilesystem) Size() (int64, int64) {
// This is to cache the size for some time in order not to // This is to cache the size for some time in order not to
// stress the underlying filesystem too much. // stress the underlying filesystem too much.

View File

@@ -24,6 +24,7 @@ type dummyFilesystem struct{}
func (d *dummyFilesystem) Base() string { return "/" } func (d *dummyFilesystem) Base() string { return "/" }
func (d *dummyFilesystem) Rebase(string) error { return nil } func (d *dummyFilesystem) Rebase(string) error { return nil }
func (d *dummyFilesystem) Type() string { return "dummy" }
func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 } func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 }
func (d *dummyFilesystem) Resize(int64) {} func (d *dummyFilesystem) Resize(int64) {}
func (d *dummyFilesystem) Files() int64 { return 0 } func (d *dummyFilesystem) Files() int64 { return 0 }

View File

@@ -44,6 +44,9 @@ type Filesystem interface {
// Rebase sets a new base path for this filesystem // Rebase sets a new base path for this filesystem
Rebase(string) error Rebase(string) error
// Type returns the type of this filesystem
Type() string
// Size returns the consumed size and capacity of the filesystem in bytes. The // Size returns the consumed size and capacity of the filesystem in bytes. The
// capacity is negative if the filesystem can consume as much space as it can. // capacity is negative if the filesystem can consume as much space as it can.
Size() (int64, int64) Size() (int64, int64)

View File

@@ -146,6 +146,8 @@ func NewMemFilesystem(config MemConfig) Filesystem {
fs.logger = log.New("") fs.logger = log.New("")
} }
fs.logger = fs.logger.WithField("type", "mem")
fs.files = make(map[string]*memFile) fs.files = make(map[string]*memFile)
fs.dataPool = sync.Pool{ fs.dataPool = sync.Pool{
@@ -172,6 +174,10 @@ func (fs *memFilesystem) Rebase(base string) error {
return nil return nil
} }
func (fs *memFilesystem) Type() string {
return "memfs"
}
func (fs *memFilesystem) Size() (int64, int64) { func (fs *memFilesystem) Size() (int64, int64) {
fs.filesLock.RLock() fs.filesLock.RLock()
defer fs.filesLock.RUnlock() defer fs.filesLock.RUnlock()

View File

@@ -66,6 +66,7 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
} }
fs.logger = fs.logger.WithFields(log.Fields{ fs.logger = fs.logger.WithFields(log.Fields{
"type": "s3",
"bucket": fs.bucket, "bucket": fs.bucket,
"region": fs.region, "region": fs.region,
"endpoint": fs.endpoint, "endpoint": fs.endpoint,
@@ -107,6 +108,10 @@ func (fs *s3fs) Rebase(base string) error {
return nil return nil
} }
func (fs *s3fs) Type() string {
return "s3fs"
}
func (fs *s3fs) Size() (int64, int64) { func (fs *s3fs) Size() (int64, int64) {
size := int64(0) size := int64(0)