apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-10-07 00:43:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor internal filesystem handling

Browse Source
This commit is contained in:
Ingo Oppermann
2022-08-22 19:10:59 +03:00
parent 9a49e371e3
commit cb0bc494f9
18 changed files with 955 additions and 1981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
app/api/api.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/datarhei/core/v16/ffmpeg"
"github.com/datarhei/core/v16/http"
"github.com/datarhei/core/v16/http/cache"
httpfs "github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/io/fs"
@@ -60,24 +61,23 @@ type API interface {
}
type api struct {
restream restream.Restreamer
ffmpeg ffmpeg.FFmpeg
diskfs fs.Filesystem
memfs fs.Filesystem
s3fs fs.Filesystem
rtmpserver rtmp.Server
srtserver srt.Server
metrics monitor.HistoryMonitor
prom prometheus.Metrics
service service.Service
sessions session.Registry
sessionsLimiter net.IPLimiter
cache cache.Cacher
mainserver *gohttp.Server
sidecarserver *gohttp.Server
httpjwt jwt.JWT
update update.Checker
replacer replace.Replacer
restream restream.Restreamer
ffmpeg ffmpeg.FFmpeg
diskfs fs.Filesystem
memfs fs.Filesystem
s3fs fs.Filesystem
rtmpserver rtmp.Server
srtserver srt.Server
metrics monitor.HistoryMonitor
prom prometheus.Metrics
service service.Service
sessions session.Registry
cache cache.Cacher
mainserver *gohttp.Server
sidecarserver *gohttp.Server
httpjwt jwt.JWT
update update.Checker
replacer replace.Replacer
errorChan chan error
@@ -372,7 +372,7 @@ func (a *api) start() error {
diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
Dir: cfg.Storage.Disk.Dir,
Size: cfg.Storage.Disk.Size * 1024 * 1024,
Logger: a.log.logger.core.WithComponent("DiskFS"),
Logger: a.log.logger.core.WithComponent("FS"),
})
if err != nil {
return err
@@ -401,7 +401,7 @@ func (a *api) start() error {
Base: baseMemFS.String(),
Size: cfg.Storage.Memory.Size * 1024 * 1024,
Purge: cfg.Storage.Memory.Purge,
Logger: a.log.logger.core.WithComponent("MemFS"),
Logger: a.log.logger.core.WithComponent("FS"),
})
a.memfs = memfs
@@ -435,7 +435,7 @@ func (a *api) start() error {
Region: cfg.Storage.S3.Region,
Bucket: cfg.Storage.S3.Bucket,
UseSSL: cfg.Storage.S3.UseSSL,
Logger: a.log.logger.core.WithComponent("S3"),
Logger: a.log.logger.core.WithComponent("FS"),
})
if err != nil {
return err
@@ -665,7 +665,7 @@ func (a *api) start() error {
}
if cfg.Storage.Disk.Cache.Enable {
diskCache, err := cache.NewLRUCache(cache.LRUConfig{
cache, err := cache.NewLRUCache(cache.LRUConfig{
TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second,
MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024,
MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024,
@@ -675,10 +675,10 @@ func (a *api) start() error {
})
if err != nil {
return fmt.Errorf("unable to create disk cache: %w", err)
return fmt.Errorf("unable to create cache: %w", err)
}
a.cache = diskCache
a.cache = cache
}
var autocertManager *autocert.Manager
@@ -829,25 +829,50 @@ func (a *api) start() error {
a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address)
mainserverhandler, err := http.NewServer(http.Config{
serverConfig := http.Config{
Logger: a.log.logger.main,
LogBuffer: a.log.buffer,
Restream: a.restream,
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
Filesystem: a.memfs,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
Filesystem: a.s3fs,
Filesystems: []httpfs.FS{
{
Name: "diskfs",
Mountpoint: "/",
AllowWrite: false,
Username: "",
Password: "",
DefaultFile: "index.html",
DefaultContentType: "text/html",
Gzip: true,
Filesystem: diskfs,
Cache: a.cache,
},
{
Name: "memfs",
Mountpoint: "/memfs",
AllowWrite: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.memfs,
Cache: a.cache,
},
{
Name: "s3fs",
Mountpoint: "/s3",
AllowWrite: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.s3fs,
Cache: a.cache,
},
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
@@ -858,11 +883,12 @@ func (a *api) start() error {
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
}
mainserverhandler, err := http.NewServer(serverConfig)
if err != nil {
return fmt.Errorf("unable to create server: %w", err)
@@ -897,40 +923,10 @@ func (a *api) start() error {
a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address)
sidecarserverhandler, err := http.NewServer(http.Config{
Logger: a.log.logger.sidecar,
LogBuffer: a.log.buffer,
Restream: a.restream,
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
Filesystem: a.memfs,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
Filesystem: a.s3fs,
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
Cors: http.CorsConfig{
Origins: cfg.Storage.CORS.Origins,
},
RTMP: a.rtmpserver,
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
serverConfig.Logger = a.log.logger.sidecar
serverConfig.IPLimiter = iplimiter
sidecarserverhandler, err := http.NewServer(serverConfig)
if err != nil {
return fmt.Errorf("unable to create sidecar HTTP server: %w", err)

584
docs/docs.go
View File

@@ -302,20 +302,53 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/disk": {
"/api/v3/fs": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"description": "Listall registered filesystems",
"produces": [
"application/json"
],
"summary": "List all files on the filesystem",
"operationId": "diskfs-3-list-files",
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
@@ -348,21 +381,28 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/disk/{path}": {
"/api/v3/fs/{name}/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the filesystem. The contents of that file are returned.",
"description": "Fetch a file from a filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-3-get-file",
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -398,7 +438,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the filesystem",
"description": "Writes or overwrites a file on a filesystem",
"consumes": [
"application/data"
],
@@ -406,9 +446,16 @@ const docTemplate = `{
"text/plain",
"application/json"
],
"summary": "Add a file to the filesystem",
"operationId": "diskfs-3-put-file",
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -456,13 +503,20 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the filesystem",
"description": "Remove a file from a filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the filesystem",
"operationId": "diskfs-3-delete-file",
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -487,240 +541,6 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": {
"get": {
"security": [
@@ -1982,140 +1802,6 @@ const docTemplate = `{
}
}
},
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": {
"get": {
"description": "Prometheus metrics",
@@ -2175,46 +1861,6 @@ const docTemplate = `{
}
}
}
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
}
},
"definitions": {
@@ -2798,6 +2444,46 @@ const docTemplate = `{
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},
@@ -2869,6 +2555,20 @@ const docTemplate = `{
}
}
},
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {
@@ -4406,6 +4106,46 @@ const docTemplate = `{
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},

584
docs/swagger.json
View File

@@ -294,20 +294,53 @@
}
}
},
"/api/v3/fs/disk": {
"/api/v3/fs": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"description": "Listall registered filesystems",
"produces": [
"application/json"
],
"summary": "List all files on the filesystem",
"operationId": "diskfs-3-list-files",
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
@@ -340,21 +373,28 @@
}
}
},
"/api/v3/fs/disk/{path}": {
"/api/v3/fs/{name}/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the filesystem. The contents of that file are returned.",
"description": "Fetch a file from a filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-3-get-file",
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -390,7 +430,7 @@
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the filesystem",
"description": "Writes or overwrites a file on a filesystem",
"consumes": [
"application/data"
],
@@ -398,9 +438,16 @@
"text/plain",
"application/json"
],
"summary": "Add a file to the filesystem",
"operationId": "diskfs-3-put-file",
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -448,13 +495,20 @@
"ApiKeyAuth": []
}
],
"description": "Remove a file from the filesystem",
"description": "Remove a file from a filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the filesystem",
"operationId": "diskfs-3-delete-file",
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -479,240 +533,6 @@
}
}
},
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": {
"get": {
"security": [
@@ -1974,140 +1794,6 @@
}
}
},
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": {
"get": {
"description": "Prometheus metrics",
@@ -2167,46 +1853,6 @@
}
}
}
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
}
},
"definitions": {
@@ -2790,6 +2436,46 @@
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},
@@ -2861,6 +2547,20 @@
}
}
},
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {
@@ -4398,6 +4098,46 @@
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},

397
docs/swagger.yaml
View File

@@ -379,6 +379,32 @@ definitions:
type: object
mimetypes_file:
type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object
tls:
properties:
@@ -424,6 +450,15 @@ definitions:
size_bytes:
type: integer
type: object
api.FilesystemInfo:
properties:
mount:
type: string
name:
type: string
type:
type: string
type: object
api.GraphQuery:
properties:
query:
@@ -1491,6 +1526,32 @@ definitions:
type: object
mimetypes_file:
type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object
tls:
properties:
@@ -1705,34 +1766,6 @@ info:
title: datarhei Core API
version: "3.0"
paths:
/{path}:
get:
description: Fetch a file from the filesystem. If the file is a directory, a
index.html is returned, if it exists.
operationId: diskfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the filesystem
/api:
get:
description: API version and build infos in case auth is valid or not required.
@@ -1911,12 +1944,33 @@ paths:
security:
- ApiKeyAuth: []
summary: Reload the currently active configuration
/api/v3/fs/disk:
/api/v3/fs:
get:
description: List all files on the filesystem. The listing can be ordered by
name, size, or date of last modification in ascending or descending order.
operationId: diskfs-3-list-files
description: Listall registered filesystems
operationId: filesystem-3-list
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FilesystemInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all registered filesystems
/api/v3/fs/{name}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
operationId: filesystem-3-list-files
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: glob pattern for file names
in: query
name: glob
@@ -1940,12 +1994,17 @@ paths:
type: array
security:
- ApiKeyAuth: []
summary: List all files on the filesystem
/api/v3/fs/disk/{path}:
summary: List all files on a filesystem
/api/v3/fs/{name}/{path}:
delete:
description: Remove a file from the filesystem
operationId: diskfs-3-delete-file
description: Remove a file from a filesystem
operationId: filesystem-3-delete-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -1964,12 +2023,16 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the filesystem
summary: Remove a file from a filesystem
get:
description: Fetch a file from the filesystem. The contents of that file are
returned.
operationId: diskfs-3-get-file
description: Fetch a file from a filesystem
operationId: filesystem-3-get-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -1993,13 +2056,18 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the filesystem
summary: Fetch a file from a filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the filesystem
operationId: diskfs-3-put-file
description: Writes or overwrites a file on a filesystem
operationId: filesystem-3-put-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -2031,160 +2099,7 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the filesystem
/api/v3/fs/mem:
get:
description: List all files on the memory filesystem. The listing can be ordered
by name, size, or date of last modification in ascending or descending order.
operationId: memfs-3-list-files
parameters:
- description: glob pattern for file names
in: query
name: glob
type: string
- description: none, name, size, lastmod
in: query
name: sort
type: string
- description: asc, desc
in: query
name: order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FileInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all files on the memory filesystem
/api/v3/fs/mem/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-3-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-3-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the memory filesystem
patch:
consumes:
- application/data
description: Create a link to a file in the memory filesystem. The file linked
to has to exist.
operationId: memfs-3-patch
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: Path to the file to link to
in: body
name: url
required: true
schema:
type: string
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Create a link to a file in the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-3-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the memory filesystem
summary: Add a file to a filesystem
/api/v3/log:
get:
description: Get the last log lines of the Restreamer application
@@ -3019,94 +2934,6 @@ paths:
schema:
$ref: '#/definitions/api.Error'
summary: Fetch minimal statistics about a process
/memfs/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Add a file to the memory filesystem
/metrics:
get:
description: Prometheus metrics

7
http/api/fs.go
View File

@@ -6,3 +6,10 @@ type FileInfo struct {
Size int64 `json:"size_bytes" jsonschema:"minimum=0"`
LastMod int64 `json:"last_modified" jsonschema:"minimum=0"`
}
// FilesystemInfo represents information about a filesystem
type FilesystemInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Mount string `json:"mount"`
}

23
http/fs/fs.go Normal file
View File

@@ -0,0 +1,23 @@
package fs
import (
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/io/fs"
)
type FS struct {
Name string
Mountpoint string
AllowWrite bool
Username string
Password string
DefaultFile string
DefaultContentType string
Gzip bool
Filesystem fs.Filesystem
Cache cache.Cacher
}

207
http/handler/api/diskfs.go
View File

@@ -1,207 +0,0 @@
package api
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
handler *handler.DiskFSHandler
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
handler: handler.NewDiskFS(fs, cache),
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. The contents of that file are returned.
// @ID diskfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the filesystem
// @Description Writes or overwrites a file on the filesystem
// @ID diskfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [put]
func (h *DiskFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.cache != nil {
h.cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, path)
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the filesystem
// @Description Remove a file from the filesystem
// @ID diskfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [delete]
func (h *DiskFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.cache != nil {
h.cache.Delete(path)
}
return c.String(http.StatusOK, "OK")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the filesystem
// @Description List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID diskfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk [get]
func (h *DiskFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

146
http/handler/api/filesystems.go Normal file
View File

@@ -0,0 +1,146 @@
package api
import (
"net/http"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
type FSConfig struct {
Type string
Mountpoint string
Handler *handler.FSHandler
}
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
filesystems map[string]FSConfig
}
// NewFS return a new FSHanlder type. You have to provide a filesystem to act on.
func NewFS(filesystems map[string]FSConfig) *FSHandler {
return &FSHandler{
filesystems: filesystems,
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @ID filesystem-3-get-file
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @ID filesystem-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.PutFile(c)
}
// DeleteFileAPI removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @ID filesystem-3-delete-file
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.DeleteFile(c)
}
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID filesystem-3-list-files
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.ListFiles(c)
}
// List lists all registered filesystems
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @ID filesystem-3-list
// @Produce json
// @Success 200 {array} api.FilesystemInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs [get]
func (h *FSHandler) List(c echo.Context) error {
fss := []api.FilesystemInfo{}
for name, config := range h.filesystems {
fss = append(fss, api.FilesystemInfo{
Name: name,
Type: config.Type,
Mount: config.Mountpoint,
})
}
return c.JSON(http.StatusOK, fss)
}

172
http/handler/api/memfs.go
View File

@@ -1,172 +0,0 @@
package api
import (
"io"
"net/http"
"net/url"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
handler *handler.MemFSHandler
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
handler: handler.NewMemFS(fs),
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
return h.handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
return h.handler.PutFile(c)
}
// DeleteFileAPI removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
return h.handler.DeleteFile(c)
}
// PatchFile creates a symbolic link to a file in the filesystem
// @Summary Create a link to a file in the memory filesystem
// @Description Create a link to a file in the memory filesystem. The file linked to has to exist.
// @ID memfs-3-patch
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param url body string true "Path to the file to link to"
// @Success 201 {string} string
// @Failure 400 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [patch]
func (h *MemFSHandler) PatchFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
body, err := io.ReadAll(req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "Failed reading request body", "%s", err)
}
u, err := url.Parse(string(body))
if err != nil {
return api.Err(http.StatusBadRequest, "Body doesn't contain a valid path", "%s", err)
}
if err := h.filesystem.Symlink(u.Path, path); err != nil {
return api.Err(http.StatusBadRequest, "Failed to create symlink", "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
return c.String(http.StatusCreated, "")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the memory filesystem
// @Description List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID memfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem [get]
func (h *MemFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

88
http/handler/diskfs.go
View File

@@ -1,88 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.
// @ID diskfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
path = filepath.Join(path, "index.html")
file.Close()
file = h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}

164
http/handler/filesystem.go Normal file
View File

@@ -0,0 +1,164 @@
package handler
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
fs fs.FS
}
// NewFS return a new FSHandler type. You have to provide a filesystem to act on.
func NewFS(fs fs.FS) *FSHandler {
return &FSHandler{
fs: fs,
}
}
func (h *FSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if len(h.fs.DefaultFile) != 0 {
if stat.IsDir() {
path = filepath.Join(path, h.fs.DefaultFile)
file.Close()
file = h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
func (h *FSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.fs.Filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
func (h *FSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.fs.Filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}
func (h *FSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.fs.Filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

130
http/handler/memfs.go
View File

@@ -1,130 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /memfs/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
stat, _ := file.Stat()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}

271
http/server.go
View File

@@ -29,19 +29,19 @@
package http
import (
"fmt"
"net/http"
"strings"
"github.com/datarhei/core/v16/config"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/errorhandler"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/graph/resolver"
"github.com/datarhei/core/v16/http/handler"
api "github.com/datarhei/core/v16/http/handler/api"
"github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/http/validator"
"github.com/datarhei/core/v16/io/fs"
"github.com/datarhei/core/v16/log"
"github.com/datarhei/core/v16/monitor"
"github.com/datarhei/core/v16/net"
@@ -79,9 +79,7 @@ type Config struct {
Metrics monitor.HistoryReader
Prometheus prometheus.Reader
MimeTypesFile string
DiskFS fs.Filesystem
MemFS MemFSConfig
S3FS MemFSConfig
Filesystems []fs.FS
IPLimiter net.IPLimiter
Profiling bool
Cors CorsConfig
@@ -89,19 +87,11 @@ type Config struct {
SRT srt.Server
JWT jwt.JWT
Config config.Store
Cache cache.Cacher
Sessions session.RegistryReader
Router router.Router
ReadOnly bool
}
type MemFSConfig struct {
EnableAuth bool
Username string
Password string
Filesystem fs.Filesystem
}
type CorsConfig struct {
Origins []string
}
@@ -115,9 +105,6 @@ type server struct {
handler struct {
about *api.AboutHandler
memfs *handler.MemFSHandler
s3fs *handler.MemFSHandler
diskfs *handler.DiskFSHandler
prometheus *handler.PrometheusHandler
profiling *handler.ProfilingHandler
ping *handler.PingHandler
@@ -129,9 +116,6 @@ type server struct {
log *api.LogHandler
restream *api.RestreamHandler
playout *api.PlayoutHandler
memfs *api.MemFSHandler
s3fs *api.MemFSHandler
diskfs *api.DiskFSHandler
rtmp *api.RTMPHandler
srt *api.SRTHandler
config *api.ConfigHandler
@@ -150,18 +134,12 @@ type server struct {
session echo.MiddlewareFunc
}
memfs struct {
enableAuth bool
username string
password string
}
diskfs fs.Filesystem
gzip struct {
mimetypes []string
}
filesystems map[string]*filesystem
router *echo.Echo
mimeTypesFile string
profiling bool
@@ -169,28 +147,56 @@ type server struct {
readOnly bool
}
type filesystem struct {
fs.FS
handler *handler.FSHandler
}
func NewServer(config Config) (Server, error) {
s := &server{
logger: config.Logger,
mimeTypesFile: config.MimeTypesFile,
profiling: config.Profiling,
diskfs: config.DiskFS,
readOnly: config.ReadOnly,
}
s.v3handler.diskfs = api.NewDiskFS(
config.DiskFS,
config.Cache,
)
s.filesystems = map[string]*filesystem{}
s.handler.diskfs = handler.NewDiskFS(
config.DiskFS,
config.Cache,
)
corsPrefixes := map[string][]string{
"/api": {"*"},
}
s.memfs.enableAuth = config.MemFS.EnableAuth
s.memfs.username = config.MemFS.Username
s.memfs.password = config.MemFS.Password
for _, fs := range config.Filesystems {
if _, ok := s.filesystems[fs.Name]; ok {
return nil, fmt.Errorf("the filesystem name '%s' is already in use", fs.Name)
}
if !strings.HasPrefix(fs.Mountpoint, "/") {
fs.Mountpoint = "/" + fs.Mountpoint
}
if !strings.HasSuffix(fs.Mountpoint, "/") {
fs.Mountpoint = strings.TrimSuffix(fs.Mountpoint, "/")
}
if _, ok := corsPrefixes[fs.Mountpoint]; ok {
return nil, fmt.Errorf("the mount point '%s' is already in use (%s)", fs.Mountpoint, fs.Name)
}
corsPrefixes[fs.Mountpoint] = config.Cors.Origins
filesystem := &filesystem{
FS: fs,
handler: handler.NewFS(fs),
}
s.filesystems[filesystem.Name] = filesystem
}
if _, ok := corsPrefixes["/"]; !ok {
return nil, fmt.Errorf("one filesystem must be mounted at /")
}
if config.Logger == nil {
s.logger = log.New("HTTP")
@@ -222,26 +228,6 @@ func NewServer(config Config) (Server, error) {
)
}
if config.MemFS.Filesystem != nil {
s.v3handler.memfs = api.NewMemFS(
config.MemFS.Filesystem,
)
s.handler.memfs = handler.NewMemFS(
config.MemFS.Filesystem,
)
}
if config.S3FS.Filesystem != nil {
s.v3handler.s3fs = api.NewMemFS(
config.S3FS.Filesystem,
)
s.handler.s3fs = handler.NewMemFS(
config.S3FS.Filesystem,
)
}
if config.Prometheus != nil {
s.handler.prometheus = handler.NewPrometheus(
config.Prometheus.HTTPHandler(),
@@ -300,12 +286,6 @@ func NewServer(config Config) (Server, error) {
Logger: s.logger,
})
if config.Cache != nil {
s.middleware.cache = mwcache.NewWithConfig(mwcache.Config{
Cache: config.Cache,
})
}
s.v3handler.widget = api.NewWidget(api.WidgetConfig{
Restream: config.Restream,
Registry: config.Sessions,
@@ -316,12 +296,7 @@ func NewServer(config Config) (Server, error) {
})
if middleware, err := mwcors.NewWithConfig(mwcors.Config{
Prefixes: map[string][]string{
"/": config.Cors.Origins,
"/api": {"*"},
"/memfs": config.Cors.Origins,
"/s3": config.Cors.Origins,
},
Prefixes: corsPrefixes,
}); err != nil {
return nil, err
} else {
@@ -447,105 +422,48 @@ func (s *server) setRoutes() {
doc.Use(gzipMiddleware)
doc.GET("", echoSwagger.WrapHandler)
// Serve static data
fs := s.router.Group("/*")
fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "text/html",
}))
fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.cache != nil {
fs.Use(s.middleware.cache)
}
fs.GET("", s.handler.diskfs.GetFile)
fs.HEAD("", s.handler.diskfs.GetFile)
// Memory FS
if s.handler.memfs != nil {
memfs := s.router.Group("/memfs/*")
memfs.Use(mwmime.NewWithConfig(mwmime.Config{
// Mount filesystems
for _, filesystem := range s.filesystems {
fs := s.router.Group(filesystem.Mountpoint + "/*")
fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
DefaultContentType: filesystem.DefaultContentType,
}))
memfs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
memfs.Use(s.middleware.session)
if filesystem.Gzip {
fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
}
memfs.HEAD("", s.handler.memfs.GetFile)
memfs.GET("", s.handler.memfs.GetFile)
if filesystem.Cache != nil {
mwcache := mwcache.NewWithConfig(mwcache.Config{
Cache: filesystem.Cache,
})
fs.Use(mwcache)
}
var authmw echo.MiddlewareFunc
fs.GET("", filesystem.handler.GetFile)
fs.HEAD("", filesystem.handler.GetFile)
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
if len(filesystem.Username) != 0 || len(filesystem.Password) != 0 {
authmw := middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == filesystem.Username && password == filesystem.Password {
return true, nil
}
return false, nil
})
memfs.POST("", s.handler.memfs.PutFile, authmw)
memfs.PUT("", s.handler.memfs.PutFile, authmw)
memfs.DELETE("", s.handler.memfs.DeleteFile, authmw)
fs.POST("", filesystem.handler.PutFile, authmw)
fs.PUT("", filesystem.handler.PutFile, authmw)
fs.DELETE("", filesystem.handler.DeleteFile, authmw)
} else {
memfs.POST("", s.handler.memfs.PutFile)
memfs.PUT("", s.handler.memfs.PutFile)
memfs.DELETE("", s.handler.memfs.DeleteFile)
}
}
// S3 FS
if s.handler.s3fs != nil {
s3fs := s.router.Group("/s3/*")
s3fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
s3fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
s3fs.Use(s.middleware.session)
}
s3fs.HEAD("", s.handler.s3fs.GetFile)
s3fs.GET("", s.handler.s3fs.GetFile)
var authmw echo.MiddlewareFunc
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
return true, nil
}
return false, nil
})
s3fs.POST("", s.handler.s3fs.PutFile, authmw)
s3fs.PUT("", s.handler.s3fs.PutFile, authmw)
s3fs.DELETE("", s.handler.s3fs.DeleteFile, authmw)
} else {
s3fs.POST("", s.handler.s3fs.PutFile)
s3fs.PUT("", s.handler.s3fs.PutFile)
s3fs.DELETE("", s.handler.s3fs.DeleteFile)
}
if s.middleware.cache != nil {
s3fs.Use(s.middleware.cache)
fs.POST("", filesystem.handler.PutFile)
fs.PUT("", filesystem.handler.PutFile)
fs.DELETE("", filesystem.handler.DeleteFile)
}
}
@@ -643,44 +561,33 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
}
}
// v3 Memory FS
if s.v3handler.memfs != nil {
v3.GET("/fs/mem", s.v3handler.memfs.ListFiles)
v3.GET("/fs/mem/*", s.v3handler.memfs.GetFile)
if !s.readOnly {
v3.DELETE("/fs/mem/*", s.v3handler.memfs.DeleteFile)
v3.PUT("/fs/mem/*", s.v3handler.memfs.PutFile)
v3.PATCH("/fs/mem/*", s.v3handler.memfs.PatchFile)
// v3 Filesystems
fshandlers := map[string]api.FSConfig{}
for _, fs := range s.filesystems {
fshandlers[fs.Name] = api.FSConfig{
Type: fs.Filesystem.Type(),
Mountpoint: fs.Mountpoint,
Handler: fs.handler,
}
}
// v3 S3 FS
if s.v3handler.s3fs != nil {
v3.GET("/fs/s3", s.v3handler.s3fs.ListFiles)
v3.GET("/fs/s3/*", s.v3handler.s3fs.GetFile)
handler := api.NewFS(fshandlers)
if !s.readOnly {
v3.DELETE("/fs/s3/*", s.v3handler.s3fs.DeleteFile)
v3.PUT("/fs/s3/*", s.v3handler.s3fs.PutFile)
v3.PATCH("/fs/s3/*", s.v3handler.s3fs.PatchFile)
}
}
v3.GET("/fs", handler.List)
// v3 Disk FS
v3.GET("/fs/disk", s.v3handler.diskfs.ListFiles)
v3.GET("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
v3.HEAD("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
if !s.readOnly {
v3.PUT("/fs/disk/*", s.v3handler.diskfs.PutFile)
v3.DELETE("/fs/disk/*", s.v3handler.diskfs.DeleteFile)
v3.PUT("/fs/:name/*", handler.PutFile)
v3.DELETE("/fs/:name/*", handler.DeleteFile)
}
// v3 RTMP

6
io/fs/disk.go
View File

@@ -135,6 +135,8 @@ func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
fs.logger = log.New("")
}
fs.logger = fs.logger.WithField("type", "disk")
if err := fs.Rebase(config.Dir); err != nil {
return nil, err
}
@@ -172,6 +174,10 @@ func (fs *diskFilesystem) Rebase(base string) error {
return nil
}
func (fs *diskFilesystem) Type() string {
return "diskfs"
}
func (fs *diskFilesystem) Size() (int64, int64) {
// This is to cache the size for some time in order not to
// stress the underlying filesystem too much.

1
io/fs/dummy.go
View File

@@ -24,6 +24,7 @@ type dummyFilesystem struct{}
func (d *dummyFilesystem) Base() string { return "/" }
func (d *dummyFilesystem) Rebase(string) error { return nil }
func (d *dummyFilesystem) Type() string { return "dummy" }
func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 }
func (d *dummyFilesystem) Resize(int64) {}
func (d *dummyFilesystem) Files() int64 { return 0 }

3
io/fs/fs.go
View File

@@ -44,6 +44,9 @@ type Filesystem interface {
// Rebase sets a new base path for this filesystem
Rebase(string) error
// Type returns the type of this filesystem
Type() string
// Size returns the consumed size and capacity of the filesystem in bytes. The
// capacity is negative if the filesystem can consume as much space as it can.
Size() (int64, int64)

6
io/fs/mem.go
View File

@@ -146,6 +146,8 @@ func NewMemFilesystem(config MemConfig) Filesystem {
fs.logger = log.New("")
}
fs.logger = fs.logger.WithField("type", "mem")
fs.files = make(map[string]*memFile)
fs.dataPool = sync.Pool{
@@ -172,6 +174,10 @@ func (fs *memFilesystem) Rebase(base string) error {
return nil
}
func (fs *memFilesystem) Type() string {
return "memfs"
}
func (fs *memFilesystem) Size() (int64, int64) {
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()

5
io/fs/s3.go
View File

@@ -66,6 +66,7 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
}
fs.logger = fs.logger.WithFields(log.Fields{
"type": "s3",
"bucket": fs.bucket,
"region": fs.region,
"endpoint": fs.endpoint,
@@ -107,6 +108,10 @@ func (fs *s3fs) Rebase(base string) error {
return nil
}
func (fs *s3fs) Type() string {
return "s3fs"
}
func (fs *s3fs) Size() (int64, int64) {
size := int64(0)