apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-10-07 00:43:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor internal filesystem handling

Browse Source
This commit is contained in:
Ingo Oppermann
2022-08-22 19:10:59 +03:00
parent 9a49e371e3
commit cb0bc494f9
18 changed files with 955 additions and 1981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
app/api/api.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/datarhei/core/v16/ffmpeg" "github.com/datarhei/core/v16/ffmpeg"
"github.com/datarhei/core/v16/http" "github.com/datarhei/core/v16/http"
"github.com/datarhei/core/v16/http/cache" "github.com/datarhei/core/v16/http/cache"
httpfs "github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/io/fs" "github.com/datarhei/core/v16/io/fs"
@@ -60,24 +61,23 @@ type API interface {
} }
type api struct { type api struct {
restream restream.Restreamer restream restream.Restreamer
ffmpeg ffmpeg.FFmpeg ffmpeg ffmpeg.FFmpeg
diskfs fs.Filesystem diskfs fs.Filesystem
memfs fs.Filesystem memfs fs.Filesystem
s3fs fs.Filesystem s3fs fs.Filesystem
rtmpserver rtmp.Server rtmpserver rtmp.Server
srtserver srt.Server srtserver srt.Server
metrics monitor.HistoryMonitor metrics monitor.HistoryMonitor
prom prometheus.Metrics prom prometheus.Metrics
service service.Service service service.Service
sessions session.Registry sessions session.Registry
sessionsLimiter net.IPLimiter cache cache.Cacher
cache cache.Cacher mainserver *gohttp.Server
mainserver *gohttp.Server sidecarserver *gohttp.Server
sidecarserver *gohttp.Server httpjwt jwt.JWT
httpjwt jwt.JWT update update.Checker
update update.Checker replacer replace.Replacer
replacer replace.Replacer
errorChan chan error errorChan chan error
@@ -372,7 +372,7 @@ func (a *api) start() error {
diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{ diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
Dir: cfg.Storage.Disk.Dir, Dir: cfg.Storage.Disk.Dir,
Size: cfg.Storage.Disk.Size * 1024 * 1024, Size: cfg.Storage.Disk.Size * 1024 * 1024,
Logger: a.log.logger.core.WithComponent("DiskFS"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
if err != nil { if err != nil {
return err return err
@@ -401,7 +401,7 @@ func (a *api) start() error {
Base: baseMemFS.String(), Base: baseMemFS.String(),
Size: cfg.Storage.Memory.Size * 1024 * 1024, Size: cfg.Storage.Memory.Size * 1024 * 1024,
Purge: cfg.Storage.Memory.Purge, Purge: cfg.Storage.Memory.Purge,
Logger: a.log.logger.core.WithComponent("MemFS"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
a.memfs = memfs a.memfs = memfs
@@ -435,7 +435,7 @@ func (a *api) start() error {
Region: cfg.Storage.S3.Region, Region: cfg.Storage.S3.Region,
Bucket: cfg.Storage.S3.Bucket, Bucket: cfg.Storage.S3.Bucket,
UseSSL: cfg.Storage.S3.UseSSL, UseSSL: cfg.Storage.S3.UseSSL,
Logger: a.log.logger.core.WithComponent("S3"), Logger: a.log.logger.core.WithComponent("FS"),
}) })
if err != nil { if err != nil {
return err return err
@@ -665,7 +665,7 @@ func (a *api) start() error {
} }
if cfg.Storage.Disk.Cache.Enable { if cfg.Storage.Disk.Cache.Enable {
diskCache, err := cache.NewLRUCache(cache.LRUConfig{ cache, err := cache.NewLRUCache(cache.LRUConfig{
TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second,
MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024,
MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024,
@@ -675,10 +675,10 @@ func (a *api) start() error {
}) })
if err != nil { if err != nil {
return fmt.Errorf("unable to create disk cache: %w", err) return fmt.Errorf("unable to create cache: %w", err)
} }
a.cache = diskCache a.cache = cache
} }
var autocertManager *autocert.Manager var autocertManager *autocert.Manager
@@ -829,25 +829,50 @@ func (a *api) start() error {
a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address) a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address)
mainserverhandler, err := http.NewServer(http.Config{ serverConfig := http.Config{
Logger: a.log.logger.main, Logger: a.log.logger.main,
LogBuffer: a.log.buffer, LogBuffer: a.log.buffer,
Restream: a.restream, Restream: a.restream,
Metrics: a.metrics, Metrics: a.metrics,
Prometheus: a.prom, Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes, MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs, Filesystems: []httpfs.FS{
MemFS: http.MemFSConfig{ {
EnableAuth: cfg.Storage.Memory.Auth.Enable, Name: "diskfs",
Username: cfg.Storage.Memory.Auth.Username, Mountpoint: "/",
Password: cfg.Storage.Memory.Auth.Password, AllowWrite: false,
Filesystem: a.memfs, Username: "",
}, Password: "",
S3FS: http.MemFSConfig{ DefaultFile: "index.html",
EnableAuth: cfg.Storage.S3.Auth.Enable, DefaultContentType: "text/html",
Username: cfg.Storage.S3.Auth.Username, Gzip: true,
Password: cfg.Storage.S3.Auth.Password, Filesystem: diskfs,
Filesystem: a.s3fs, Cache: a.cache,
},
{
Name: "memfs",
Mountpoint: "/memfs",
AllowWrite: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.memfs,
Cache: a.cache,
},
{
Name: "s3fs",
Mountpoint: "/s3",
AllowWrite: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.s3fs,
Cache: a.cache,
},
}, },
IPLimiter: iplimiter, IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling, Profiling: cfg.Debug.Profiling,
@@ -858,11 +883,12 @@ func (a *api) start() error {
SRT: a.srtserver, SRT: a.srtserver,
JWT: a.httpjwt, JWT: a.httpjwt,
Config: a.config.store, Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions, Sessions: a.sessions,
Router: router, Router: router,
ReadOnly: cfg.API.ReadOnly, ReadOnly: cfg.API.ReadOnly,
}) }
mainserverhandler, err := http.NewServer(serverConfig)
if err != nil { if err != nil {
return fmt.Errorf("unable to create server: %w", err) return fmt.Errorf("unable to create server: %w", err)
@@ -897,40 +923,10 @@ func (a *api) start() error {
a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address) a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address)
sidecarserverhandler, err := http.NewServer(http.Config{ serverConfig.Logger = a.log.logger.sidecar
Logger: a.log.logger.sidecar, serverConfig.IPLimiter = iplimiter
LogBuffer: a.log.buffer,
Restream: a.restream, sidecarserverhandler, err := http.NewServer(serverConfig)
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
Filesystem: a.memfs,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
Filesystem: a.s3fs,
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
Cors: http.CorsConfig{
Origins: cfg.Storage.CORS.Origins,
},
RTMP: a.rtmpserver,
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
if err != nil { if err != nil {
return fmt.Errorf("unable to create sidecar HTTP server: %w", err) return fmt.Errorf("unable to create sidecar HTTP server: %w", err)

584
docs/docs.go
View File

@@ -302,20 +302,53 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/disk": { "/api/v3/fs": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", "description": "Listall registered filesystems",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"summary": "List all files on the filesystem", "summary": "List all registered filesystems",
"operationId": "diskfs-3-list-files", "operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "glob pattern for file names", "description": "glob pattern for file names",
@@ -348,21 +381,28 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/disk/{path}": { "/api/v3/fs/{name}/{path}": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Fetch a file from the filesystem. The contents of that file are returned.", "description": "Fetch a file from a filesystem",
"produces": [ "produces": [
"application/data", "application/data",
"application/json" "application/json"
], ],
"summary": "Fetch a file from the filesystem", "summary": "Fetch a file from a filesystem",
"operationId": "diskfs-3-get-file", "operationId": "filesystem-3-get-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -398,7 +438,7 @@ const docTemplate = `{
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Writes or overwrites a file on the filesystem", "description": "Writes or overwrites a file on a filesystem",
"consumes": [ "consumes": [
"application/data" "application/data"
], ],
@@ -406,9 +446,16 @@ const docTemplate = `{
"text/plain", "text/plain",
"application/json" "application/json"
], ],
"summary": "Add a file to the filesystem", "summary": "Add a file to a filesystem",
"operationId": "diskfs-3-put-file", "operationId": "filesystem-3-put-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -456,13 +503,20 @@ const docTemplate = `{
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Remove a file from the filesystem", "description": "Remove a file from a filesystem",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"summary": "Remove a file from the filesystem", "summary": "Remove a file from a filesystem",
"operationId": "diskfs-3-delete-file", "operationId": "filesystem-3-delete-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -487,240 +541,6 @@ const docTemplate = `{
} }
} }
}, },
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": { "/api/v3/log": {
"get": { "get": {
"security": [ "security": [
@@ -1982,140 +1802,6 @@ const docTemplate = `{
} }
} }
}, },
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": { "/metrics": {
"get": { "get": {
"description": "Prometheus metrics", "description": "Prometheus metrics",
@@ -2175,46 +1861,6 @@ const docTemplate = `{
} }
} }
} }
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@@ -2798,6 +2444,46 @@ const docTemplate = `{
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },
@@ -2869,6 +2555,20 @@ const docTemplate = `{
} }
} }
}, },
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": { "api.GraphQuery": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -4406,6 +4106,46 @@ const docTemplate = `{
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },

584
docs/swagger.json
View File

@@ -294,20 +294,53 @@
} }
} }
}, },
"/api/v3/fs/disk": { "/api/v3/fs": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", "description": "Listall registered filesystems",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"summary": "List all files on the filesystem", "summary": "List all registered filesystems",
"operationId": "diskfs-3-list-files", "operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "glob pattern for file names", "description": "glob pattern for file names",
@@ -340,21 +373,28 @@
} }
} }
}, },
"/api/v3/fs/disk/{path}": { "/api/v3/fs/{name}/{path}": {
"get": { "get": {
"security": [ "security": [
{ {
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Fetch a file from the filesystem. The contents of that file are returned.", "description": "Fetch a file from a filesystem",
"produces": [ "produces": [
"application/data", "application/data",
"application/json" "application/json"
], ],
"summary": "Fetch a file from the filesystem", "summary": "Fetch a file from a filesystem",
"operationId": "diskfs-3-get-file", "operationId": "filesystem-3-get-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -390,7 +430,7 @@
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Writes or overwrites a file on the filesystem", "description": "Writes or overwrites a file on a filesystem",
"consumes": [ "consumes": [
"application/data" "application/data"
], ],
@@ -398,9 +438,16 @@
"text/plain", "text/plain",
"application/json" "application/json"
], ],
"summary": "Add a file to the filesystem", "summary": "Add a file to a filesystem",
"operationId": "diskfs-3-put-file", "operationId": "filesystem-3-put-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -448,13 +495,20 @@
"ApiKeyAuth": [] "ApiKeyAuth": []
} }
], ],
"description": "Remove a file from the filesystem", "description": "Remove a file from a filesystem",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"summary": "Remove a file from the filesystem", "summary": "Remove a file from a filesystem",
"operationId": "diskfs-3-delete-file", "operationId": "filesystem-3-delete-file",
"parameters": [ "parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{ {
"type": "string", "type": "string",
"description": "Path to file", "description": "Path to file",
@@ -479,240 +533,6 @@
} }
} }
}, },
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": { "/api/v3/log": {
"get": { "get": {
"security": [ "security": [
@@ -1974,140 +1794,6 @@
} }
} }
}, },
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": { "/metrics": {
"get": { "get": {
"description": "Prometheus metrics", "description": "Prometheus metrics",
@@ -2167,46 +1853,6 @@
} }
} }
} }
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@@ -2790,6 +2436,46 @@
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },
@@ -2861,6 +2547,20 @@
} }
} }
}, },
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": { "api.GraphQuery": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -4398,6 +4098,46 @@
}, },
"mimetypes_file": { "mimetypes_file": {
"type": "string" "type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
} }
} }
}, },

397
docs/swagger.yaml
View File

@@ -379,6 +379,32 @@ definitions:
type: object type: object
mimetypes_file: mimetypes_file:
type: string type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object type: object
tls: tls:
properties: properties:
@@ -424,6 +450,15 @@ definitions:
size_bytes: size_bytes:
type: integer type: integer
type: object type: object
api.FilesystemInfo:
properties:
mount:
type: string
name:
type: string
type:
type: string
type: object
api.GraphQuery: api.GraphQuery:
properties: properties:
query: query:
@@ -1491,6 +1526,32 @@ definitions:
type: object type: object
mimetypes_file: mimetypes_file:
type: string type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object type: object
tls: tls:
properties: properties:
@@ -1705,34 +1766,6 @@ info:
title: datarhei Core API title: datarhei Core API
version: "3.0" version: "3.0"
paths: paths:
/{path}:
get:
description: Fetch a file from the filesystem. If the file is a directory, a
index.html is returned, if it exists.
operationId: diskfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the filesystem
/api: /api:
get: get:
description: API version and build infos in case auth is valid or not required. description: API version and build infos in case auth is valid or not required.
@@ -1911,12 +1944,33 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Reload the currently active configuration summary: Reload the currently active configuration
/api/v3/fs/disk: /api/v3/fs:
get: get:
description: List all files on the filesystem. The listing can be ordered by description: Listall registered filesystems
name, size, or date of last modification in ascending or descending order. operationId: filesystem-3-list
operationId: diskfs-3-list-files produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FilesystemInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all registered filesystems
/api/v3/fs/{name}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
operationId: filesystem-3-list-files
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: glob pattern for file names - description: glob pattern for file names
in: query in: query
name: glob name: glob
@@ -1940,12 +1994,17 @@ paths:
type: array type: array
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: List all files on the filesystem summary: List all files on a filesystem
/api/v3/fs/disk/{path}: /api/v3/fs/{name}/{path}:
delete: delete:
description: Remove a file from the filesystem description: Remove a file from a filesystem
operationId: diskfs-3-delete-file operationId: filesystem-3-delete-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -1964,12 +2023,16 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Remove a file from the filesystem summary: Remove a file from a filesystem
get: get:
description: Fetch a file from the filesystem. The contents of that file are description: Fetch a file from a filesystem
returned. operationId: filesystem-3-get-file
operationId: diskfs-3-get-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -1993,13 +2056,18 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Fetch a file from the filesystem summary: Fetch a file from a filesystem
put: put:
consumes: consumes:
- application/data - application/data
description: Writes or overwrites a file on the filesystem description: Writes or overwrites a file on a filesystem
operationId: diskfs-3-put-file operationId: filesystem-3-put-file
parameters: parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file - description: Path to file
in: path in: path
name: path name: path
@@ -2031,160 +2099,7 @@ paths:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Add a file to the filesystem summary: Add a file to a filesystem
/api/v3/fs/mem:
get:
description: List all files on the memory filesystem. The listing can be ordered
by name, size, or date of last modification in ascending or descending order.
operationId: memfs-3-list-files
parameters:
- description: glob pattern for file names
in: query
name: glob
type: string
- description: none, name, size, lastmod
in: query
name: sort
type: string
- description: asc, desc
in: query
name: order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FileInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all files on the memory filesystem
/api/v3/fs/mem/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-3-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-3-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the memory filesystem
patch:
consumes:
- application/data
description: Create a link to a file in the memory filesystem. The file linked
to has to exist.
operationId: memfs-3-patch
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: Path to the file to link to
in: body
name: url
required: true
schema:
type: string
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Create a link to a file in the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-3-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the memory filesystem
/api/v3/log: /api/v3/log:
get: get:
description: Get the last log lines of the Restreamer application description: Get the last log lines of the Restreamer application
@@ -3019,94 +2934,6 @@ paths:
schema: schema:
$ref: '#/definitions/api.Error' $ref: '#/definitions/api.Error'
summary: Fetch minimal statistics about a process summary: Fetch minimal statistics about a process
/memfs/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Add a file to the memory filesystem
/metrics: /metrics:
get: get:
description: Prometheus metrics description: Prometheus metrics

7
http/api/fs.go
View File

@@ -6,3 +6,10 @@ type FileInfo struct {
Size int64 `json:"size_bytes" jsonschema:"minimum=0"` Size int64 `json:"size_bytes" jsonschema:"minimum=0"`
LastMod int64 `json:"last_modified" jsonschema:"minimum=0"` LastMod int64 `json:"last_modified" jsonschema:"minimum=0"`
} }
// FilesystemInfo represents information about a filesystem
type FilesystemInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Mount string `json:"mount"`
}

23
http/fs/fs.go Normal file
View File

@@ -0,0 +1,23 @@
package fs
import (
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/io/fs"
)
type FS struct {
Name string
Mountpoint string
AllowWrite bool
Username string
Password string
DefaultFile string
DefaultContentType string
Gzip bool
Filesystem fs.Filesystem
Cache cache.Cacher
}

207
http/handler/api/diskfs.go
View File

@@ -1,207 +0,0 @@
package api
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
handler *handler.DiskFSHandler
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
handler: handler.NewDiskFS(fs, cache),
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. The contents of that file are returned.
// @ID diskfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the filesystem
// @Description Writes or overwrites a file on the filesystem
// @ID diskfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [put]
func (h *DiskFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.cache != nil {
h.cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, path)
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the filesystem
// @Description Remove a file from the filesystem
// @ID diskfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [delete]
func (h *DiskFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.cache != nil {
h.cache.Delete(path)
}
return c.String(http.StatusOK, "OK")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the filesystem
// @Description List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID diskfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk [get]
func (h *DiskFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

146
http/handler/api/filesystems.go Normal file
View File

@@ -0,0 +1,146 @@
package api
import (
"net/http"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
type FSConfig struct {
Type string
Mountpoint string
Handler *handler.FSHandler
}
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
filesystems map[string]FSConfig
}
// NewFS return a new FSHanlder type. You have to provide a filesystem to act on.
func NewFS(filesystems map[string]FSConfig) *FSHandler {
return &FSHandler{
filesystems: filesystems,
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @ID filesystem-3-get-file
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @ID filesystem-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.PutFile(c)
}
// DeleteFileAPI removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @ID filesystem-3-delete-file
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.DeleteFile(c)
}
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID filesystem-3-list-files
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.ListFiles(c)
}
// List lists all registered filesystems
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @ID filesystem-3-list
// @Produce json
// @Success 200 {array} api.FilesystemInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs [get]
func (h *FSHandler) List(c echo.Context) error {
fss := []api.FilesystemInfo{}
for name, config := range h.filesystems {
fss = append(fss, api.FilesystemInfo{
Name: name,
Type: config.Type,
Mount: config.Mountpoint,
})
}
return c.JSON(http.StatusOK, fss)
}

172
http/handler/api/memfs.go
View File

@@ -1,172 +0,0 @@
package api
import (
"io"
"net/http"
"net/url"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
handler *handler.MemFSHandler
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
handler: handler.NewMemFS(fs),
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
return h.handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
return h.handler.PutFile(c)
}
// DeleteFileAPI removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
return h.handler.DeleteFile(c)
}
// PatchFile creates a symbolic link to a file in the filesystem
// @Summary Create a link to a file in the memory filesystem
// @Description Create a link to a file in the memory filesystem. The file linked to has to exist.
// @ID memfs-3-patch
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param url body string true "Path to the file to link to"
// @Success 201 {string} string
// @Failure 400 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [patch]
func (h *MemFSHandler) PatchFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
body, err := io.ReadAll(req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "Failed reading request body", "%s", err)
}
u, err := url.Parse(string(body))
if err != nil {
return api.Err(http.StatusBadRequest, "Body doesn't contain a valid path", "%s", err)
}
if err := h.filesystem.Symlink(u.Path, path); err != nil {
return api.Err(http.StatusBadRequest, "Failed to create symlink", "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
return c.String(http.StatusCreated, "")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the memory filesystem
// @Description List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID memfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem [get]
func (h *MemFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

88
http/handler/diskfs.go
View File

@@ -1,88 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.
// @ID diskfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
path = filepath.Join(path, "index.html")
file.Close()
file = h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}

164
http/handler/filesystem.go Normal file
View File

@@ -0,0 +1,164 @@
package handler
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
fs fs.FS
}
// NewFS return a new FSHandler type. You have to provide a filesystem to act on.
func NewFS(fs fs.FS) *FSHandler {
return &FSHandler{
fs: fs,
}
}
func (h *FSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if len(h.fs.DefaultFile) != 0 {
if stat.IsDir() {
path = filepath.Join(path, h.fs.DefaultFile)
file.Close()
file = h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
func (h *FSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.fs.Filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
func (h *FSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.fs.Filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}
func (h *FSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.fs.Filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

130
http/handler/memfs.go
View File

@@ -1,130 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /memfs/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
stat, _ := file.Stat()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}

271
http/server.go
View File

@@ -29,19 +29,19 @@
package http package http
import ( import (
"fmt"
"net/http" "net/http"
"strings" "strings"
"github.com/datarhei/core/v16/config" "github.com/datarhei/core/v16/config"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/errorhandler" "github.com/datarhei/core/v16/http/errorhandler"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/graph/resolver" "github.com/datarhei/core/v16/http/graph/resolver"
"github.com/datarhei/core/v16/http/handler" "github.com/datarhei/core/v16/http/handler"
api "github.com/datarhei/core/v16/http/handler/api" api "github.com/datarhei/core/v16/http/handler/api"
"github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/http/validator" "github.com/datarhei/core/v16/http/validator"
"github.com/datarhei/core/v16/io/fs"
"github.com/datarhei/core/v16/log" "github.com/datarhei/core/v16/log"
"github.com/datarhei/core/v16/monitor" "github.com/datarhei/core/v16/monitor"
"github.com/datarhei/core/v16/net" "github.com/datarhei/core/v16/net"
@@ -79,9 +79,7 @@ type Config struct {
Metrics monitor.HistoryReader Metrics monitor.HistoryReader
Prometheus prometheus.Reader Prometheus prometheus.Reader
MimeTypesFile string MimeTypesFile string
DiskFS fs.Filesystem Filesystems []fs.FS
MemFS MemFSConfig
S3FS MemFSConfig
IPLimiter net.IPLimiter IPLimiter net.IPLimiter
Profiling bool Profiling bool
Cors CorsConfig Cors CorsConfig
@@ -89,19 +87,11 @@ type Config struct {
SRT srt.Server SRT srt.Server
JWT jwt.JWT JWT jwt.JWT
Config config.Store Config config.Store
Cache cache.Cacher
Sessions session.RegistryReader Sessions session.RegistryReader
Router router.Router Router router.Router
ReadOnly bool ReadOnly bool
} }
type MemFSConfig struct {
EnableAuth bool
Username string
Password string
Filesystem fs.Filesystem
}
type CorsConfig struct { type CorsConfig struct {
Origins []string Origins []string
} }
@@ -115,9 +105,6 @@ type server struct {
handler struct { handler struct {
about *api.AboutHandler about *api.AboutHandler
memfs *handler.MemFSHandler
s3fs *handler.MemFSHandler
diskfs *handler.DiskFSHandler
prometheus *handler.PrometheusHandler prometheus *handler.PrometheusHandler
profiling *handler.ProfilingHandler profiling *handler.ProfilingHandler
ping *handler.PingHandler ping *handler.PingHandler
@@ -129,9 +116,6 @@ type server struct {
log *api.LogHandler log *api.LogHandler
restream *api.RestreamHandler restream *api.RestreamHandler
playout *api.PlayoutHandler playout *api.PlayoutHandler
memfs *api.MemFSHandler
s3fs *api.MemFSHandler
diskfs *api.DiskFSHandler
rtmp *api.RTMPHandler rtmp *api.RTMPHandler
srt *api.SRTHandler srt *api.SRTHandler
config *api.ConfigHandler config *api.ConfigHandler
@@ -150,18 +134,12 @@ type server struct {
session echo.MiddlewareFunc session echo.MiddlewareFunc
} }
memfs struct {
enableAuth bool
username string
password string
}
diskfs fs.Filesystem
gzip struct { gzip struct {
mimetypes []string mimetypes []string
} }
filesystems map[string]*filesystem
router *echo.Echo router *echo.Echo
mimeTypesFile string mimeTypesFile string
profiling bool profiling bool
@@ -169,28 +147,56 @@ type server struct {
readOnly bool readOnly bool
} }
type filesystem struct {
fs.FS
handler *handler.FSHandler
}
func NewServer(config Config) (Server, error) { func NewServer(config Config) (Server, error) {
s := &server{ s := &server{
logger: config.Logger, logger: config.Logger,
mimeTypesFile: config.MimeTypesFile, mimeTypesFile: config.MimeTypesFile,
profiling: config.Profiling, profiling: config.Profiling,
diskfs: config.DiskFS,
readOnly: config.ReadOnly, readOnly: config.ReadOnly,
} }
s.v3handler.diskfs = api.NewDiskFS( s.filesystems = map[string]*filesystem{}
config.DiskFS,
config.Cache,
)
s.handler.diskfs = handler.NewDiskFS( corsPrefixes := map[string][]string{
config.DiskFS, "/api": {"*"},
config.Cache, }
)
s.memfs.enableAuth = config.MemFS.EnableAuth for _, fs := range config.Filesystems {
s.memfs.username = config.MemFS.Username if _, ok := s.filesystems[fs.Name]; ok {
s.memfs.password = config.MemFS.Password return nil, fmt.Errorf("the filesystem name '%s' is already in use", fs.Name)
}
if !strings.HasPrefix(fs.Mountpoint, "/") {
fs.Mountpoint = "/" + fs.Mountpoint
}
if !strings.HasSuffix(fs.Mountpoint, "/") {
fs.Mountpoint = strings.TrimSuffix(fs.Mountpoint, "/")
}
if _, ok := corsPrefixes[fs.Mountpoint]; ok {
return nil, fmt.Errorf("the mount point '%s' is already in use (%s)", fs.Mountpoint, fs.Name)
}
corsPrefixes[fs.Mountpoint] = config.Cors.Origins
filesystem := &filesystem{
FS: fs,
handler: handler.NewFS(fs),
}
s.filesystems[filesystem.Name] = filesystem
}
if _, ok := corsPrefixes["/"]; !ok {
return nil, fmt.Errorf("one filesystem must be mounted at /")
}
if config.Logger == nil { if config.Logger == nil {
s.logger = log.New("HTTP") s.logger = log.New("HTTP")
@@ -222,26 +228,6 @@ func NewServer(config Config) (Server, error) {
) )
} }
if config.MemFS.Filesystem != nil {
s.v3handler.memfs = api.NewMemFS(
config.MemFS.Filesystem,
)
s.handler.memfs = handler.NewMemFS(
config.MemFS.Filesystem,
)
}
if config.S3FS.Filesystem != nil {
s.v3handler.s3fs = api.NewMemFS(
config.S3FS.Filesystem,
)
s.handler.s3fs = handler.NewMemFS(
config.S3FS.Filesystem,
)
}
if config.Prometheus != nil { if config.Prometheus != nil {
s.handler.prometheus = handler.NewPrometheus( s.handler.prometheus = handler.NewPrometheus(
config.Prometheus.HTTPHandler(), config.Prometheus.HTTPHandler(),
@@ -300,12 +286,6 @@ func NewServer(config Config) (Server, error) {
Logger: s.logger, Logger: s.logger,
}) })
if config.Cache != nil {
s.middleware.cache = mwcache.NewWithConfig(mwcache.Config{
Cache: config.Cache,
})
}
s.v3handler.widget = api.NewWidget(api.WidgetConfig{ s.v3handler.widget = api.NewWidget(api.WidgetConfig{
Restream: config.Restream, Restream: config.Restream,
Registry: config.Sessions, Registry: config.Sessions,
@@ -316,12 +296,7 @@ func NewServer(config Config) (Server, error) {
}) })
if middleware, err := mwcors.NewWithConfig(mwcors.Config{ if middleware, err := mwcors.NewWithConfig(mwcors.Config{
Prefixes: map[string][]string{ Prefixes: corsPrefixes,
"/": config.Cors.Origins,
"/api": {"*"},
"/memfs": config.Cors.Origins,
"/s3": config.Cors.Origins,
},
}); err != nil { }); err != nil {
return nil, err return nil, err
} else { } else {
@@ -447,105 +422,48 @@ func (s *server) setRoutes() {
doc.Use(gzipMiddleware) doc.Use(gzipMiddleware)
doc.GET("", echoSwagger.WrapHandler) doc.GET("", echoSwagger.WrapHandler)
// Serve static data // Mount filesystems
fs := s.router.Group("/*") for _, filesystem := range s.filesystems {
fs.Use(mwmime.NewWithConfig(mwmime.Config{ fs := s.router.Group(filesystem.Mountpoint + "/*")
MimeTypesFile: s.mimeTypesFile, fs.Use(mwmime.NewWithConfig(mwmime.Config{
DefaultContentType: "text/html",
}))
fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.cache != nil {
fs.Use(s.middleware.cache)
}
fs.GET("", s.handler.diskfs.GetFile)
fs.HEAD("", s.handler.diskfs.GetFile)
// Memory FS
if s.handler.memfs != nil {
memfs := s.router.Group("/memfs/*")
memfs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data", DefaultContentType: filesystem.DefaultContentType,
})) }))
memfs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed, if filesystem.Gzip {
MinLength: 1000, fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
ContentTypes: s.gzip.mimetypes, Level: mwgzip.BestSpeed,
})) MinLength: 1000,
if s.middleware.session != nil { ContentTypes: s.gzip.mimetypes,
memfs.Use(s.middleware.session) }))
} }
memfs.HEAD("", s.handler.memfs.GetFile) if filesystem.Cache != nil {
memfs.GET("", s.handler.memfs.GetFile) mwcache := mwcache.NewWithConfig(mwcache.Config{
Cache: filesystem.Cache,
})
fs.Use(mwcache)
}
var authmw echo.MiddlewareFunc fs.GET("", filesystem.handler.GetFile)
fs.HEAD("", filesystem.handler.GetFile)
if s.memfs.enableAuth { if len(filesystem.Username) != 0 || len(filesystem.Password) != 0 {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { authmw := middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password { if username == filesystem.Username && password == filesystem.Password {
return true, nil return true, nil
} }
return false, nil return false, nil
}) })
memfs.POST("", s.handler.memfs.PutFile, authmw) fs.POST("", filesystem.handler.PutFile, authmw)
memfs.PUT("", s.handler.memfs.PutFile, authmw) fs.PUT("", filesystem.handler.PutFile, authmw)
memfs.DELETE("", s.handler.memfs.DeleteFile, authmw) fs.DELETE("", filesystem.handler.DeleteFile, authmw)
} else { } else {
memfs.POST("", s.handler.memfs.PutFile) fs.POST("", filesystem.handler.PutFile)
memfs.PUT("", s.handler.memfs.PutFile) fs.PUT("", filesystem.handler.PutFile)
memfs.DELETE("", s.handler.memfs.DeleteFile) fs.DELETE("", filesystem.handler.DeleteFile)
}
}
// S3 FS
if s.handler.s3fs != nil {
s3fs := s.router.Group("/s3/*")
s3fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
s3fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
s3fs.Use(s.middleware.session)
}
s3fs.HEAD("", s.handler.s3fs.GetFile)
s3fs.GET("", s.handler.s3fs.GetFile)
var authmw echo.MiddlewareFunc
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
return true, nil
}
return false, nil
})
s3fs.POST("", s.handler.s3fs.PutFile, authmw)
s3fs.PUT("", s.handler.s3fs.PutFile, authmw)
s3fs.DELETE("", s.handler.s3fs.DeleteFile, authmw)
} else {
s3fs.POST("", s.handler.s3fs.PutFile)
s3fs.PUT("", s.handler.s3fs.PutFile)
s3fs.DELETE("", s.handler.s3fs.DeleteFile)
}
if s.middleware.cache != nil {
s3fs.Use(s.middleware.cache)
} }
} }
@@ -643,44 +561,33 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
} }
} }
// v3 Memory FS // v3 Filesystems
if s.v3handler.memfs != nil { fshandlers := map[string]api.FSConfig{}
v3.GET("/fs/mem", s.v3handler.memfs.ListFiles) for _, fs := range s.filesystems {
v3.GET("/fs/mem/*", s.v3handler.memfs.GetFile) fshandlers[fs.Name] = api.FSConfig{
Type: fs.Filesystem.Type(),
if !s.readOnly { Mountpoint: fs.Mountpoint,
v3.DELETE("/fs/mem/*", s.v3handler.memfs.DeleteFile) Handler: fs.handler,
v3.PUT("/fs/mem/*", s.v3handler.memfs.PutFile)
v3.PATCH("/fs/mem/*", s.v3handler.memfs.PatchFile)
} }
} }
// v3 S3 FS handler := api.NewFS(fshandlers)
if s.v3handler.s3fs != nil {
v3.GET("/fs/s3", s.v3handler.s3fs.ListFiles)
v3.GET("/fs/s3/*", s.v3handler.s3fs.GetFile)
if !s.readOnly { v3.GET("/fs", handler.List)
v3.DELETE("/fs/s3/*", s.v3handler.s3fs.DeleteFile)
v3.PUT("/fs/s3/*", s.v3handler.s3fs.PutFile)
v3.PATCH("/fs/s3/*", s.v3handler.s3fs.PatchFile)
}
}
// v3 Disk FS v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/disk", s.v3handler.diskfs.ListFiles) v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.GET("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data", DefaultContentType: "application/data",
})) }))
v3.HEAD("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{ v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile, MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data", DefaultContentType: "application/data",
})) }))
if !s.readOnly { if !s.readOnly {
v3.PUT("/fs/disk/*", s.v3handler.diskfs.PutFile) v3.PUT("/fs/:name/*", handler.PutFile)
v3.DELETE("/fs/disk/*", s.v3handler.diskfs.DeleteFile) v3.DELETE("/fs/:name/*", handler.DeleteFile)
} }
// v3 RTMP // v3 RTMP

6
io/fs/disk.go
View File

@@ -135,6 +135,8 @@ func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
fs.logger = log.New("") fs.logger = log.New("")
} }
fs.logger = fs.logger.WithField("type", "disk")
if err := fs.Rebase(config.Dir); err != nil { if err := fs.Rebase(config.Dir); err != nil {
return nil, err return nil, err
} }
@@ -172,6 +174,10 @@ func (fs *diskFilesystem) Rebase(base string) error {
return nil return nil
} }
func (fs *diskFilesystem) Type() string {
return "diskfs"
}
func (fs *diskFilesystem) Size() (int64, int64) { func (fs *diskFilesystem) Size() (int64, int64) {
// This is to cache the size for some time in order not to // This is to cache the size for some time in order not to
// stress the underlying filesystem too much. // stress the underlying filesystem too much.

1
io/fs/dummy.go
View File

@@ -24,6 +24,7 @@ type dummyFilesystem struct{}
func (d *dummyFilesystem) Base() string { return "/" } func (d *dummyFilesystem) Base() string { return "/" }
func (d *dummyFilesystem) Rebase(string) error { return nil } func (d *dummyFilesystem) Rebase(string) error { return nil }
func (d *dummyFilesystem) Type() string { return "dummy" }
func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 } func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 }
func (d *dummyFilesystem) Resize(int64) {} func (d *dummyFilesystem) Resize(int64) {}
func (d *dummyFilesystem) Files() int64 { return 0 } func (d *dummyFilesystem) Files() int64 { return 0 }

3
io/fs/fs.go
View File

@@ -44,6 +44,9 @@ type Filesystem interface {
// Rebase sets a new base path for this filesystem // Rebase sets a new base path for this filesystem
Rebase(string) error Rebase(string) error
// Type returns the type of this filesystem
Type() string
// Size returns the consumed size and capacity of the filesystem in bytes. The // Size returns the consumed size and capacity of the filesystem in bytes. The
// capacity is negative if the filesystem can consume as much space as it can. // capacity is negative if the filesystem can consume as much space as it can.
Size() (int64, int64) Size() (int64, int64)

6
io/fs/mem.go
View File

@@ -146,6 +146,8 @@ func NewMemFilesystem(config MemConfig) Filesystem {
fs.logger = log.New("") fs.logger = log.New("")
} }
fs.logger = fs.logger.WithField("type", "mem")
fs.files = make(map[string]*memFile) fs.files = make(map[string]*memFile)
fs.dataPool = sync.Pool{ fs.dataPool = sync.Pool{
@@ -172,6 +174,10 @@ func (fs *memFilesystem) Rebase(base string) error {
return nil return nil
} }
func (fs *memFilesystem) Type() string {
return "memfs"
}
func (fs *memFilesystem) Size() (int64, int64) { func (fs *memFilesystem) Size() (int64, int64) {
fs.filesLock.RLock() fs.filesLock.RLock()
defer fs.filesLock.RUnlock() defer fs.filesLock.RUnlock()

5
io/fs/s3.go
View File

@@ -66,6 +66,7 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
} }
fs.logger = fs.logger.WithFields(log.Fields{ fs.logger = fs.logger.WithFields(log.Fields{
"type": "s3",
"bucket": fs.bucket, "bucket": fs.bucket,
"region": fs.region, "region": fs.region,
"endpoint": fs.endpoint, "endpoint": fs.endpoint,
@@ -107,6 +108,10 @@ func (fs *s3fs) Rebase(base string) error {
return nil return nil
} }
func (fs *s3fs) Type() string {
return "s3fs"
}
func (fs *s3fs) Size() (int64, int64) { func (fs *s3fs) Size() (int64, int64) {
size := int64(0) size := int64(0)