apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-10-07 00:43:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor internal filesystem handling

Browse Source
This commit is contained in:
Ingo Oppermann
2022-08-22 19:10:59 +03:00
parent 9a49e371e3
commit cb0bc494f9
18 changed files with 955 additions and 1981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
app/api/api.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/datarhei/core/v16/ffmpeg"
"github.com/datarhei/core/v16/http"
"github.com/datarhei/core/v16/http/cache"
httpfs "github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/io/fs"
@@ -71,7 +72,6 @@ type api struct {
prom prometheus.Metrics
service service.Service
sessions session.Registry
sessionsLimiter net.IPLimiter
cache cache.Cacher
mainserver *gohttp.Server
sidecarserver *gohttp.Server
@@ -372,7 +372,7 @@ func (a *api) start() error {
diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
Dir: cfg.Storage.Disk.Dir,
Size: cfg.Storage.Disk.Size * 1024 * 1024,
Logger: a.log.logger.core.WithComponent("DiskFS"),
Logger: a.log.logger.core.WithComponent("FS"),
})
if err != nil {
return err
@@ -401,7 +401,7 @@ func (a *api) start() error {
Base: baseMemFS.String(),
Size: cfg.Storage.Memory.Size * 1024 * 1024,
Purge: cfg.Storage.Memory.Purge,
Logger: a.log.logger.core.WithComponent("MemFS"),
Logger: a.log.logger.core.WithComponent("FS"),
})
a.memfs = memfs
@@ -435,7 +435,7 @@ func (a *api) start() error {
Region: cfg.Storage.S3.Region,
Bucket: cfg.Storage.S3.Bucket,
UseSSL: cfg.Storage.S3.UseSSL,
Logger: a.log.logger.core.WithComponent("S3"),
Logger: a.log.logger.core.WithComponent("FS"),
})
if err != nil {
return err
@@ -665,7 +665,7 @@ func (a *api) start() error {
}
if cfg.Storage.Disk.Cache.Enable {
diskCache, err := cache.NewLRUCache(cache.LRUConfig{
cache, err := cache.NewLRUCache(cache.LRUConfig{
TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second,
MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024,
MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024,
@@ -675,10 +675,10 @@ func (a *api) start() error {
})
if err != nil {
return fmt.Errorf("unable to create disk cache: %w", err)
return fmt.Errorf("unable to create cache: %w", err)
}
a.cache = diskCache
a.cache = cache
}
var autocertManager *autocert.Manager
@@ -829,25 +829,50 @@ func (a *api) start() error {
a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address)
mainserverhandler, err := http.NewServer(http.Config{
serverConfig := http.Config{
Logger: a.log.logger.main,
LogBuffer: a.log.buffer,
Restream: a.restream,
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Filesystems: []httpfs.FS{
{
Name: "diskfs",
Mountpoint: "/",
AllowWrite: false,
Username: "",
Password: "",
DefaultFile: "index.html",
DefaultContentType: "text/html",
Gzip: true,
Filesystem: diskfs,
Cache: a.cache,
},
{
Name: "memfs",
Mountpoint: "/memfs",
AllowWrite: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.memfs,
Cache: a.cache,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
{
Name: "s3fs",
Mountpoint: "/s3",
AllowWrite: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
DefaultFile: "",
DefaultContentType: "application/data",
Gzip: true,
Filesystem: a.s3fs,
Cache: a.cache,
},
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
@@ -858,11 +883,12 @@ func (a *api) start() error {
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
}
mainserverhandler, err := http.NewServer(serverConfig)
if err != nil {
return fmt.Errorf("unable to create server: %w", err)
@@ -897,40 +923,10 @@ func (a *api) start() error {
a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address)
sidecarserverhandler, err := http.NewServer(http.Config{
Logger: a.log.logger.sidecar,
LogBuffer: a.log.buffer,
Restream: a.restream,
Metrics: a.metrics,
Prometheus: a.prom,
MimeTypesFile: cfg.Storage.MimeTypes,
DiskFS: a.diskfs,
MemFS: http.MemFSConfig{
EnableAuth: cfg.Storage.Memory.Auth.Enable,
Username: cfg.Storage.Memory.Auth.Username,
Password: cfg.Storage.Memory.Auth.Password,
Filesystem: a.memfs,
},
S3FS: http.MemFSConfig{
EnableAuth: cfg.Storage.S3.Auth.Enable,
Username: cfg.Storage.S3.Auth.Username,
Password: cfg.Storage.S3.Auth.Password,
Filesystem: a.s3fs,
},
IPLimiter: iplimiter,
Profiling: cfg.Debug.Profiling,
Cors: http.CorsConfig{
Origins: cfg.Storage.CORS.Origins,
},
RTMP: a.rtmpserver,
SRT: a.srtserver,
JWT: a.httpjwt,
Config: a.config.store,
Cache: a.cache,
Sessions: a.sessions,
Router: router,
ReadOnly: cfg.API.ReadOnly,
})
serverConfig.Logger = a.log.logger.sidecar
serverConfig.IPLimiter = iplimiter
sidecarserverhandler, err := http.NewServer(serverConfig)
if err != nil {
return fmt.Errorf("unable to create sidecar HTTP server: %w", err)

584
docs/docs.go
View File

@@ -302,20 +302,53 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/disk": {
"/api/v3/fs": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"description": "Listall registered filesystems",
"produces": [
"application/json"
],
"summary": "List all files on the filesystem",
"operationId": "diskfs-3-list-files",
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
@@ -348,21 +381,28 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/disk/{path}": {
"/api/v3/fs/{name}/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the filesystem. The contents of that file are returned.",
"description": "Fetch a file from a filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-3-get-file",
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -398,7 +438,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the filesystem",
"description": "Writes or overwrites a file on a filesystem",
"consumes": [
"application/data"
],
@@ -406,9 +446,16 @@ const docTemplate = `{
"text/plain",
"application/json"
],
"summary": "Add a file to the filesystem",
"operationId": "diskfs-3-put-file",
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -456,13 +503,20 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the filesystem",
"description": "Remove a file from a filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the filesystem",
"operationId": "diskfs-3-delete-file",
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -487,240 +541,6 @@ const docTemplate = `{
}
}
},
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": {
"get": {
"security": [
@@ -1982,140 +1802,6 @@ const docTemplate = `{
}
}
},
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": {
"get": {
"description": "Prometheus metrics",
@@ -2175,46 +1861,6 @@ const docTemplate = `{
}
}
}
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
}
},
"definitions": {
@@ -2798,6 +2444,46 @@ const docTemplate = `{
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},
@@ -2869,6 +2555,20 @@ const docTemplate = `{
}
}
},
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {
@@ -4406,6 +4106,46 @@ const docTemplate = `{
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},

584
docs/swagger.json
View File

@@ -294,20 +294,53 @@
}
}
},
"/api/v3/fs/disk": {
"/api/v3/fs": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"description": "Listall registered filesystems",
"produces": [
"application/json"
],
"summary": "List all files on the filesystem",
"operationId": "diskfs-3-list-files",
"summary": "List all registered filesystems",
"operationId": "filesystem-3-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FilesystemInfo"
}
}
}
}
}
},
"/api/v3/fs/{name}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on a filesystem",
"operationId": "filesystem-3-list-files",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "glob pattern for file names",
@@ -340,21 +373,28 @@
}
}
},
"/api/v3/fs/disk/{path}": {
"/api/v3/fs/{name}/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the filesystem. The contents of that file are returned.",
"description": "Fetch a file from a filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-3-get-file",
"summary": "Fetch a file from a filesystem",
"operationId": "filesystem-3-get-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -390,7 +430,7 @@
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the filesystem",
"description": "Writes or overwrites a file on a filesystem",
"consumes": [
"application/data"
],
@@ -398,9 +438,16 @@
"text/plain",
"application/json"
],
"summary": "Add a file to the filesystem",
"operationId": "diskfs-3-put-file",
"summary": "Add a file to a filesystem",
"operationId": "filesystem-3-put-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -448,13 +495,20 @@
"ApiKeyAuth": []
}
],
"description": "Remove a file from the filesystem",
"description": "Remove a file from a filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the filesystem",
"operationId": "diskfs-3-delete-file",
"summary": "Remove a file from a filesystem",
"operationId": "filesystem-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Name of the filesystem",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Path to file",
@@ -479,240 +533,6 @@
}
}
},
"/api/v3/fs/mem": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
"produces": [
"application/json"
],
"summary": "List all files on the memory filesystem",
"operationId": "memfs-3-list-files",
"parameters": [
{
"type": "string",
"description": "glob pattern for file names",
"name": "glob",
"in": "query"
},
{
"type": "string",
"description": "none, name, size, lastmod",
"name": "sort",
"in": "query"
},
{
"type": "string",
"description": "asc, desc",
"name": "order",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.FileInfo"
}
}
}
}
}
},
"/api/v3/fs/mem/{path}": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-3-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-3-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-3-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Create a link to a file in the memory filesystem",
"operationId": "memfs-3-patch",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "Path to the file to link to",
"name": "url",
"in": "body",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/api/v3/log": {
"get": {
"security": [
@@ -1974,140 +1794,6 @@
}
}
},
"/memfs/{path}": {
"get": {
"description": "Fetch a file from the memory filesystem",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the memory filesystem",
"operationId": "memfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"put": {
"security": [
{
"BasicAuth": []
}
],
"description": "Writes or overwrites a file on the memory filesystem",
"consumes": [
"application/data"
],
"produces": [
"text/plain",
"application/json"
],
"summary": "Add a file to the memory filesystem",
"operationId": "memfs-put-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
},
{
"description": "File data",
"name": "data",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer"
}
}
}
],
"responses": {
"201": {
"description": "Created",
"schema": {
"type": "string"
}
},
"204": {
"description": "No Content",
"schema": {
"type": "string"
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
},
"delete": {
"security": [
{
"BasicAuth": []
}
],
"description": "Remove a file from the memory filesystem",
"produces": [
"text/plain"
],
"summary": "Remove a file from the memory filesystem",
"operationId": "memfs-delete-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
},
"/metrics": {
"get": {
"description": "Prometheus metrics",
@@ -2167,46 +1853,6 @@
}
}
}
},
"/{path}": {
"get": {
"description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.",
"produces": [
"application/data",
"application/json"
],
"summary": "Fetch a file from the filesystem",
"operationId": "diskfs-get-file",
"parameters": [
{
"type": "string",
"description": "Path to file",
"name": "path",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"301": {
"description": "Moved Permanently",
"schema": {
"type": "string"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.Error"
}
}
}
}
}
},
"definitions": {
@@ -2790,6 +2436,46 @@
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},
@@ -2861,6 +2547,20 @@
}
}
},
"api.FilesystemInfo": {
"type": "object",
"properties": {
"mount": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"api.GraphQuery": {
"type": "object",
"properties": {
@@ -4398,6 +4098,46 @@
},
"mimetypes_file": {
"type": "string"
},
"s3": {
"type": "object",
"properties": {
"access_key_id": {
"type": "string"
},
"auth": {
"type": "object",
"properties": {
"enable": {
"type": "boolean"
},
"password": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bucket": {
"type": "string"
},
"enable": {
"type": "boolean"
},
"endpoint": {
"type": "string"
},
"region": {
"type": "string"
},
"secret_access_key": {
"type": "string"
},
"use_ssl": {
"type": "boolean"
}
}
}
}
},

397
docs/swagger.yaml
View File

@@ -379,6 +379,32 @@ definitions:
type: object
mimetypes_file:
type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object
tls:
properties:
@@ -424,6 +450,15 @@ definitions:
size_bytes:
type: integer
type: object
api.FilesystemInfo:
properties:
mount:
type: string
name:
type: string
type:
type: string
type: object
api.GraphQuery:
properties:
query:
@@ -1491,6 +1526,32 @@ definitions:
type: object
mimetypes_file:
type: string
s3:
properties:
access_key_id:
type: string
auth:
properties:
enable:
type: boolean
password:
type: string
username:
type: string
type: object
bucket:
type: string
enable:
type: boolean
endpoint:
type: string
region:
type: string
secret_access_key:
type: string
use_ssl:
type: boolean
type: object
type: object
tls:
properties:
@@ -1705,34 +1766,6 @@ info:
title: datarhei Core API
version: "3.0"
paths:
/{path}:
get:
description: Fetch a file from the filesystem. If the file is a directory, a
index.html is returned, if it exists.
operationId: diskfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the filesystem
/api:
get:
description: API version and build infos in case auth is valid or not required.
@@ -1911,12 +1944,33 @@ paths:
security:
- ApiKeyAuth: []
summary: Reload the currently active configuration
/api/v3/fs/disk:
/api/v3/fs:
get:
description: List all files on the filesystem. The listing can be ordered by
name, size, or date of last modification in ascending or descending order.
operationId: diskfs-3-list-files
description: Listall registered filesystems
operationId: filesystem-3-list
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FilesystemInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all registered filesystems
/api/v3/fs/{name}:
get:
description: List all files on a filesystem. The listing can be ordered by name,
size, or date of last modification in ascending or descending order.
operationId: filesystem-3-list-files
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: glob pattern for file names
in: query
name: glob
@@ -1940,12 +1994,17 @@ paths:
type: array
security:
- ApiKeyAuth: []
summary: List all files on the filesystem
/api/v3/fs/disk/{path}:
summary: List all files on a filesystem
/api/v3/fs/{name}/{path}:
delete:
description: Remove a file from the filesystem
operationId: diskfs-3-delete-file
description: Remove a file from a filesystem
operationId: filesystem-3-delete-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -1964,12 +2023,16 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the filesystem
summary: Remove a file from a filesystem
get:
description: Fetch a file from the filesystem. The contents of that file are
returned.
operationId: diskfs-3-get-file
description: Fetch a file from a filesystem
operationId: filesystem-3-get-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -1993,13 +2056,18 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the filesystem
summary: Fetch a file from a filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the filesystem
operationId: diskfs-3-put-file
description: Writes or overwrites a file on a filesystem
operationId: filesystem-3-put-file
parameters:
- description: Name of the filesystem
in: path
name: name
required: true
type: string
- description: Path to file
in: path
name: path
@@ -2031,160 +2099,7 @@ paths:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the filesystem
/api/v3/fs/mem:
get:
description: List all files on the memory filesystem. The listing can be ordered
by name, size, or date of last modification in ascending or descending order.
operationId: memfs-3-list-files
parameters:
- description: glob pattern for file names
in: query
name: glob
type: string
- description: none, name, size, lastmod
in: query
name: sort
type: string
- description: asc, desc
in: query
name: order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/api.FileInfo'
type: array
security:
- ApiKeyAuth: []
summary: List all files on the memory filesystem
/api/v3/fs/mem/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-3-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-3-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Fetch a file from the memory filesystem
patch:
consumes:
- application/data
description: Create a link to a file in the memory filesystem. The file linked
to has to exist.
operationId: memfs-3-patch
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: Path to the file to link to
in: body
name: url
required: true
schema:
type: string
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Create a link to a file in the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-3-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- ApiKeyAuth: []
summary: Add a file to the memory filesystem
summary: Add a file to a filesystem
/api/v3/log:
get:
description: Get the last log lines of the Restreamer application
@@ -3019,94 +2934,6 @@ paths:
schema:
$ref: '#/definitions/api.Error'
summary: Fetch minimal statistics about a process
/memfs/{path}:
delete:
description: Remove a file from the memory filesystem
operationId: memfs-delete-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- text/plain
responses:
"200":
description: OK
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Remove a file from the memory filesystem
get:
description: Fetch a file from the memory filesystem
operationId: memfs-get-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
produces:
- application/data
- application/json
responses:
"200":
description: OK
schema:
type: file
"301":
description: Moved Permanently
schema:
type: string
"404":
description: Not Found
schema:
$ref: '#/definitions/api.Error'
summary: Fetch a file from the memory filesystem
put:
consumes:
- application/data
description: Writes or overwrites a file on the memory filesystem
operationId: memfs-put-file
parameters:
- description: Path to file
in: path
name: path
required: true
type: string
- description: File data
in: body
name: data
required: true
schema:
items:
type: integer
type: array
produces:
- text/plain
- application/json
responses:
"201":
description: Created
schema:
type: string
"204":
description: No Content
schema:
type: string
"507":
description: Insufficient Storage
schema:
$ref: '#/definitions/api.Error'
security:
- BasicAuth: []
summary: Add a file to the memory filesystem
/metrics:
get:
description: Prometheus metrics

7
http/api/fs.go
View File

@@ -6,3 +6,10 @@ type FileInfo struct {
Size int64 `json:"size_bytes" jsonschema:"minimum=0"`
LastMod int64 `json:"last_modified" jsonschema:"minimum=0"`
}
// FilesystemInfo represents information about a filesystem
type FilesystemInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Mount string `json:"mount"`
}

23
http/fs/fs.go Normal file
View File

@@ -0,0 +1,23 @@
package fs
import (
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/io/fs"
)
type FS struct {
Name string
Mountpoint string
AllowWrite bool
Username string
Password string
DefaultFile string
DefaultContentType string
Gzip bool
Filesystem fs.Filesystem
Cache cache.Cacher
}

207
http/handler/api/diskfs.go
View File

@@ -1,207 +0,0 @@
package api
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
handler *handler.DiskFSHandler
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
handler: handler.NewDiskFS(fs, cache),
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. The contents of that file are returned.
// @ID diskfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the filesystem
// @Description Writes or overwrites a file on the filesystem
// @ID diskfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [put]
func (h *DiskFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.cache != nil {
h.cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, path)
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the filesystem
// @Description Remove a file from the filesystem
// @ID diskfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk/{path} [delete]
func (h *DiskFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.cache != nil {
h.cache.Delete(path)
}
return c.String(http.StatusOK, "OK")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the filesystem
// @Description List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID diskfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/disk [get]
func (h *DiskFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

146
http/handler/api/filesystems.go Normal file
View File

@@ -0,0 +1,146 @@
package api
import (
"net/http"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
type FSConfig struct {
Type string
Mountpoint string
Handler *handler.FSHandler
}
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
filesystems map[string]FSConfig
}
// NewFS return a new FSHanlder type. You have to provide a filesystem to act on.
func NewFS(filesystems map[string]FSConfig) *FSHandler {
return &FSHandler{
filesystems: filesystems,
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from a filesystem
// @Description Fetch a file from a filesystem
// @ID filesystem-3-get-file
// @Produce application/data
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [get]
func (h *FSHandler) GetFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to a filesystem
// @Description Writes or overwrites a file on a filesystem
// @ID filesystem-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [put]
func (h *FSHandler) PutFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.PutFile(c)
}
// DeleteFileAPI removes a file from a filesystem
// @Summary Remove a file from a filesystem
// @Description Remove a file from a filesystem
// @ID filesystem-3-delete-file
// @Produce text/plain
// @Param name path string true "Name of the filesystem"
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name}/{path} [delete]
func (h *FSHandler) DeleteFile(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.DeleteFile(c)
}
// ListFiles lists all files on a filesystem
// @Summary List all files on a filesystem
// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID filesystem-3-list-files
// @Produce json
// @Param name path string true "Name of the filesystem"
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/{name} [get]
func (h *FSHandler) ListFiles(c echo.Context) error {
name := util.PathParam(c, "name")
config, ok := h.filesystems[name]
if !ok {
return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name)
}
return config.Handler.ListFiles(c)
}
// List lists all registered filesystems
// @Summary List all registered filesystems
// @Description Listall registered filesystems
// @ID filesystem-3-list
// @Produce json
// @Success 200 {array} api.FilesystemInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs [get]
func (h *FSHandler) List(c echo.Context) error {
fss := []api.FilesystemInfo{}
for name, config := range h.filesystems {
fss = append(fss, api.FilesystemInfo{
Name: name,
Type: config.Type,
Mount: config.Mountpoint,
})
}
return c.JSON(http.StatusOK, fss)
}

172
http/handler/api/memfs.go
View File

@@ -1,172 +0,0 @@
package api
import (
"io"
"net/http"
"net/url"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
handler *handler.MemFSHandler
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
handler: handler.NewMemFS(fs),
}
}
// GetFileAPI returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-3-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
return h.handler.GetFile(c)
}
// PutFileAPI adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-3-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
return h.handler.PutFile(c)
}
// DeleteFileAPI removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-3-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
return h.handler.DeleteFile(c)
}
// PatchFile creates a symbolic link to a file in the filesystem
// @Summary Create a link to a file in the memory filesystem
// @Description Create a link to a file in the memory filesystem. The file linked to has to exist.
// @ID memfs-3-patch
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param url body string true "Path to the file to link to"
// @Success 201 {string} string
// @Failure 400 {object} api.Error
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem/{path} [patch]
func (h *MemFSHandler) PatchFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
body, err := io.ReadAll(req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "Failed reading request body", "%s", err)
}
u, err := url.Parse(string(body))
if err != nil {
return api.Err(http.StatusBadRequest, "Body doesn't contain a valid path", "%s", err)
}
if err := h.filesystem.Symlink(u.Path, path); err != nil {
return api.Err(http.StatusBadRequest, "Failed to create symlink", "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
return c.String(http.StatusCreated, "")
}
// ListFiles lists all files on the filesystem
// @Summary List all files on the memory filesystem
// @Description List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.
// @ID memfs-3-list-files
// @Produce json
// @Param glob query string false "glob pattern for file names"
// @Param sort query string false "none, name, size, lastmod"
// @Param order query string false "asc, desc"
// @Success 200 {array} api.FileInfo
// @Security ApiKeyAuth
// @Router /api/v3/fs/mem [get]
func (h *MemFSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

88
http/handler/diskfs.go
View File

@@ -1,88 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The DiskFSHandler type provides handlers for manipulating a filesystem
type DiskFSHandler struct {
cache cache.Cacher
filesystem fs.Filesystem
}
// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally
// a Cacher where files will be purged from if the Cacher is related to the filesystem.
func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler {
return &DiskFSHandler{
cache: cache,
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the filesystem
// @Description Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.
// @ID diskfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /{path} [get]
func (h *DiskFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if stat.IsDir() {
path = filepath.Join(path, "index.html")
file.Close()
file = h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}

164
http/handler/filesystem.go Normal file
View File

@@ -0,0 +1,164 @@
package handler
import (
"net/http"
"path/filepath"
"sort"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/labstack/echo/v4"
)
// The FSHandler type provides handlers for manipulating a filesystem
type FSHandler struct {
fs fs.FS
}
// NewFS return a new FSHandler type. You have to provide a filesystem to act on.
func NewFS(fs fs.FS) *FSHandler {
return &FSHandler{
fs: fs,
}
}
func (h *FSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ := file.Stat()
if len(h.fs.DefaultFile) != 0 {
if stat.IsDir() {
path = filepath.Join(path, h.fs.DefaultFile)
file.Close()
file = h.fs.Filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
stat, _ = file.Stat()
}
}
defer file.Close()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
func (h *FSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.fs.Filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
func (h *FSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.fs.Filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
if h.fs.Cache != nil {
h.fs.Cache.Delete(path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}
func (h *FSHandler) ListFiles(c echo.Context) error {
pattern := util.DefaultQuery(c, "glob", "")
sortby := util.DefaultQuery(c, "sort", "none")
order := util.DefaultQuery(c, "order", "asc")
files := h.fs.Filesystem.List(pattern)
var sortFunc func(i, j int) bool
switch sortby {
case "name":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() }
} else {
sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() }
}
case "size":
if order == "desc" {
sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() }
} else {
sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() }
}
default:
if order == "asc" {
sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }
} else {
sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) }
}
}
sort.Slice(files, sortFunc)
var fileinfos []api.FileInfo = make([]api.FileInfo, len(files))
for i, f := range files {
fileinfos[i] = api.FileInfo{
Name: f.Name(),
Size: f.Size(),
LastMod: f.ModTime().Unix(),
}
}
return c.JSON(http.StatusOK, fileinfos)
}

130
http/handler/memfs.go
View File

@@ -1,130 +0,0 @@
package handler
import (
"net/http"
"path/filepath"
"github.com/datarhei/core/v16/http/api"
"github.com/datarhei/core/v16/http/handler/util"
"github.com/datarhei/core/v16/io/fs"
"github.com/labstack/echo/v4"
)
// The MemFSHandler type provides handlers for manipulating a filesystem
type MemFSHandler struct {
filesystem fs.Filesystem
}
// NewMemFS return a new MemFS type. You have to provide a filesystem to act on.
func NewMemFS(fs fs.Filesystem) *MemFSHandler {
return &MemFSHandler{
filesystem: fs,
}
}
// GetFile returns the file at the given path
// @Summary Fetch a file from the memory filesystem
// @Description Fetch a file from the memory filesystem
// @ID memfs-get-file
// @Produce application/data
// @Produce json
// @Param path path string true "Path to file"
// @Success 200 {file} byte
// @Success 301 {string} string
// @Failure 404 {object} api.Error
// @Router /memfs/{path} [get]
func (h *MemFSHandler) GetFile(c echo.Context) error {
path := util.PathWildcardParam(c)
mimeType := c.Response().Header().Get(echo.HeaderContentType)
c.Response().Header().Del(echo.HeaderContentType)
file := h.filesystem.Open(path)
if file == nil {
return api.Err(http.StatusNotFound, "File not found", path)
}
defer file.Close()
stat, _ := file.Stat()
c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
if path, ok := stat.IsLink(); ok {
path = filepath.Clean("/" + path)
if path[0] == '/' {
path = path[1:]
}
return c.Redirect(http.StatusMovedPermanently, path)
}
c.Response().Header().Set(echo.HeaderContentType, mimeType)
if c.Request().Method == "HEAD" {
return c.Blob(http.StatusOK, "application/data", nil)
}
return c.Stream(http.StatusOK, "application/data", file)
}
// PutFile adds or overwrites a file at the given path
// @Summary Add a file to the memory filesystem
// @Description Writes or overwrites a file on the memory filesystem
// @ID memfs-put-file
// @Accept application/data
// @Produce text/plain
// @Produce json
// @Param path path string true "Path to file"
// @Param data body []byte true "File data"
// @Success 201 {string} string
// @Success 204 {string} string
// @Failure 507 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [put]
func (h *MemFSHandler) PutFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
req := c.Request()
_, created, err := h.filesystem.Store(path, req.Body)
if err != nil {
return api.Err(http.StatusBadRequest, "%s", err)
}
c.Response().Header().Set("Content-Location", req.URL.RequestURI())
if created {
return c.String(http.StatusCreated, "")
}
return c.NoContent(http.StatusNoContent)
}
// DeleteFile removes a file from the filesystem
// @Summary Remove a file from the memory filesystem
// @Description Remove a file from the memory filesystem
// @ID memfs-delete-file
// @Produce text/plain
// @Param path path string true "Path to file"
// @Success 200 {string} string
// @Failure 404 {object} api.Error
// @Security BasicAuth
// @Router /memfs/{path} [delete]
func (h *MemFSHandler) DeleteFile(c echo.Context) error {
path := util.PathWildcardParam(c)
c.Response().Header().Del(echo.HeaderContentType)
size := h.filesystem.Delete(path)
if size < 0 {
return api.Err(http.StatusNotFound, "File not found", path)
}
return c.String(http.StatusOK, "Deleted: "+path)
}

257
http/server.go
View File

@@ -29,19 +29,19 @@
package http
import (
"fmt"
"net/http"
"strings"
"github.com/datarhei/core/v16/config"
"github.com/datarhei/core/v16/http/cache"
"github.com/datarhei/core/v16/http/errorhandler"
"github.com/datarhei/core/v16/http/fs"
"github.com/datarhei/core/v16/http/graph/resolver"
"github.com/datarhei/core/v16/http/handler"
api "github.com/datarhei/core/v16/http/handler/api"
"github.com/datarhei/core/v16/http/jwt"
"github.com/datarhei/core/v16/http/router"
"github.com/datarhei/core/v16/http/validator"
"github.com/datarhei/core/v16/io/fs"
"github.com/datarhei/core/v16/log"
"github.com/datarhei/core/v16/monitor"
"github.com/datarhei/core/v16/net"
@@ -79,9 +79,7 @@ type Config struct {
Metrics monitor.HistoryReader
Prometheus prometheus.Reader
MimeTypesFile string
DiskFS fs.Filesystem
MemFS MemFSConfig
S3FS MemFSConfig
Filesystems []fs.FS
IPLimiter net.IPLimiter
Profiling bool
Cors CorsConfig
@@ -89,19 +87,11 @@ type Config struct {
SRT srt.Server
JWT jwt.JWT
Config config.Store
Cache cache.Cacher
Sessions session.RegistryReader
Router router.Router
ReadOnly bool
}
type MemFSConfig struct {
EnableAuth bool
Username string
Password string
Filesystem fs.Filesystem
}
type CorsConfig struct {
Origins []string
}
@@ -115,9 +105,6 @@ type server struct {
handler struct {
about *api.AboutHandler
memfs *handler.MemFSHandler
s3fs *handler.MemFSHandler
diskfs *handler.DiskFSHandler
prometheus *handler.PrometheusHandler
profiling *handler.ProfilingHandler
ping *handler.PingHandler
@@ -129,9 +116,6 @@ type server struct {
log *api.LogHandler
restream *api.RestreamHandler
playout *api.PlayoutHandler
memfs *api.MemFSHandler
s3fs *api.MemFSHandler
diskfs *api.DiskFSHandler
rtmp *api.RTMPHandler
srt *api.SRTHandler
config *api.ConfigHandler
@@ -150,18 +134,12 @@ type server struct {
session echo.MiddlewareFunc
}
memfs struct {
enableAuth bool
username string
password string
}
diskfs fs.Filesystem
gzip struct {
mimetypes []string
}
filesystems map[string]*filesystem
router *echo.Echo
mimeTypesFile string
profiling bool
@@ -169,28 +147,56 @@ type server struct {
readOnly bool
}
type filesystem struct {
fs.FS
handler *handler.FSHandler
}
func NewServer(config Config) (Server, error) {
s := &server{
logger: config.Logger,
mimeTypesFile: config.MimeTypesFile,
profiling: config.Profiling,
diskfs: config.DiskFS,
readOnly: config.ReadOnly,
}
s.v3handler.diskfs = api.NewDiskFS(
config.DiskFS,
config.Cache,
)
s.filesystems = map[string]*filesystem{}
s.handler.diskfs = handler.NewDiskFS(
config.DiskFS,
config.Cache,
)
corsPrefixes := map[string][]string{
"/api": {"*"},
}
s.memfs.enableAuth = config.MemFS.EnableAuth
s.memfs.username = config.MemFS.Username
s.memfs.password = config.MemFS.Password
for _, fs := range config.Filesystems {
if _, ok := s.filesystems[fs.Name]; ok {
return nil, fmt.Errorf("the filesystem name '%s' is already in use", fs.Name)
}
if !strings.HasPrefix(fs.Mountpoint, "/") {
fs.Mountpoint = "/" + fs.Mountpoint
}
if !strings.HasSuffix(fs.Mountpoint, "/") {
fs.Mountpoint = strings.TrimSuffix(fs.Mountpoint, "/")
}
if _, ok := corsPrefixes[fs.Mountpoint]; ok {
return nil, fmt.Errorf("the mount point '%s' is already in use (%s)", fs.Mountpoint, fs.Name)
}
corsPrefixes[fs.Mountpoint] = config.Cors.Origins
filesystem := &filesystem{
FS: fs,
handler: handler.NewFS(fs),
}
s.filesystems[filesystem.Name] = filesystem
}
if _, ok := corsPrefixes["/"]; !ok {
return nil, fmt.Errorf("one filesystem must be mounted at /")
}
if config.Logger == nil {
s.logger = log.New("HTTP")
@@ -222,26 +228,6 @@ func NewServer(config Config) (Server, error) {
)
}
if config.MemFS.Filesystem != nil {
s.v3handler.memfs = api.NewMemFS(
config.MemFS.Filesystem,
)
s.handler.memfs = handler.NewMemFS(
config.MemFS.Filesystem,
)
}
if config.S3FS.Filesystem != nil {
s.v3handler.s3fs = api.NewMemFS(
config.S3FS.Filesystem,
)
s.handler.s3fs = handler.NewMemFS(
config.S3FS.Filesystem,
)
}
if config.Prometheus != nil {
s.handler.prometheus = handler.NewPrometheus(
config.Prometheus.HTTPHandler(),
@@ -300,12 +286,6 @@ func NewServer(config Config) (Server, error) {
Logger: s.logger,
})
if config.Cache != nil {
s.middleware.cache = mwcache.NewWithConfig(mwcache.Config{
Cache: config.Cache,
})
}
s.v3handler.widget = api.NewWidget(api.WidgetConfig{
Restream: config.Restream,
Registry: config.Sessions,
@@ -316,12 +296,7 @@ func NewServer(config Config) (Server, error) {
})
if middleware, err := mwcors.NewWithConfig(mwcors.Config{
Prefixes: map[string][]string{
"/": config.Cors.Origins,
"/api": {"*"},
"/memfs": config.Cors.Origins,
"/s3": config.Cors.Origins,
},
Prefixes: corsPrefixes,
}); err != nil {
return nil, err
} else {
@@ -447,105 +422,48 @@ func (s *server) setRoutes() {
doc.Use(gzipMiddleware)
doc.GET("", echoSwagger.WrapHandler)
// Serve static data
fs := s.router.Group("/*")
// Mount filesystems
for _, filesystem := range s.filesystems {
fs := s.router.Group(filesystem.Mountpoint + "/*")
fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "text/html",
DefaultContentType: filesystem.DefaultContentType,
}))
if filesystem.Gzip {
fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.cache != nil {
fs.Use(s.middleware.cache)
}
fs.GET("", s.handler.diskfs.GetFile)
fs.HEAD("", s.handler.diskfs.GetFile)
// Memory FS
if s.handler.memfs != nil {
memfs := s.router.Group("/memfs/*")
memfs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
memfs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
memfs.Use(s.middleware.session)
if filesystem.Cache != nil {
mwcache := mwcache.NewWithConfig(mwcache.Config{
Cache: filesystem.Cache,
})
fs.Use(mwcache)
}
memfs.HEAD("", s.handler.memfs.GetFile)
memfs.GET("", s.handler.memfs.GetFile)
fs.GET("", filesystem.handler.GetFile)
fs.HEAD("", filesystem.handler.GetFile)
var authmw echo.MiddlewareFunc
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
if len(filesystem.Username) != 0 || len(filesystem.Password) != 0 {
authmw := middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == filesystem.Username && password == filesystem.Password {
return true, nil
}
return false, nil
})
memfs.POST("", s.handler.memfs.PutFile, authmw)
memfs.PUT("", s.handler.memfs.PutFile, authmw)
memfs.DELETE("", s.handler.memfs.DeleteFile, authmw)
fs.POST("", filesystem.handler.PutFile, authmw)
fs.PUT("", filesystem.handler.PutFile, authmw)
fs.DELETE("", filesystem.handler.DeleteFile, authmw)
} else {
memfs.POST("", s.handler.memfs.PutFile)
memfs.PUT("", s.handler.memfs.PutFile)
memfs.DELETE("", s.handler.memfs.DeleteFile)
}
}
// S3 FS
if s.handler.s3fs != nil {
s3fs := s.router.Group("/s3/*")
s3fs.Use(mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
s3fs.Use(mwgzip.NewWithConfig(mwgzip.Config{
Level: mwgzip.BestSpeed,
MinLength: 1000,
ContentTypes: s.gzip.mimetypes,
}))
if s.middleware.session != nil {
s3fs.Use(s.middleware.session)
}
s3fs.HEAD("", s.handler.s3fs.GetFile)
s3fs.GET("", s.handler.s3fs.GetFile)
var authmw echo.MiddlewareFunc
if s.memfs.enableAuth {
authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) {
if username == s.memfs.username && password == s.memfs.password {
return true, nil
}
return false, nil
})
s3fs.POST("", s.handler.s3fs.PutFile, authmw)
s3fs.PUT("", s.handler.s3fs.PutFile, authmw)
s3fs.DELETE("", s.handler.s3fs.DeleteFile, authmw)
} else {
s3fs.POST("", s.handler.s3fs.PutFile)
s3fs.PUT("", s.handler.s3fs.PutFile)
s3fs.DELETE("", s.handler.s3fs.DeleteFile)
}
if s.middleware.cache != nil {
s3fs.Use(s.middleware.cache)
fs.POST("", filesystem.handler.PutFile)
fs.PUT("", filesystem.handler.PutFile)
fs.DELETE("", filesystem.handler.DeleteFile)
}
}
@@ -643,44 +561,33 @@ func (s *server) setRoutesV3(v3 *echo.Group) {
}
}
// v3 Memory FS
if s.v3handler.memfs != nil {
v3.GET("/fs/mem", s.v3handler.memfs.ListFiles)
v3.GET("/fs/mem/*", s.v3handler.memfs.GetFile)
if !s.readOnly {
v3.DELETE("/fs/mem/*", s.v3handler.memfs.DeleteFile)
v3.PUT("/fs/mem/*", s.v3handler.memfs.PutFile)
v3.PATCH("/fs/mem/*", s.v3handler.memfs.PatchFile)
// v3 Filesystems
fshandlers := map[string]api.FSConfig{}
for _, fs := range s.filesystems {
fshandlers[fs.Name] = api.FSConfig{
Type: fs.Filesystem.Type(),
Mountpoint: fs.Mountpoint,
Handler: fs.handler,
}
}
// v3 S3 FS
if s.v3handler.s3fs != nil {
v3.GET("/fs/s3", s.v3handler.s3fs.ListFiles)
v3.GET("/fs/s3/*", s.v3handler.s3fs.GetFile)
handler := api.NewFS(fshandlers)
if !s.readOnly {
v3.DELETE("/fs/s3/*", s.v3handler.s3fs.DeleteFile)
v3.PUT("/fs/s3/*", s.v3handler.s3fs.PutFile)
v3.PATCH("/fs/s3/*", s.v3handler.s3fs.PatchFile)
}
}
v3.GET("/fs", handler.List)
// v3 Disk FS
v3.GET("/fs/disk", s.v3handler.diskfs.ListFiles)
v3.GET("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.GET("/fs/:name", handler.ListFiles)
v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
v3.HEAD("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{
v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{
MimeTypesFile: s.mimeTypesFile,
DefaultContentType: "application/data",
}))
if !s.readOnly {
v3.PUT("/fs/disk/*", s.v3handler.diskfs.PutFile)
v3.DELETE("/fs/disk/*", s.v3handler.diskfs.DeleteFile)
v3.PUT("/fs/:name/*", handler.PutFile)
v3.DELETE("/fs/:name/*", handler.DeleteFile)
}
// v3 RTMP

6
io/fs/disk.go
View File

@@ -135,6 +135,8 @@ func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
fs.logger = log.New("")
}
fs.logger = fs.logger.WithField("type", "disk")
if err := fs.Rebase(config.Dir); err != nil {
return nil, err
}
@@ -172,6 +174,10 @@ func (fs *diskFilesystem) Rebase(base string) error {
return nil
}
func (fs *diskFilesystem) Type() string {
return "diskfs"
}
func (fs *diskFilesystem) Size() (int64, int64) {
// This is to cache the size for some time in order not to
// stress the underlying filesystem too much.

1
io/fs/dummy.go
View File

@@ -24,6 +24,7 @@ type dummyFilesystem struct{}
func (d *dummyFilesystem) Base() string { return "/" }
func (d *dummyFilesystem) Rebase(string) error { return nil }
func (d *dummyFilesystem) Type() string { return "dummy" }
func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 }
func (d *dummyFilesystem) Resize(int64) {}
func (d *dummyFilesystem) Files() int64 { return 0 }

3
io/fs/fs.go
View File

@@ -44,6 +44,9 @@ type Filesystem interface {
// Rebase sets a new base path for this filesystem
Rebase(string) error
// Type returns the type of this filesystem
Type() string
// Size returns the consumed size and capacity of the filesystem in bytes. The
// capacity is negative if the filesystem can consume as much space as it can.
Size() (int64, int64)

6
io/fs/mem.go
View File

@@ -146,6 +146,8 @@ func NewMemFilesystem(config MemConfig) Filesystem {
fs.logger = log.New("")
}
fs.logger = fs.logger.WithField("type", "mem")
fs.files = make(map[string]*memFile)
fs.dataPool = sync.Pool{
@@ -172,6 +174,10 @@ func (fs *memFilesystem) Rebase(base string) error {
return nil
}
func (fs *memFilesystem) Type() string {
return "memfs"
}
func (fs *memFilesystem) Size() (int64, int64) {
fs.filesLock.RLock()
defer fs.filesLock.RUnlock()

5
io/fs/s3.go
View File

@@ -66,6 +66,7 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
}
fs.logger = fs.logger.WithFields(log.Fields{
"type": "s3",
"bucket": fs.bucket,
"region": fs.region,
"endpoint": fs.endpoint,
@@ -107,6 +108,10 @@ func (fs *s3fs) Rebase(base string) error {
return nil
}
func (fs *s3fs) Type() string {
return "s3fs"
}
func (fs *s3fs) Size() (int64, int64) {
size := int64(0)