From cb0bc494f9c5536daab43e6b7f5ff4aec575cee7 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Mon, 22 Aug 2022 19:10:59 +0300 Subject: [PATCH] Refactor internal filesystem handling --- app/api/api.go | 142 ++++---- docs/docs.go | 584 +++++++++----------------------- docs/swagger.json | 584 +++++++++----------------------- docs/swagger.yaml | 397 ++++++---------------- http/api/fs.go | 7 + http/fs/fs.go | 23 ++ http/handler/api/diskfs.go | 207 ----------- http/handler/api/filesystems.go | 146 ++++++++ http/handler/api/memfs.go | 172 ---------- http/handler/diskfs.go | 88 ----- http/handler/filesystem.go | 164 +++++++++ http/handler/memfs.go | 130 ------- http/server.go | 271 +++++---------- io/fs/disk.go | 6 + io/fs/dummy.go | 1 + io/fs/fs.go | 3 + io/fs/mem.go | 6 + io/fs/s3.go | 5 + 18 files changed, 955 insertions(+), 1981 deletions(-) create mode 100644 http/fs/fs.go delete mode 100644 http/handler/api/diskfs.go create mode 100644 http/handler/api/filesystems.go delete mode 100644 http/handler/api/memfs.go delete mode 100644 http/handler/diskfs.go create mode 100644 http/handler/filesystem.go delete mode 100644 http/handler/memfs.go diff --git a/app/api/api.go b/app/api/api.go index b0f6a938..644708d5 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -19,6 +19,7 @@ import ( "github.com/datarhei/core/v16/ffmpeg" "github.com/datarhei/core/v16/http" "github.com/datarhei/core/v16/http/cache" + httpfs "github.com/datarhei/core/v16/http/fs" "github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/io/fs" @@ -60,24 +61,23 @@ type API interface { } type api struct { - restream restream.Restreamer - ffmpeg ffmpeg.FFmpeg - diskfs fs.Filesystem - memfs fs.Filesystem - s3fs fs.Filesystem - rtmpserver rtmp.Server - srtserver srt.Server - metrics monitor.HistoryMonitor - prom prometheus.Metrics - service service.Service - sessions session.Registry - sessionsLimiter net.IPLimiter - cache cache.Cacher - mainserver *gohttp.Server - sidecarserver *gohttp.Server - httpjwt jwt.JWT - update update.Checker - replacer replace.Replacer + restream restream.Restreamer + ffmpeg ffmpeg.FFmpeg + diskfs fs.Filesystem + memfs fs.Filesystem + s3fs fs.Filesystem + rtmpserver rtmp.Server + srtserver srt.Server + metrics monitor.HistoryMonitor + prom prometheus.Metrics + service service.Service + sessions session.Registry + cache cache.Cacher + mainserver *gohttp.Server + sidecarserver *gohttp.Server + httpjwt jwt.JWT + update update.Checker + replacer replace.Replacer errorChan chan error @@ -372,7 +372,7 @@ func (a *api) start() error { diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{ Dir: cfg.Storage.Disk.Dir, Size: cfg.Storage.Disk.Size * 1024 * 1024, - Logger: a.log.logger.core.WithComponent("DiskFS"), + Logger: a.log.logger.core.WithComponent("FS"), }) if err != nil { return err @@ -401,7 +401,7 @@ func (a *api) start() error { Base: baseMemFS.String(), Size: cfg.Storage.Memory.Size * 1024 * 1024, Purge: cfg.Storage.Memory.Purge, - Logger: a.log.logger.core.WithComponent("MemFS"), + Logger: a.log.logger.core.WithComponent("FS"), }) a.memfs = memfs @@ -435,7 +435,7 @@ func (a *api) start() error { Region: cfg.Storage.S3.Region, Bucket: cfg.Storage.S3.Bucket, UseSSL: cfg.Storage.S3.UseSSL, - Logger: a.log.logger.core.WithComponent("S3"), + Logger: a.log.logger.core.WithComponent("FS"), }) if err != nil { return err @@ -665,7 +665,7 @@ func (a *api) start() error { } if cfg.Storage.Disk.Cache.Enable { - diskCache, err := cache.NewLRUCache(cache.LRUConfig{ + cache, err := cache.NewLRUCache(cache.LRUConfig{ TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, @@ -675,10 +675,10 @@ func (a *api) start() error { }) if err != nil { - return fmt.Errorf("unable to create disk cache: %w", err) + return fmt.Errorf("unable to create cache: %w", err) } - a.cache = diskCache + a.cache = cache } var autocertManager *autocert.Manager @@ -829,25 +829,50 @@ func (a *api) start() error { a.log.logger.main = a.log.logger.core.WithComponent(logcontext).WithField("address", cfg.Address) - mainserverhandler, err := http.NewServer(http.Config{ + serverConfig := http.Config{ Logger: a.log.logger.main, LogBuffer: a.log.buffer, Restream: a.restream, Metrics: a.metrics, Prometheus: a.prom, MimeTypesFile: cfg.Storage.MimeTypes, - DiskFS: a.diskfs, - MemFS: http.MemFSConfig{ - EnableAuth: cfg.Storage.Memory.Auth.Enable, - Username: cfg.Storage.Memory.Auth.Username, - Password: cfg.Storage.Memory.Auth.Password, - Filesystem: a.memfs, - }, - S3FS: http.MemFSConfig{ - EnableAuth: cfg.Storage.S3.Auth.Enable, - Username: cfg.Storage.S3.Auth.Username, - Password: cfg.Storage.S3.Auth.Password, - Filesystem: a.s3fs, + Filesystems: []httpfs.FS{ + { + Name: "diskfs", + Mountpoint: "/", + AllowWrite: false, + Username: "", + Password: "", + DefaultFile: "index.html", + DefaultContentType: "text/html", + Gzip: true, + Filesystem: diskfs, + Cache: a.cache, + }, + { + Name: "memfs", + Mountpoint: "/memfs", + AllowWrite: cfg.Storage.Memory.Auth.Enable, + Username: cfg.Storage.Memory.Auth.Username, + Password: cfg.Storage.Memory.Auth.Password, + DefaultFile: "", + DefaultContentType: "application/data", + Gzip: true, + Filesystem: a.memfs, + Cache: a.cache, + }, + { + Name: "s3fs", + Mountpoint: "/s3", + AllowWrite: cfg.Storage.S3.Auth.Enable, + Username: cfg.Storage.S3.Auth.Username, + Password: cfg.Storage.S3.Auth.Password, + DefaultFile: "", + DefaultContentType: "application/data", + Gzip: true, + Filesystem: a.s3fs, + Cache: a.cache, + }, }, IPLimiter: iplimiter, Profiling: cfg.Debug.Profiling, @@ -858,11 +883,12 @@ func (a *api) start() error { SRT: a.srtserver, JWT: a.httpjwt, Config: a.config.store, - Cache: a.cache, Sessions: a.sessions, Router: router, ReadOnly: cfg.API.ReadOnly, - }) + } + + mainserverhandler, err := http.NewServer(serverConfig) if err != nil { return fmt.Errorf("unable to create server: %w", err) @@ -897,40 +923,10 @@ func (a *api) start() error { a.log.logger.sidecar = a.log.logger.core.WithComponent("HTTP").WithField("address", cfg.Address) - sidecarserverhandler, err := http.NewServer(http.Config{ - Logger: a.log.logger.sidecar, - LogBuffer: a.log.buffer, - Restream: a.restream, - Metrics: a.metrics, - Prometheus: a.prom, - MimeTypesFile: cfg.Storage.MimeTypes, - DiskFS: a.diskfs, - MemFS: http.MemFSConfig{ - EnableAuth: cfg.Storage.Memory.Auth.Enable, - Username: cfg.Storage.Memory.Auth.Username, - Password: cfg.Storage.Memory.Auth.Password, - Filesystem: a.memfs, - }, - S3FS: http.MemFSConfig{ - EnableAuth: cfg.Storage.S3.Auth.Enable, - Username: cfg.Storage.S3.Auth.Username, - Password: cfg.Storage.S3.Auth.Password, - Filesystem: a.s3fs, - }, - IPLimiter: iplimiter, - Profiling: cfg.Debug.Profiling, - Cors: http.CorsConfig{ - Origins: cfg.Storage.CORS.Origins, - }, - RTMP: a.rtmpserver, - SRT: a.srtserver, - JWT: a.httpjwt, - Config: a.config.store, - Cache: a.cache, - Sessions: a.sessions, - Router: router, - ReadOnly: cfg.API.ReadOnly, - }) + serverConfig.Logger = a.log.logger.sidecar + serverConfig.IPLimiter = iplimiter + + sidecarserverhandler, err := http.NewServer(serverConfig) if err != nil { return fmt.Errorf("unable to create sidecar HTTP server: %w", err) diff --git a/docs/docs.go b/docs/docs.go index 9baa93ac..7781fd89 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -302,20 +302,53 @@ const docTemplate = `{ } } }, - "/api/v3/fs/disk": { + "/api/v3/fs": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", + "description": "Listall registered filesystems", "produces": [ "application/json" ], - "summary": "List all files on the filesystem", - "operationId": "diskfs-3-list-files", + "summary": "List all registered filesystems", + "operationId": "filesystem-3-list", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.FilesystemInfo" + } + } + } + } + } + }, + "/api/v3/fs/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", + "produces": [ + "application/json" + ], + "summary": "List all files on a filesystem", + "operationId": "filesystem-3-list-files", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "glob pattern for file names", @@ -348,21 +381,28 @@ const docTemplate = `{ } } }, - "/api/v3/fs/disk/{path}": { + "/api/v3/fs/{name}/{path}": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Fetch a file from the filesystem. The contents of that file are returned.", + "description": "Fetch a file from a filesystem", "produces": [ "application/data", "application/json" ], - "summary": "Fetch a file from the filesystem", - "operationId": "diskfs-3-get-file", + "summary": "Fetch a file from a filesystem", + "operationId": "filesystem-3-get-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -398,7 +438,7 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Writes or overwrites a file on the filesystem", + "description": "Writes or overwrites a file on a filesystem", "consumes": [ "application/data" ], @@ -406,9 +446,16 @@ const docTemplate = `{ "text/plain", "application/json" ], - "summary": "Add a file to the filesystem", - "operationId": "diskfs-3-put-file", + "summary": "Add a file to a filesystem", + "operationId": "filesystem-3-put-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -456,13 +503,20 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Remove a file from the filesystem", + "description": "Remove a file from a filesystem", "produces": [ "text/plain" ], - "summary": "Remove a file from the filesystem", - "operationId": "diskfs-3-delete-file", + "summary": "Remove a file from a filesystem", + "operationId": "filesystem-3-delete-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -487,240 +541,6 @@ const docTemplate = `{ } } }, - "/api/v3/fs/mem": { - "get": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", - "produces": [ - "application/json" - ], - "summary": "List all files on the memory filesystem", - "operationId": "memfs-3-list-files", - "parameters": [ - { - "type": "string", - "description": "glob pattern for file names", - "name": "glob", - "in": "query" - }, - { - "type": "string", - "description": "none, name, size, lastmod", - "name": "sort", - "in": "query" - }, - { - "type": "string", - "description": "asc, desc", - "name": "order", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/api.FileInfo" - } - } - } - } - } - }, - "/api/v3/fs/mem/{path}": { - "get": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Fetch a file from the memory filesystem", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the memory filesystem", - "operationId": "memfs-3-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "put": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Writes or overwrites a file on the memory filesystem", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Add a file to the memory filesystem", - "operationId": "memfs-3-put-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "File data", - "name": "data", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer" - } - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "204": { - "description": "No Content", - "schema": { - "type": "string" - } - }, - "507": { - "description": "Insufficient Storage", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "delete": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Remove a file from the memory filesystem", - "produces": [ - "text/plain" - ], - "summary": "Remove a file from the memory filesystem", - "operationId": "memfs-3-delete-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "patch": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Create a link to a file in the memory filesystem. The file linked to has to exist.", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Create a link to a file in the memory filesystem", - "operationId": "memfs-3-patch", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "Path to the file to link to", - "name": "url", - "in": "body", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } - }, "/api/v3/log": { "get": { "security": [ @@ -1982,140 +1802,6 @@ const docTemplate = `{ } } }, - "/memfs/{path}": { - "get": { - "description": "Fetch a file from the memory filesystem", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the memory filesystem", - "operationId": "memfs-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "put": { - "security": [ - { - "BasicAuth": [] - } - ], - "description": "Writes or overwrites a file on the memory filesystem", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Add a file to the memory filesystem", - "operationId": "memfs-put-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "File data", - "name": "data", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer" - } - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "204": { - "description": "No Content", - "schema": { - "type": "string" - } - }, - "507": { - "description": "Insufficient Storage", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "delete": { - "security": [ - { - "BasicAuth": [] - } - ], - "description": "Remove a file from the memory filesystem", - "produces": [ - "text/plain" - ], - "summary": "Remove a file from the memory filesystem", - "operationId": "memfs-delete-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } - }, "/metrics": { "get": { "description": "Prometheus metrics", @@ -2175,46 +1861,6 @@ const docTemplate = `{ } } } - }, - "/{path}": { - "get": { - "description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the filesystem", - "operationId": "diskfs-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } } }, "definitions": { @@ -2798,6 +2444,46 @@ const docTemplate = `{ }, "mimetypes_file": { "type": "string" + }, + "s3": { + "type": "object", + "properties": { + "access_key_id": { + "type": "string" + }, + "auth": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "bucket": { + "type": "string" + }, + "enable": { + "type": "boolean" + }, + "endpoint": { + "type": "string" + }, + "region": { + "type": "string" + }, + "secret_access_key": { + "type": "string" + }, + "use_ssl": { + "type": "boolean" + } + } } } }, @@ -2869,6 +2555,20 @@ const docTemplate = `{ } } }, + "api.FilesystemInfo": { + "type": "object", + "properties": { + "mount": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, "api.GraphQuery": { "type": "object", "properties": { @@ -4406,6 +4106,46 @@ const docTemplate = `{ }, "mimetypes_file": { "type": "string" + }, + "s3": { + "type": "object", + "properties": { + "access_key_id": { + "type": "string" + }, + "auth": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "bucket": { + "type": "string" + }, + "enable": { + "type": "boolean" + }, + "endpoint": { + "type": "string" + }, + "region": { + "type": "string" + }, + "secret_access_key": { + "type": "string" + }, + "use_ssl": { + "type": "boolean" + } + } } } }, diff --git a/docs/swagger.json b/docs/swagger.json index 35c2db46..893746a5 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -294,20 +294,53 @@ } } }, - "/api/v3/fs/disk": { + "/api/v3/fs": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", + "description": "Listall registered filesystems", "produces": [ "application/json" ], - "summary": "List all files on the filesystem", - "operationId": "diskfs-3-list-files", + "summary": "List all registered filesystems", + "operationId": "filesystem-3-list", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/api.FilesystemInfo" + } + } + } + } + } + }, + "/api/v3/fs/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", + "produces": [ + "application/json" + ], + "summary": "List all files on a filesystem", + "operationId": "filesystem-3-list-files", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "glob pattern for file names", @@ -340,21 +373,28 @@ } } }, - "/api/v3/fs/disk/{path}": { + "/api/v3/fs/{name}/{path}": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Fetch a file from the filesystem. The contents of that file are returned.", + "description": "Fetch a file from a filesystem", "produces": [ "application/data", "application/json" ], - "summary": "Fetch a file from the filesystem", - "operationId": "diskfs-3-get-file", + "summary": "Fetch a file from a filesystem", + "operationId": "filesystem-3-get-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -390,7 +430,7 @@ "ApiKeyAuth": [] } ], - "description": "Writes or overwrites a file on the filesystem", + "description": "Writes or overwrites a file on a filesystem", "consumes": [ "application/data" ], @@ -398,9 +438,16 @@ "text/plain", "application/json" ], - "summary": "Add a file to the filesystem", - "operationId": "diskfs-3-put-file", + "summary": "Add a file to a filesystem", + "operationId": "filesystem-3-put-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -448,13 +495,20 @@ "ApiKeyAuth": [] } ], - "description": "Remove a file from the filesystem", + "description": "Remove a file from a filesystem", "produces": [ "text/plain" ], - "summary": "Remove a file from the filesystem", - "operationId": "diskfs-3-delete-file", + "summary": "Remove a file from a filesystem", + "operationId": "filesystem-3-delete-file", "parameters": [ + { + "type": "string", + "description": "Name of the filesystem", + "name": "name", + "in": "path", + "required": true + }, { "type": "string", "description": "Path to file", @@ -479,240 +533,6 @@ } } }, - "/api/v3/fs/mem": { - "get": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.", - "produces": [ - "application/json" - ], - "summary": "List all files on the memory filesystem", - "operationId": "memfs-3-list-files", - "parameters": [ - { - "type": "string", - "description": "glob pattern for file names", - "name": "glob", - "in": "query" - }, - { - "type": "string", - "description": "none, name, size, lastmod", - "name": "sort", - "in": "query" - }, - { - "type": "string", - "description": "asc, desc", - "name": "order", - "in": "query" - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/api.FileInfo" - } - } - } - } - } - }, - "/api/v3/fs/mem/{path}": { - "get": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Fetch a file from the memory filesystem", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the memory filesystem", - "operationId": "memfs-3-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "put": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Writes or overwrites a file on the memory filesystem", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Add a file to the memory filesystem", - "operationId": "memfs-3-put-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "File data", - "name": "data", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer" - } - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "204": { - "description": "No Content", - "schema": { - "type": "string" - } - }, - "507": { - "description": "Insufficient Storage", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "delete": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Remove a file from the memory filesystem", - "produces": [ - "text/plain" - ], - "summary": "Remove a file from the memory filesystem", - "operationId": "memfs-3-delete-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "patch": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Create a link to a file in the memory filesystem. The file linked to has to exist.", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Create a link to a file in the memory filesystem", - "operationId": "memfs-3-patch", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "Path to the file to link to", - "name": "url", - "in": "body", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } - }, "/api/v3/log": { "get": { "security": [ @@ -1974,140 +1794,6 @@ } } }, - "/memfs/{path}": { - "get": { - "description": "Fetch a file from the memory filesystem", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the memory filesystem", - "operationId": "memfs-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "put": { - "security": [ - { - "BasicAuth": [] - } - ], - "description": "Writes or overwrites a file on the memory filesystem", - "consumes": [ - "application/data" - ], - "produces": [ - "text/plain", - "application/json" - ], - "summary": "Add a file to the memory filesystem", - "operationId": "memfs-put-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - }, - { - "description": "File data", - "name": "data", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer" - } - } - } - ], - "responses": { - "201": { - "description": "Created", - "schema": { - "type": "string" - } - }, - "204": { - "description": "No Content", - "schema": { - "type": "string" - } - }, - "507": { - "description": "Insufficient Storage", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - }, - "delete": { - "security": [ - { - "BasicAuth": [] - } - ], - "description": "Remove a file from the memory filesystem", - "produces": [ - "text/plain" - ], - "summary": "Remove a file from the memory filesystem", - "operationId": "memfs-delete-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } - }, "/metrics": { "get": { "description": "Prometheus metrics", @@ -2167,46 +1853,6 @@ } } } - }, - "/{path}": { - "get": { - "description": "Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists.", - "produces": [ - "application/data", - "application/json" - ], - "summary": "Fetch a file from the filesystem", - "operationId": "diskfs-get-file", - "parameters": [ - { - "type": "string", - "description": "Path to file", - "name": "path", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "file" - } - }, - "301": { - "description": "Moved Permanently", - "schema": { - "type": "string" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/api.Error" - } - } - } - } } }, "definitions": { @@ -2790,6 +2436,46 @@ }, "mimetypes_file": { "type": "string" + }, + "s3": { + "type": "object", + "properties": { + "access_key_id": { + "type": "string" + }, + "auth": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "bucket": { + "type": "string" + }, + "enable": { + "type": "boolean" + }, + "endpoint": { + "type": "string" + }, + "region": { + "type": "string" + }, + "secret_access_key": { + "type": "string" + }, + "use_ssl": { + "type": "boolean" + } + } } } }, @@ -2861,6 +2547,20 @@ } } }, + "api.FilesystemInfo": { + "type": "object", + "properties": { + "mount": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, "api.GraphQuery": { "type": "object", "properties": { @@ -4398,6 +4098,46 @@ }, "mimetypes_file": { "type": "string" + }, + "s3": { + "type": "object", + "properties": { + "access_key_id": { + "type": "string" + }, + "auth": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "bucket": { + "type": "string" + }, + "enable": { + "type": "boolean" + }, + "endpoint": { + "type": "string" + }, + "region": { + "type": "string" + }, + "secret_access_key": { + "type": "string" + }, + "use_ssl": { + "type": "boolean" + } + } } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c4990b87..8211aab0 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -379,6 +379,32 @@ definitions: type: object mimetypes_file: type: string + s3: + properties: + access_key_id: + type: string + auth: + properties: + enable: + type: boolean + password: + type: string + username: + type: string + type: object + bucket: + type: string + enable: + type: boolean + endpoint: + type: string + region: + type: string + secret_access_key: + type: string + use_ssl: + type: boolean + type: object type: object tls: properties: @@ -424,6 +450,15 @@ definitions: size_bytes: type: integer type: object + api.FilesystemInfo: + properties: + mount: + type: string + name: + type: string + type: + type: string + type: object api.GraphQuery: properties: query: @@ -1491,6 +1526,32 @@ definitions: type: object mimetypes_file: type: string + s3: + properties: + access_key_id: + type: string + auth: + properties: + enable: + type: boolean + password: + type: string + username: + type: string + type: object + bucket: + type: string + enable: + type: boolean + endpoint: + type: string + region: + type: string + secret_access_key: + type: string + use_ssl: + type: boolean + type: object type: object tls: properties: @@ -1705,34 +1766,6 @@ info: title: datarhei Core API version: "3.0" paths: - /{path}: - get: - description: Fetch a file from the filesystem. If the file is a directory, a - index.html is returned, if it exists. - operationId: diskfs-get-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - produces: - - application/data - - application/json - responses: - "200": - description: OK - schema: - type: file - "301": - description: Moved Permanently - schema: - type: string - "404": - description: Not Found - schema: - $ref: '#/definitions/api.Error' - summary: Fetch a file from the filesystem /api: get: description: API version and build infos in case auth is valid or not required. @@ -1911,12 +1944,33 @@ paths: security: - ApiKeyAuth: [] summary: Reload the currently active configuration - /api/v3/fs/disk: + /api/v3/fs: get: - description: List all files on the filesystem. The listing can be ordered by - name, size, or date of last modification in ascending or descending order. - operationId: diskfs-3-list-files + description: Listall registered filesystems + operationId: filesystem-3-list + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/api.FilesystemInfo' + type: array + security: + - ApiKeyAuth: [] + summary: List all registered filesystems + /api/v3/fs/{name}: + get: + description: List all files on a filesystem. The listing can be ordered by name, + size, or date of last modification in ascending or descending order. + operationId: filesystem-3-list-files parameters: + - description: Name of the filesystem + in: path + name: name + required: true + type: string - description: glob pattern for file names in: query name: glob @@ -1940,12 +1994,17 @@ paths: type: array security: - ApiKeyAuth: [] - summary: List all files on the filesystem - /api/v3/fs/disk/{path}: + summary: List all files on a filesystem + /api/v3/fs/{name}/{path}: delete: - description: Remove a file from the filesystem - operationId: diskfs-3-delete-file + description: Remove a file from a filesystem + operationId: filesystem-3-delete-file parameters: + - description: Name of the filesystem + in: path + name: name + required: true + type: string - description: Path to file in: path name: path @@ -1964,12 +2023,16 @@ paths: $ref: '#/definitions/api.Error' security: - ApiKeyAuth: [] - summary: Remove a file from the filesystem + summary: Remove a file from a filesystem get: - description: Fetch a file from the filesystem. The contents of that file are - returned. - operationId: diskfs-3-get-file + description: Fetch a file from a filesystem + operationId: filesystem-3-get-file parameters: + - description: Name of the filesystem + in: path + name: name + required: true + type: string - description: Path to file in: path name: path @@ -1993,13 +2056,18 @@ paths: $ref: '#/definitions/api.Error' security: - ApiKeyAuth: [] - summary: Fetch a file from the filesystem + summary: Fetch a file from a filesystem put: consumes: - application/data - description: Writes or overwrites a file on the filesystem - operationId: diskfs-3-put-file + description: Writes or overwrites a file on a filesystem + operationId: filesystem-3-put-file parameters: + - description: Name of the filesystem + in: path + name: name + required: true + type: string - description: Path to file in: path name: path @@ -2031,160 +2099,7 @@ paths: $ref: '#/definitions/api.Error' security: - ApiKeyAuth: [] - summary: Add a file to the filesystem - /api/v3/fs/mem: - get: - description: List all files on the memory filesystem. The listing can be ordered - by name, size, or date of last modification in ascending or descending order. - operationId: memfs-3-list-files - parameters: - - description: glob pattern for file names - in: query - name: glob - type: string - - description: none, name, size, lastmod - in: query - name: sort - type: string - - description: asc, desc - in: query - name: order - type: string - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/api.FileInfo' - type: array - security: - - ApiKeyAuth: [] - summary: List all files on the memory filesystem - /api/v3/fs/mem/{path}: - delete: - description: Remove a file from the memory filesystem - operationId: memfs-3-delete-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - produces: - - text/plain - responses: - "200": - description: OK - schema: - type: string - "404": - description: Not Found - schema: - $ref: '#/definitions/api.Error' - security: - - ApiKeyAuth: [] - summary: Remove a file from the memory filesystem - get: - description: Fetch a file from the memory filesystem - operationId: memfs-3-get-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - produces: - - application/data - - application/json - responses: - "200": - description: OK - schema: - type: file - "301": - description: Moved Permanently - schema: - type: string - "404": - description: Not Found - schema: - $ref: '#/definitions/api.Error' - security: - - ApiKeyAuth: [] - summary: Fetch a file from the memory filesystem - patch: - consumes: - - application/data - description: Create a link to a file in the memory filesystem. The file linked - to has to exist. - operationId: memfs-3-patch - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - - description: Path to the file to link to - in: body - name: url - required: true - schema: - type: string - produces: - - text/plain - - application/json - responses: - "201": - description: Created - schema: - type: string - "400": - description: Bad Request - schema: - $ref: '#/definitions/api.Error' - security: - - ApiKeyAuth: [] - summary: Create a link to a file in the memory filesystem - put: - consumes: - - application/data - description: Writes or overwrites a file on the memory filesystem - operationId: memfs-3-put-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - - description: File data - in: body - name: data - required: true - schema: - items: - type: integer - type: array - produces: - - text/plain - - application/json - responses: - "201": - description: Created - schema: - type: string - "204": - description: No Content - schema: - type: string - "507": - description: Insufficient Storage - schema: - $ref: '#/definitions/api.Error' - security: - - ApiKeyAuth: [] - summary: Add a file to the memory filesystem + summary: Add a file to a filesystem /api/v3/log: get: description: Get the last log lines of the Restreamer application @@ -3019,94 +2934,6 @@ paths: schema: $ref: '#/definitions/api.Error' summary: Fetch minimal statistics about a process - /memfs/{path}: - delete: - description: Remove a file from the memory filesystem - operationId: memfs-delete-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - produces: - - text/plain - responses: - "200": - description: OK - schema: - type: string - "404": - description: Not Found - schema: - $ref: '#/definitions/api.Error' - security: - - BasicAuth: [] - summary: Remove a file from the memory filesystem - get: - description: Fetch a file from the memory filesystem - operationId: memfs-get-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - produces: - - application/data - - application/json - responses: - "200": - description: OK - schema: - type: file - "301": - description: Moved Permanently - schema: - type: string - "404": - description: Not Found - schema: - $ref: '#/definitions/api.Error' - summary: Fetch a file from the memory filesystem - put: - consumes: - - application/data - description: Writes or overwrites a file on the memory filesystem - operationId: memfs-put-file - parameters: - - description: Path to file - in: path - name: path - required: true - type: string - - description: File data - in: body - name: data - required: true - schema: - items: - type: integer - type: array - produces: - - text/plain - - application/json - responses: - "201": - description: Created - schema: - type: string - "204": - description: No Content - schema: - type: string - "507": - description: Insufficient Storage - schema: - $ref: '#/definitions/api.Error' - security: - - BasicAuth: [] - summary: Add a file to the memory filesystem /metrics: get: description: Prometheus metrics diff --git a/http/api/fs.go b/http/api/fs.go index c7d12eb4..7c0b1ce9 100644 --- a/http/api/fs.go +++ b/http/api/fs.go @@ -6,3 +6,10 @@ type FileInfo struct { Size int64 `json:"size_bytes" jsonschema:"minimum=0"` LastMod int64 `json:"last_modified" jsonschema:"minimum=0"` } + +// FilesystemInfo represents information about a filesystem +type FilesystemInfo struct { + Name string `json:"name"` + Type string `json:"type"` + Mount string `json:"mount"` +} diff --git a/http/fs/fs.go b/http/fs/fs.go new file mode 100644 index 00000000..0e5eed7c --- /dev/null +++ b/http/fs/fs.go @@ -0,0 +1,23 @@ +package fs + +import ( + "github.com/datarhei/core/v16/http/cache" + "github.com/datarhei/core/v16/io/fs" +) + +type FS struct { + Name string + Mountpoint string + + AllowWrite bool + Username string + Password string + + DefaultFile string + DefaultContentType string + Gzip bool + + Filesystem fs.Filesystem + + Cache cache.Cacher +} diff --git a/http/handler/api/diskfs.go b/http/handler/api/diskfs.go deleted file mode 100644 index c143619d..00000000 --- a/http/handler/api/diskfs.go +++ /dev/null @@ -1,207 +0,0 @@ -package api - -import ( - "net/http" - "path/filepath" - "sort" - - "github.com/datarhei/core/v16/http/api" - "github.com/datarhei/core/v16/http/cache" - "github.com/datarhei/core/v16/http/handler" - "github.com/datarhei/core/v16/http/handler/util" - "github.com/datarhei/core/v16/io/fs" - - "github.com/labstack/echo/v4" -) - -// The DiskFSHandler type provides handlers for manipulating a filesystem -type DiskFSHandler struct { - cache cache.Cacher - filesystem fs.Filesystem - handler *handler.DiskFSHandler -} - -// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally -// a Cacher where files will be purged from if the Cacher is related to the filesystem. -func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler { - return &DiskFSHandler{ - cache: cache, - filesystem: fs, - handler: handler.NewDiskFS(fs, cache), - } -} - -// GetFile returns the file at the given path -// @Summary Fetch a file from the filesystem -// @Description Fetch a file from the filesystem. The contents of that file are returned. -// @ID diskfs-3-get-file -// @Produce application/data -// @Produce json -// @Param path path string true "Path to file" -// @Success 200 {file} byte -// @Success 301 {string} string -// @Failure 404 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/disk/{path} [get] -func (h *DiskFSHandler) GetFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - mimeType := c.Response().Header().Get(echo.HeaderContentType) - c.Response().Header().Del(echo.HeaderContentType) - - file := h.filesystem.Open(path) - if file == nil { - return api.Err(http.StatusNotFound, "File not found", path) - } - - stat, _ := file.Stat() - - if stat.IsDir() { - return api.Err(http.StatusNotFound, "File not found", path) - } - - defer file.Close() - - c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) - - if path, ok := stat.IsLink(); ok { - path = filepath.Clean("/" + path) - - if path[0] == '/' { - path = path[1:] - } - - return c.Redirect(http.StatusMovedPermanently, path) - } - - c.Response().Header().Set(echo.HeaderContentType, mimeType) - - if c.Request().Method == "HEAD" { - return c.Blob(http.StatusOK, "application/data", nil) - } - - return c.Stream(http.StatusOK, "application/data", file) -} - -// PutFile adds or overwrites a file at the given path -// @Summary Add a file to the filesystem -// @Description Writes or overwrites a file on the filesystem -// @ID diskfs-3-put-file -// @Accept application/data -// @Produce text/plain -// @Produce json -// @Param path path string true "Path to file" -// @Param data body []byte true "File data" -// @Success 201 {string} string -// @Success 204 {string} string -// @Failure 507 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/disk/{path} [put] -func (h *DiskFSHandler) PutFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - c.Response().Header().Del(echo.HeaderContentType) - - req := c.Request() - - _, created, err := h.filesystem.Store(path, req.Body) - if err != nil { - return api.Err(http.StatusBadRequest, "%s", err) - } - - if h.cache != nil { - h.cache.Delete(path) - } - - c.Response().Header().Set("Content-Location", req.URL.RequestURI()) - - if created { - return c.String(http.StatusCreated, path) - } - - return c.NoContent(http.StatusNoContent) -} - -// DeleteFile removes a file from the filesystem -// @Summary Remove a file from the filesystem -// @Description Remove a file from the filesystem -// @ID diskfs-3-delete-file -// @Produce text/plain -// @Param path path string true "Path to file" -// @Success 200 {string} string -// @Failure 404 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/disk/{path} [delete] -func (h *DiskFSHandler) DeleteFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - c.Response().Header().Del(echo.HeaderContentType) - - size := h.filesystem.Delete(path) - - if size < 0 { - return api.Err(http.StatusNotFound, "File not found", path) - } - - if h.cache != nil { - h.cache.Delete(path) - } - - return c.String(http.StatusOK, "OK") -} - -// ListFiles lists all files on the filesystem -// @Summary List all files on the filesystem -// @Description List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. -// @ID diskfs-3-list-files -// @Produce json -// @Param glob query string false "glob pattern for file names" -// @Param sort query string false "none, name, size, lastmod" -// @Param order query string false "asc, desc" -// @Success 200 {array} api.FileInfo -// @Security ApiKeyAuth -// @Router /api/v3/fs/disk [get] -func (h *DiskFSHandler) ListFiles(c echo.Context) error { - pattern := util.DefaultQuery(c, "glob", "") - sortby := util.DefaultQuery(c, "sort", "none") - order := util.DefaultQuery(c, "order", "asc") - - files := h.filesystem.List(pattern) - - var sortFunc func(i, j int) bool - - switch sortby { - case "name": - if order == "desc" { - sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() } - } else { - sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() } - } - case "size": - if order == "desc" { - sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() } - } else { - sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() } - } - default: - if order == "asc" { - sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) } - } else { - sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) } - } - } - - sort.Slice(files, sortFunc) - - var fileinfos []api.FileInfo = make([]api.FileInfo, len(files)) - - for i, f := range files { - fileinfos[i] = api.FileInfo{ - Name: f.Name(), - Size: f.Size(), - LastMod: f.ModTime().Unix(), - } - } - - return c.JSON(http.StatusOK, fileinfos) -} diff --git a/http/handler/api/filesystems.go b/http/handler/api/filesystems.go new file mode 100644 index 00000000..ce93812b --- /dev/null +++ b/http/handler/api/filesystems.go @@ -0,0 +1,146 @@ +package api + +import ( + "net/http" + + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/http/handler" + "github.com/datarhei/core/v16/http/handler/util" + + "github.com/labstack/echo/v4" +) + +type FSConfig struct { + Type string + Mountpoint string + Handler *handler.FSHandler +} + +// The FSHandler type provides handlers for manipulating a filesystem +type FSHandler struct { + filesystems map[string]FSConfig +} + +// NewFS return a new FSHanlder type. You have to provide a filesystem to act on. +func NewFS(filesystems map[string]FSConfig) *FSHandler { + return &FSHandler{ + filesystems: filesystems, + } +} + +// GetFileAPI returns the file at the given path +// @Summary Fetch a file from a filesystem +// @Description Fetch a file from a filesystem +// @ID filesystem-3-get-file +// @Produce application/data +// @Produce json +// @Param name path string true "Name of the filesystem" +// @Param path path string true "Path to file" +// @Success 200 {file} byte +// @Success 301 {string} string +// @Failure 404 {object} api.Error +// @Security ApiKeyAuth +// @Router /api/v3/fs/{name}/{path} [get] +func (h *FSHandler) GetFile(c echo.Context) error { + name := util.PathParam(c, "name") + + config, ok := h.filesystems[name] + if !ok { + return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name) + } + + return config.Handler.GetFile(c) +} + +// PutFileAPI adds or overwrites a file at the given path +// @Summary Add a file to a filesystem +// @Description Writes or overwrites a file on a filesystem +// @ID filesystem-3-put-file +// @Accept application/data +// @Produce text/plain +// @Produce json +// @Param name path string true "Name of the filesystem" +// @Param path path string true "Path to file" +// @Param data body []byte true "File data" +// @Success 201 {string} string +// @Success 204 {string} string +// @Failure 507 {object} api.Error +// @Security ApiKeyAuth +// @Router /api/v3/fs/{name}/{path} [put] +func (h *FSHandler) PutFile(c echo.Context) error { + name := util.PathParam(c, "name") + + config, ok := h.filesystems[name] + if !ok { + return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name) + } + + return config.Handler.PutFile(c) +} + +// DeleteFileAPI removes a file from a filesystem +// @Summary Remove a file from a filesystem +// @Description Remove a file from a filesystem +// @ID filesystem-3-delete-file +// @Produce text/plain +// @Param name path string true "Name of the filesystem" +// @Param path path string true "Path to file" +// @Success 200 {string} string +// @Failure 404 {object} api.Error +// @Security ApiKeyAuth +// @Router /api/v3/fs/{name}/{path} [delete] +func (h *FSHandler) DeleteFile(c echo.Context) error { + name := util.PathParam(c, "name") + + config, ok := h.filesystems[name] + if !ok { + return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name) + } + + return config.Handler.DeleteFile(c) +} + +// ListFiles lists all files on a filesystem +// @Summary List all files on a filesystem +// @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. +// @ID filesystem-3-list-files +// @Produce json +// @Param name path string true "Name of the filesystem" +// @Param glob query string false "glob pattern for file names" +// @Param sort query string false "none, name, size, lastmod" +// @Param order query string false "asc, desc" +// @Success 200 {array} api.FileInfo +// @Security ApiKeyAuth +// @Router /api/v3/fs/{name} [get] +func (h *FSHandler) ListFiles(c echo.Context) error { + name := util.PathParam(c, "name") + + config, ok := h.filesystems[name] + if !ok { + return api.Err(http.StatusNotFound, "File not found", "unknown filesystem: %s", name) + } + + return config.Handler.ListFiles(c) +} + +// List lists all registered filesystems +// @Summary List all registered filesystems +// @Description Listall registered filesystems +// @ID filesystem-3-list +// @Produce json +// @Success 200 {array} api.FilesystemInfo +// @Security ApiKeyAuth +// @Router /api/v3/fs [get] +func (h *FSHandler) List(c echo.Context) error { + fss := []api.FilesystemInfo{} + + for name, config := range h.filesystems { + fss = append(fss, api.FilesystemInfo{ + Name: name, + Type: config.Type, + Mount: config.Mountpoint, + }) + } + + return c.JSON(http.StatusOK, fss) +} diff --git a/http/handler/api/memfs.go b/http/handler/api/memfs.go deleted file mode 100644 index 2c6bd101..00000000 --- a/http/handler/api/memfs.go +++ /dev/null @@ -1,172 +0,0 @@ -package api - -import ( - "io" - "net/http" - "net/url" - "sort" - - "github.com/datarhei/core/v16/http/api" - "github.com/datarhei/core/v16/http/handler" - "github.com/datarhei/core/v16/http/handler/util" - "github.com/datarhei/core/v16/io/fs" - - "github.com/labstack/echo/v4" -) - -// The MemFSHandler type provides handlers for manipulating a filesystem -type MemFSHandler struct { - filesystem fs.Filesystem - handler *handler.MemFSHandler -} - -// NewMemFS return a new MemFS type. You have to provide a filesystem to act on. -func NewMemFS(fs fs.Filesystem) *MemFSHandler { - return &MemFSHandler{ - filesystem: fs, - handler: handler.NewMemFS(fs), - } -} - -// GetFileAPI returns the file at the given path -// @Summary Fetch a file from the memory filesystem -// @Description Fetch a file from the memory filesystem -// @ID memfs-3-get-file -// @Produce application/data -// @Produce json -// @Param path path string true "Path to file" -// @Success 200 {file} byte -// @Success 301 {string} string -// @Failure 404 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/mem/{path} [get] -func (h *MemFSHandler) GetFile(c echo.Context) error { - return h.handler.GetFile(c) -} - -// PutFileAPI adds or overwrites a file at the given path -// @Summary Add a file to the memory filesystem -// @Description Writes or overwrites a file on the memory filesystem -// @ID memfs-3-put-file -// @Accept application/data -// @Produce text/plain -// @Produce json -// @Param path path string true "Path to file" -// @Param data body []byte true "File data" -// @Success 201 {string} string -// @Success 204 {string} string -// @Failure 507 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/mem/{path} [put] -func (h *MemFSHandler) PutFile(c echo.Context) error { - return h.handler.PutFile(c) -} - -// DeleteFileAPI removes a file from the filesystem -// @Summary Remove a file from the memory filesystem -// @Description Remove a file from the memory filesystem -// @ID memfs-3-delete-file -// @Produce text/plain -// @Param path path string true "Path to file" -// @Success 200 {string} string -// @Failure 404 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/mem/{path} [delete] -func (h *MemFSHandler) DeleteFile(c echo.Context) error { - return h.handler.DeleteFile(c) -} - -// PatchFile creates a symbolic link to a file in the filesystem -// @Summary Create a link to a file in the memory filesystem -// @Description Create a link to a file in the memory filesystem. The file linked to has to exist. -// @ID memfs-3-patch -// @Accept application/data -// @Produce text/plain -// @Produce json -// @Param path path string true "Path to file" -// @Param url body string true "Path to the file to link to" -// @Success 201 {string} string -// @Failure 400 {object} api.Error -// @Security ApiKeyAuth -// @Router /api/v3/fs/mem/{path} [patch] -func (h *MemFSHandler) PatchFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - c.Response().Header().Del(echo.HeaderContentType) - - req := c.Request() - - body, err := io.ReadAll(req.Body) - if err != nil { - return api.Err(http.StatusBadRequest, "Failed reading request body", "%s", err) - } - - u, err := url.Parse(string(body)) - if err != nil { - return api.Err(http.StatusBadRequest, "Body doesn't contain a valid path", "%s", err) - } - - if err := h.filesystem.Symlink(u.Path, path); err != nil { - return api.Err(http.StatusBadRequest, "Failed to create symlink", "%s", err) - } - - c.Response().Header().Set("Content-Location", req.URL.RequestURI()) - - return c.String(http.StatusCreated, "") -} - -// ListFiles lists all files on the filesystem -// @Summary List all files on the memory filesystem -// @Description List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. -// @ID memfs-3-list-files -// @Produce json -// @Param glob query string false "glob pattern for file names" -// @Param sort query string false "none, name, size, lastmod" -// @Param order query string false "asc, desc" -// @Success 200 {array} api.FileInfo -// @Security ApiKeyAuth -// @Router /api/v3/fs/mem [get] -func (h *MemFSHandler) ListFiles(c echo.Context) error { - pattern := util.DefaultQuery(c, "glob", "") - sortby := util.DefaultQuery(c, "sort", "none") - order := util.DefaultQuery(c, "order", "asc") - - files := h.filesystem.List(pattern) - - var sortFunc func(i, j int) bool - - switch sortby { - case "name": - if order == "desc" { - sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() } - } else { - sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() } - } - case "size": - if order == "desc" { - sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() } - } else { - sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() } - } - default: - if order == "asc" { - sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) } - } else { - sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) } - } - } - - sort.Slice(files, sortFunc) - - var fileinfos []api.FileInfo = make([]api.FileInfo, len(files)) - - for i, f := range files { - fileinfos[i] = api.FileInfo{ - Name: f.Name(), - Size: f.Size(), - LastMod: f.ModTime().Unix(), - } - } - - return c.JSON(http.StatusOK, fileinfos) -} diff --git a/http/handler/diskfs.go b/http/handler/diskfs.go deleted file mode 100644 index 9726c258..00000000 --- a/http/handler/diskfs.go +++ /dev/null @@ -1,88 +0,0 @@ -package handler - -import ( - "net/http" - "path/filepath" - - "github.com/datarhei/core/v16/http/api" - "github.com/datarhei/core/v16/http/cache" - "github.com/datarhei/core/v16/http/handler/util" - "github.com/datarhei/core/v16/io/fs" - - "github.com/labstack/echo/v4" -) - -// The DiskFSHandler type provides handlers for manipulating a filesystem -type DiskFSHandler struct { - cache cache.Cacher - filesystem fs.Filesystem -} - -// NewDiskFS return a new DiskFS type. You have to provide a filesystem to act on and optionally -// a Cacher where files will be purged from if the Cacher is related to the filesystem. -func NewDiskFS(fs fs.Filesystem, cache cache.Cacher) *DiskFSHandler { - return &DiskFSHandler{ - cache: cache, - filesystem: fs, - } -} - -// GetFile returns the file at the given path -// @Summary Fetch a file from the filesystem -// @Description Fetch a file from the filesystem. If the file is a directory, a index.html is returned, if it exists. -// @ID diskfs-get-file -// @Produce application/data -// @Produce json -// @Param path path string true "Path to file" -// @Success 200 {file} byte -// @Success 301 {string} string -// @Failure 404 {object} api.Error -// @Router /{path} [get] -func (h *DiskFSHandler) GetFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - mimeType := c.Response().Header().Get(echo.HeaderContentType) - c.Response().Header().Del(echo.HeaderContentType) - - file := h.filesystem.Open(path) - if file == nil { - return api.Err(http.StatusNotFound, "File not found", path) - } - - stat, _ := file.Stat() - - if stat.IsDir() { - path = filepath.Join(path, "index.html") - - file.Close() - - file = h.filesystem.Open(path) - if file == nil { - return api.Err(http.StatusNotFound, "File not found", path) - } - - stat, _ = file.Stat() - } - - defer file.Close() - - c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) - - if path, ok := stat.IsLink(); ok { - path = filepath.Clean("/" + path) - - if path[0] == '/' { - path = path[1:] - } - - return c.Redirect(http.StatusMovedPermanently, path) - } - - c.Response().Header().Set(echo.HeaderContentType, mimeType) - - if c.Request().Method == "HEAD" { - return c.Blob(http.StatusOK, "application/data", nil) - } - - return c.Stream(http.StatusOK, "application/data", file) -} diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go new file mode 100644 index 00000000..254ff609 --- /dev/null +++ b/http/handler/filesystem.go @@ -0,0 +1,164 @@ +package handler + +import ( + "net/http" + "path/filepath" + "sort" + + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/http/fs" + "github.com/datarhei/core/v16/http/handler/util" + + "github.com/labstack/echo/v4" +) + +// The FSHandler type provides handlers for manipulating a filesystem +type FSHandler struct { + fs fs.FS +} + +// NewFS return a new FSHandler type. You have to provide a filesystem to act on. +func NewFS(fs fs.FS) *FSHandler { + return &FSHandler{ + fs: fs, + } +} + +func (h *FSHandler) GetFile(c echo.Context) error { + path := util.PathWildcardParam(c) + + mimeType := c.Response().Header().Get(echo.HeaderContentType) + c.Response().Header().Del(echo.HeaderContentType) + + file := h.fs.Filesystem.Open(path) + if file == nil { + return api.Err(http.StatusNotFound, "File not found", path) + } + + stat, _ := file.Stat() + + if len(h.fs.DefaultFile) != 0 { + if stat.IsDir() { + path = filepath.Join(path, h.fs.DefaultFile) + + file.Close() + + file = h.fs.Filesystem.Open(path) + if file == nil { + return api.Err(http.StatusNotFound, "File not found", path) + } + + stat, _ = file.Stat() + } + } + + defer file.Close() + + c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + + if path, ok := stat.IsLink(); ok { + path = filepath.Clean("/" + path) + + if path[0] == '/' { + path = path[1:] + } + + return c.Redirect(http.StatusMovedPermanently, path) + } + + c.Response().Header().Set(echo.HeaderContentType, mimeType) + + if c.Request().Method == "HEAD" { + return c.Blob(http.StatusOK, "application/data", nil) + } + + return c.Stream(http.StatusOK, "application/data", file) +} + +func (h *FSHandler) PutFile(c echo.Context) error { + path := util.PathWildcardParam(c) + + c.Response().Header().Del(echo.HeaderContentType) + + req := c.Request() + + _, created, err := h.fs.Filesystem.Store(path, req.Body) + if err != nil { + return api.Err(http.StatusBadRequest, "%s", err) + } + + if h.fs.Cache != nil { + h.fs.Cache.Delete(path) + } + + c.Response().Header().Set("Content-Location", req.URL.RequestURI()) + + if created { + return c.String(http.StatusCreated, "") + } + + return c.NoContent(http.StatusNoContent) +} + +func (h *FSHandler) DeleteFile(c echo.Context) error { + path := util.PathWildcardParam(c) + + c.Response().Header().Del(echo.HeaderContentType) + + size := h.fs.Filesystem.Delete(path) + + if size < 0 { + return api.Err(http.StatusNotFound, "File not found", path) + } + + if h.fs.Cache != nil { + h.fs.Cache.Delete(path) + } + + return c.String(http.StatusOK, "Deleted: "+path) +} + +func (h *FSHandler) ListFiles(c echo.Context) error { + pattern := util.DefaultQuery(c, "glob", "") + sortby := util.DefaultQuery(c, "sort", "none") + order := util.DefaultQuery(c, "order", "asc") + + files := h.fs.Filesystem.List(pattern) + + var sortFunc func(i, j int) bool + + switch sortby { + case "name": + if order == "desc" { + sortFunc = func(i, j int) bool { return files[i].Name() > files[j].Name() } + } else { + sortFunc = func(i, j int) bool { return files[i].Name() < files[j].Name() } + } + case "size": + if order == "desc" { + sortFunc = func(i, j int) bool { return files[i].Size() > files[j].Size() } + } else { + sortFunc = func(i, j int) bool { return files[i].Size() < files[j].Size() } + } + default: + if order == "asc" { + sortFunc = func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) } + } else { + sortFunc = func(i, j int) bool { return files[i].ModTime().After(files[j].ModTime()) } + } + } + + sort.Slice(files, sortFunc) + + var fileinfos []api.FileInfo = make([]api.FileInfo, len(files)) + + for i, f := range files { + fileinfos[i] = api.FileInfo{ + Name: f.Name(), + Size: f.Size(), + LastMod: f.ModTime().Unix(), + } + } + + return c.JSON(http.StatusOK, fileinfos) +} diff --git a/http/handler/memfs.go b/http/handler/memfs.go deleted file mode 100644 index 1369a6dc..00000000 --- a/http/handler/memfs.go +++ /dev/null @@ -1,130 +0,0 @@ -package handler - -import ( - "net/http" - "path/filepath" - - "github.com/datarhei/core/v16/http/api" - "github.com/datarhei/core/v16/http/handler/util" - "github.com/datarhei/core/v16/io/fs" - - "github.com/labstack/echo/v4" -) - -// The MemFSHandler type provides handlers for manipulating a filesystem -type MemFSHandler struct { - filesystem fs.Filesystem -} - -// NewMemFS return a new MemFS type. You have to provide a filesystem to act on. -func NewMemFS(fs fs.Filesystem) *MemFSHandler { - return &MemFSHandler{ - filesystem: fs, - } -} - -// GetFile returns the file at the given path -// @Summary Fetch a file from the memory filesystem -// @Description Fetch a file from the memory filesystem -// @ID memfs-get-file -// @Produce application/data -// @Produce json -// @Param path path string true "Path to file" -// @Success 200 {file} byte -// @Success 301 {string} string -// @Failure 404 {object} api.Error -// @Router /memfs/{path} [get] -func (h *MemFSHandler) GetFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - mimeType := c.Response().Header().Get(echo.HeaderContentType) - c.Response().Header().Del(echo.HeaderContentType) - - file := h.filesystem.Open(path) - if file == nil { - return api.Err(http.StatusNotFound, "File not found", path) - } - - defer file.Close() - - stat, _ := file.Stat() - - c.Response().Header().Set("Last-Modified", stat.ModTime().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) - - if path, ok := stat.IsLink(); ok { - path = filepath.Clean("/" + path) - - if path[0] == '/' { - path = path[1:] - } - - return c.Redirect(http.StatusMovedPermanently, path) - } - - c.Response().Header().Set(echo.HeaderContentType, mimeType) - - if c.Request().Method == "HEAD" { - return c.Blob(http.StatusOK, "application/data", nil) - } - - return c.Stream(http.StatusOK, "application/data", file) -} - -// PutFile adds or overwrites a file at the given path -// @Summary Add a file to the memory filesystem -// @Description Writes or overwrites a file on the memory filesystem -// @ID memfs-put-file -// @Accept application/data -// @Produce text/plain -// @Produce json -// @Param path path string true "Path to file" -// @Param data body []byte true "File data" -// @Success 201 {string} string -// @Success 204 {string} string -// @Failure 507 {object} api.Error -// @Security BasicAuth -// @Router /memfs/{path} [put] -func (h *MemFSHandler) PutFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - c.Response().Header().Del(echo.HeaderContentType) - - req := c.Request() - - _, created, err := h.filesystem.Store(path, req.Body) - if err != nil { - return api.Err(http.StatusBadRequest, "%s", err) - } - - c.Response().Header().Set("Content-Location", req.URL.RequestURI()) - - if created { - return c.String(http.StatusCreated, "") - } - - return c.NoContent(http.StatusNoContent) -} - -// DeleteFile removes a file from the filesystem -// @Summary Remove a file from the memory filesystem -// @Description Remove a file from the memory filesystem -// @ID memfs-delete-file -// @Produce text/plain -// @Param path path string true "Path to file" -// @Success 200 {string} string -// @Failure 404 {object} api.Error -// @Security BasicAuth -// @Router /memfs/{path} [delete] -func (h *MemFSHandler) DeleteFile(c echo.Context) error { - path := util.PathWildcardParam(c) - - c.Response().Header().Del(echo.HeaderContentType) - - size := h.filesystem.Delete(path) - - if size < 0 { - return api.Err(http.StatusNotFound, "File not found", path) - } - - return c.String(http.StatusOK, "Deleted: "+path) -} diff --git a/http/server.go b/http/server.go index e05b44ea..68281edf 100644 --- a/http/server.go +++ b/http/server.go @@ -29,19 +29,19 @@ package http import ( + "fmt" "net/http" "strings" "github.com/datarhei/core/v16/config" - "github.com/datarhei/core/v16/http/cache" "github.com/datarhei/core/v16/http/errorhandler" + "github.com/datarhei/core/v16/http/fs" "github.com/datarhei/core/v16/http/graph/resolver" "github.com/datarhei/core/v16/http/handler" api "github.com/datarhei/core/v16/http/handler/api" "github.com/datarhei/core/v16/http/jwt" "github.com/datarhei/core/v16/http/router" "github.com/datarhei/core/v16/http/validator" - "github.com/datarhei/core/v16/io/fs" "github.com/datarhei/core/v16/log" "github.com/datarhei/core/v16/monitor" "github.com/datarhei/core/v16/net" @@ -79,9 +79,7 @@ type Config struct { Metrics monitor.HistoryReader Prometheus prometheus.Reader MimeTypesFile string - DiskFS fs.Filesystem - MemFS MemFSConfig - S3FS MemFSConfig + Filesystems []fs.FS IPLimiter net.IPLimiter Profiling bool Cors CorsConfig @@ -89,19 +87,11 @@ type Config struct { SRT srt.Server JWT jwt.JWT Config config.Store - Cache cache.Cacher Sessions session.RegistryReader Router router.Router ReadOnly bool } -type MemFSConfig struct { - EnableAuth bool - Username string - Password string - Filesystem fs.Filesystem -} - type CorsConfig struct { Origins []string } @@ -115,9 +105,6 @@ type server struct { handler struct { about *api.AboutHandler - memfs *handler.MemFSHandler - s3fs *handler.MemFSHandler - diskfs *handler.DiskFSHandler prometheus *handler.PrometheusHandler profiling *handler.ProfilingHandler ping *handler.PingHandler @@ -129,9 +116,6 @@ type server struct { log *api.LogHandler restream *api.RestreamHandler playout *api.PlayoutHandler - memfs *api.MemFSHandler - s3fs *api.MemFSHandler - diskfs *api.DiskFSHandler rtmp *api.RTMPHandler srt *api.SRTHandler config *api.ConfigHandler @@ -150,18 +134,12 @@ type server struct { session echo.MiddlewareFunc } - memfs struct { - enableAuth bool - username string - password string - } - - diskfs fs.Filesystem - gzip struct { mimetypes []string } + filesystems map[string]*filesystem + router *echo.Echo mimeTypesFile string profiling bool @@ -169,28 +147,56 @@ type server struct { readOnly bool } +type filesystem struct { + fs.FS + + handler *handler.FSHandler +} + func NewServer(config Config) (Server, error) { s := &server{ logger: config.Logger, mimeTypesFile: config.MimeTypesFile, profiling: config.Profiling, - diskfs: config.DiskFS, readOnly: config.ReadOnly, } - s.v3handler.diskfs = api.NewDiskFS( - config.DiskFS, - config.Cache, - ) + s.filesystems = map[string]*filesystem{} - s.handler.diskfs = handler.NewDiskFS( - config.DiskFS, - config.Cache, - ) + corsPrefixes := map[string][]string{ + "/api": {"*"}, + } - s.memfs.enableAuth = config.MemFS.EnableAuth - s.memfs.username = config.MemFS.Username - s.memfs.password = config.MemFS.Password + for _, fs := range config.Filesystems { + if _, ok := s.filesystems[fs.Name]; ok { + return nil, fmt.Errorf("the filesystem name '%s' is already in use", fs.Name) + } + + if !strings.HasPrefix(fs.Mountpoint, "/") { + fs.Mountpoint = "/" + fs.Mountpoint + } + + if !strings.HasSuffix(fs.Mountpoint, "/") { + fs.Mountpoint = strings.TrimSuffix(fs.Mountpoint, "/") + } + + if _, ok := corsPrefixes[fs.Mountpoint]; ok { + return nil, fmt.Errorf("the mount point '%s' is already in use (%s)", fs.Mountpoint, fs.Name) + } + + corsPrefixes[fs.Mountpoint] = config.Cors.Origins + + filesystem := &filesystem{ + FS: fs, + handler: handler.NewFS(fs), + } + + s.filesystems[filesystem.Name] = filesystem + } + + if _, ok := corsPrefixes["/"]; !ok { + return nil, fmt.Errorf("one filesystem must be mounted at /") + } if config.Logger == nil { s.logger = log.New("HTTP") @@ -222,26 +228,6 @@ func NewServer(config Config) (Server, error) { ) } - if config.MemFS.Filesystem != nil { - s.v3handler.memfs = api.NewMemFS( - config.MemFS.Filesystem, - ) - - s.handler.memfs = handler.NewMemFS( - config.MemFS.Filesystem, - ) - } - - if config.S3FS.Filesystem != nil { - s.v3handler.s3fs = api.NewMemFS( - config.S3FS.Filesystem, - ) - - s.handler.s3fs = handler.NewMemFS( - config.S3FS.Filesystem, - ) - } - if config.Prometheus != nil { s.handler.prometheus = handler.NewPrometheus( config.Prometheus.HTTPHandler(), @@ -300,12 +286,6 @@ func NewServer(config Config) (Server, error) { Logger: s.logger, }) - if config.Cache != nil { - s.middleware.cache = mwcache.NewWithConfig(mwcache.Config{ - Cache: config.Cache, - }) - } - s.v3handler.widget = api.NewWidget(api.WidgetConfig{ Restream: config.Restream, Registry: config.Sessions, @@ -316,12 +296,7 @@ func NewServer(config Config) (Server, error) { }) if middleware, err := mwcors.NewWithConfig(mwcors.Config{ - Prefixes: map[string][]string{ - "/": config.Cors.Origins, - "/api": {"*"}, - "/memfs": config.Cors.Origins, - "/s3": config.Cors.Origins, - }, + Prefixes: corsPrefixes, }); err != nil { return nil, err } else { @@ -447,105 +422,48 @@ func (s *server) setRoutes() { doc.Use(gzipMiddleware) doc.GET("", echoSwagger.WrapHandler) - // Serve static data - fs := s.router.Group("/*") - fs.Use(mwmime.NewWithConfig(mwmime.Config{ - MimeTypesFile: s.mimeTypesFile, - DefaultContentType: "text/html", - })) - fs.Use(mwgzip.NewWithConfig(mwgzip.Config{ - Level: mwgzip.BestSpeed, - MinLength: 1000, - ContentTypes: s.gzip.mimetypes, - })) - if s.middleware.cache != nil { - fs.Use(s.middleware.cache) - } - - fs.GET("", s.handler.diskfs.GetFile) - fs.HEAD("", s.handler.diskfs.GetFile) - - // Memory FS - if s.handler.memfs != nil { - memfs := s.router.Group("/memfs/*") - memfs.Use(mwmime.NewWithConfig(mwmime.Config{ + // Mount filesystems + for _, filesystem := range s.filesystems { + fs := s.router.Group(filesystem.Mountpoint + "/*") + fs.Use(mwmime.NewWithConfig(mwmime.Config{ MimeTypesFile: s.mimeTypesFile, - DefaultContentType: "application/data", + DefaultContentType: filesystem.DefaultContentType, })) - memfs.Use(mwgzip.NewWithConfig(mwgzip.Config{ - Level: mwgzip.BestSpeed, - MinLength: 1000, - ContentTypes: s.gzip.mimetypes, - })) - if s.middleware.session != nil { - memfs.Use(s.middleware.session) + + if filesystem.Gzip { + fs.Use(mwgzip.NewWithConfig(mwgzip.Config{ + Level: mwgzip.BestSpeed, + MinLength: 1000, + ContentTypes: s.gzip.mimetypes, + })) } - memfs.HEAD("", s.handler.memfs.GetFile) - memfs.GET("", s.handler.memfs.GetFile) + if filesystem.Cache != nil { + mwcache := mwcache.NewWithConfig(mwcache.Config{ + Cache: filesystem.Cache, + }) + fs.Use(mwcache) + } - var authmw echo.MiddlewareFunc + fs.GET("", filesystem.handler.GetFile) + fs.HEAD("", filesystem.handler.GetFile) - if s.memfs.enableAuth { - authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { - if username == s.memfs.username && password == s.memfs.password { + if len(filesystem.Username) != 0 || len(filesystem.Password) != 0 { + authmw := middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { + if username == filesystem.Username && password == filesystem.Password { return true, nil } return false, nil }) - memfs.POST("", s.handler.memfs.PutFile, authmw) - memfs.PUT("", s.handler.memfs.PutFile, authmw) - memfs.DELETE("", s.handler.memfs.DeleteFile, authmw) + fs.POST("", filesystem.handler.PutFile, authmw) + fs.PUT("", filesystem.handler.PutFile, authmw) + fs.DELETE("", filesystem.handler.DeleteFile, authmw) } else { - memfs.POST("", s.handler.memfs.PutFile) - memfs.PUT("", s.handler.memfs.PutFile) - memfs.DELETE("", s.handler.memfs.DeleteFile) - } - } - - // S3 FS - if s.handler.s3fs != nil { - s3fs := s.router.Group("/s3/*") - s3fs.Use(mwmime.NewWithConfig(mwmime.Config{ - MimeTypesFile: s.mimeTypesFile, - DefaultContentType: "application/data", - })) - s3fs.Use(mwgzip.NewWithConfig(mwgzip.Config{ - Level: mwgzip.BestSpeed, - MinLength: 1000, - ContentTypes: s.gzip.mimetypes, - })) - if s.middleware.session != nil { - s3fs.Use(s.middleware.session) - } - - s3fs.HEAD("", s.handler.s3fs.GetFile) - s3fs.GET("", s.handler.s3fs.GetFile) - - var authmw echo.MiddlewareFunc - - if s.memfs.enableAuth { - authmw = middleware.BasicAuth(func(username, password string, c echo.Context) (bool, error) { - if username == s.memfs.username && password == s.memfs.password { - return true, nil - } - - return false, nil - }) - - s3fs.POST("", s.handler.s3fs.PutFile, authmw) - s3fs.PUT("", s.handler.s3fs.PutFile, authmw) - s3fs.DELETE("", s.handler.s3fs.DeleteFile, authmw) - } else { - s3fs.POST("", s.handler.s3fs.PutFile) - s3fs.PUT("", s.handler.s3fs.PutFile) - s3fs.DELETE("", s.handler.s3fs.DeleteFile) - } - - if s.middleware.cache != nil { - s3fs.Use(s.middleware.cache) + fs.POST("", filesystem.handler.PutFile) + fs.PUT("", filesystem.handler.PutFile) + fs.DELETE("", filesystem.handler.DeleteFile) } } @@ -643,44 +561,33 @@ func (s *server) setRoutesV3(v3 *echo.Group) { } } - // v3 Memory FS - if s.v3handler.memfs != nil { - v3.GET("/fs/mem", s.v3handler.memfs.ListFiles) - v3.GET("/fs/mem/*", s.v3handler.memfs.GetFile) - - if !s.readOnly { - v3.DELETE("/fs/mem/*", s.v3handler.memfs.DeleteFile) - v3.PUT("/fs/mem/*", s.v3handler.memfs.PutFile) - v3.PATCH("/fs/mem/*", s.v3handler.memfs.PatchFile) + // v3 Filesystems + fshandlers := map[string]api.FSConfig{} + for _, fs := range s.filesystems { + fshandlers[fs.Name] = api.FSConfig{ + Type: fs.Filesystem.Type(), + Mountpoint: fs.Mountpoint, + Handler: fs.handler, } } - // v3 S3 FS - if s.v3handler.s3fs != nil { - v3.GET("/fs/s3", s.v3handler.s3fs.ListFiles) - v3.GET("/fs/s3/*", s.v3handler.s3fs.GetFile) + handler := api.NewFS(fshandlers) - if !s.readOnly { - v3.DELETE("/fs/s3/*", s.v3handler.s3fs.DeleteFile) - v3.PUT("/fs/s3/*", s.v3handler.s3fs.PutFile) - v3.PATCH("/fs/s3/*", s.v3handler.s3fs.PatchFile) - } - } + v3.GET("/fs", handler.List) - // v3 Disk FS - v3.GET("/fs/disk", s.v3handler.diskfs.ListFiles) - v3.GET("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{ + v3.GET("/fs/:name", handler.ListFiles) + v3.GET("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ MimeTypesFile: s.mimeTypesFile, DefaultContentType: "application/data", })) - v3.HEAD("/fs/disk/*", s.v3handler.diskfs.GetFile, mwmime.NewWithConfig(mwmime.Config{ + v3.HEAD("/fs/:name/*", handler.GetFile, mwmime.NewWithConfig(mwmime.Config{ MimeTypesFile: s.mimeTypesFile, DefaultContentType: "application/data", })) if !s.readOnly { - v3.PUT("/fs/disk/*", s.v3handler.diskfs.PutFile) - v3.DELETE("/fs/disk/*", s.v3handler.diskfs.DeleteFile) + v3.PUT("/fs/:name/*", handler.PutFile) + v3.DELETE("/fs/:name/*", handler.DeleteFile) } // v3 RTMP diff --git a/io/fs/disk.go b/io/fs/disk.go index e65f6178..ff98e0f2 100644 --- a/io/fs/disk.go +++ b/io/fs/disk.go @@ -135,6 +135,8 @@ func NewDiskFilesystem(config DiskConfig) (Filesystem, error) { fs.logger = log.New("") } + fs.logger = fs.logger.WithField("type", "disk") + if err := fs.Rebase(config.Dir); err != nil { return nil, err } @@ -172,6 +174,10 @@ func (fs *diskFilesystem) Rebase(base string) error { return nil } +func (fs *diskFilesystem) Type() string { + return "diskfs" +} + func (fs *diskFilesystem) Size() (int64, int64) { // This is to cache the size for some time in order not to // stress the underlying filesystem too much. diff --git a/io/fs/dummy.go b/io/fs/dummy.go index 442d1586..e4c0dfdc 100644 --- a/io/fs/dummy.go +++ b/io/fs/dummy.go @@ -24,6 +24,7 @@ type dummyFilesystem struct{} func (d *dummyFilesystem) Base() string { return "/" } func (d *dummyFilesystem) Rebase(string) error { return nil } +func (d *dummyFilesystem) Type() string { return "dummy" } func (d *dummyFilesystem) Size() (int64, int64) { return 0, -1 } func (d *dummyFilesystem) Resize(int64) {} func (d *dummyFilesystem) Files() int64 { return 0 } diff --git a/io/fs/fs.go b/io/fs/fs.go index fd45c4a2..9e7da017 100644 --- a/io/fs/fs.go +++ b/io/fs/fs.go @@ -44,6 +44,9 @@ type Filesystem interface { // Rebase sets a new base path for this filesystem Rebase(string) error + // Type returns the type of this filesystem + Type() string + // Size returns the consumed size and capacity of the filesystem in bytes. The // capacity is negative if the filesystem can consume as much space as it can. Size() (int64, int64) diff --git a/io/fs/mem.go b/io/fs/mem.go index fd7cd13c..7a2c028e 100644 --- a/io/fs/mem.go +++ b/io/fs/mem.go @@ -146,6 +146,8 @@ func NewMemFilesystem(config MemConfig) Filesystem { fs.logger = log.New("") } + fs.logger = fs.logger.WithField("type", "mem") + fs.files = make(map[string]*memFile) fs.dataPool = sync.Pool{ @@ -172,6 +174,10 @@ func (fs *memFilesystem) Rebase(base string) error { return nil } +func (fs *memFilesystem) Type() string { + return "memfs" +} + func (fs *memFilesystem) Size() (int64, int64) { fs.filesLock.RLock() defer fs.filesLock.RUnlock() diff --git a/io/fs/s3.go b/io/fs/s3.go index 95d6c839..c5649252 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -66,6 +66,7 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) { } fs.logger = fs.logger.WithFields(log.Fields{ + "type": "s3", "bucket": fs.bucket, "region": fs.region, "endpoint": fs.endpoint, @@ -107,6 +108,10 @@ func (fs *s3fs) Rebase(base string) error { return nil } +func (fs *s3fs) Type() string { + return "s3fs" +} + func (fs *s3fs) Size() (int64, int64) { size := int64(0)