From 273ca0abbcaba290090c8ddef6fde657383d41ab Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Tue, 2 Aug 2022 19:10:28 +0200 Subject: [PATCH] Add cache block list for extensions not to cache --- README.md | 13 ++- app/api/api.go | 16 +-- app/import/main.go | 2 - app/import/main_test.go | 2 - app/version.go | 4 +- config/config.go | 198 ++------------------------------ config/data.go | 233 +++++++++++++++++++++++++++++++++++++ config/data_v1.go | 154 +++++++++++++++++++++++++ config/data_v2.go | 247 ++++++++++++++++++++++++++++++++++++++++ config/json.go | 52 +++++++-- http/cache/lru.go | 59 ++++++---- http/cache/lru_test.go | 30 +++-- 12 files changed, 760 insertions(+), 250 deletions(-) create mode 100644 config/data.go create mode 100644 config/data_v1.go create mode 100644 config/data_v2.go diff --git a/README.md b/README.md index c322d57f..0c1baf20 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Core + The cloud-native audio/video processing API. -[![License: MIT](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)]([https://opensource.org/licenses/MI](https://www.apache.org/licenses/LICENSE-2.0)) +[![License: MIT](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)](<[https://opensource.org/licenses/MI](https://www.apache.org/licenses/LICENSE-2.0)>) [![CodeQL](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml) [![tests](https://github.com/datarhei/core/actions/workflows/go-tests.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/go-tests.yml) [![codecov](https://codecov.io/gh/datarhei/core/branch/main/graph/badge.svg?token=90YMPZRAFK)](https://codecov.io/gh/datarhei/core) @@ -119,7 +120,8 @@ The currently known environment variables (but not all will be respected) are: | CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES | `0` | Max. allowed cache size, 0 for unlimited. | | CORE_STORAGE_DISK_CACHE_TTLSECONDS | `300` | Seconds to keep files in cache. | | CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES | `1` | Max. file size to put in cache. | -| CORE_STORAGE_DISK_CACHE_TYPES | (not set) | List of file extensions to cache (space-separated, e.g. ".html .js"), empty for all. | +| CORE_STORAGE_DISK_CACHE_TYPES_ALLOW | (not set) | List of file extensions to cache (space-separated, e.g. ".html .js"), empty for all. | +| CORE_STORAGE_DISK_CACHE_TYPES_BLOCK | (not set) | List of file extensions not to cache (space-separated, e.g. ".m3u8 .mpd"), empty for none. | | CORE_STORAGE_MEMORY_AUTH_ENABLE | `true` | Enable basic auth for PUT,POST, and DELETE on /memfs. | | CORE_STORAGE_MEMORY_AUTH_USERNAME | (not set) | Username for Basic-Auth of `/memfs`. Required if auth is enabled. | | CORE_STORAGE_MEMORY_AUTH_PASSWORD | (not set) | Password for Basic-Auth of `/memfs`. Required if auth is enabled. | @@ -180,7 +182,7 @@ All other values will be filled with default values and persisted on disk. The e ``` { - "version": 1, + "version": 3, "id": "[will be generated if not given]", "name": "[will be generated if not given]", "address": ":8080", @@ -238,7 +240,10 @@ All other values will be filled with default values and persisted on disk. The e "max_size_mbytes": 0, "ttl_seconds": 300, "max_file_size_mbytes": 1, - "types": [] + "types": { + "allow": [], + "block": [] + } } }, "memory": { diff --git a/app/api/api.go b/app/api/api.go index eed68c73..73708929 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -154,11 +154,6 @@ func (a *api) Reload() error { } cfg := store.Get() - if err := cfg.Migrate(); err == nil { - store.Set(cfg) - } else { - return err - } cfg.Merge() @@ -631,11 +626,12 @@ func (a *api) start() error { if cfg.Storage.Disk.Cache.Enable { diskCache, err := cache.NewLRUCache(cache.LRUConfig{ - TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, - MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, - MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, - Extensions: cfg.Storage.Disk.Cache.Types, - Logger: a.log.logger.core.WithComponent("HTTPCache"), + TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, + MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, + MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, + AllowExtensions: cfg.Storage.Disk.Cache.Types.Allow, + BlockExtensions: cfg.Storage.Disk.Cache.Types.Block, + Logger: a.log.logger.core.WithComponent("HTTPCache"), }) if err != nil { diff --git a/app/import/main.go b/app/import/main.go index c3a874ac..b5b89b24 100644 --- a/app/import/main.go +++ b/app/import/main.go @@ -33,7 +33,6 @@ func doImport(logger log.Logger, configstore config.Store) error { logger.Info().Log("Database import") cfg := configstore.Get() - cfg.Migrate() // Merging the persisted config with the environment variables cfg.Merge() @@ -117,7 +116,6 @@ func doImport(logger log.Logger, configstore config.Store) error { // Get the unmerged config for persisting cfg = configstore.Get() - cfg.Migrate() // Add static routes to mimic the old URLs cfg.Router.Routes["/hls/live.stream.m3u8"] = "/memfs/" + importConfig.id + ".m3u8" diff --git a/app/import/main_test.go b/app/import/main_test.go index b9ad95d1..b85bfafe 100644 --- a/app/import/main_test.go +++ b/app/import/main_test.go @@ -11,8 +11,6 @@ func TestImport(t *testing.T) { configstore := config.NewDummyStore() cfg := configstore.Get() - cfg.Version = 1 - cfg.Migrate() err := configstore.Set(cfg) require.NoError(t, err) diff --git a/app/version.go b/app/version.go index f0b9e610..0b6b5332 100644 --- a/app/version.go +++ b/app/version.go @@ -29,8 +29,8 @@ func (v versionInfo) MinorString() string { // Version of the app var Version = versionInfo{ Major: 16, - Minor: 9, - Patch: 1, + Minor: 10, + Patch: 0, } // Commit is the git commit the app is build from. It should be filled in during compilation diff --git a/config/config.go b/config/config.go index 336a1745..3af347dc 100644 --- a/config/config.go +++ b/config/config.go @@ -6,8 +6,6 @@ import ( "fmt" "net" "os" - "strconv" - "strings" "time" "github.com/datarhei/core/v16/math/rand" @@ -16,7 +14,7 @@ import ( "github.com/google/uuid" ) -const version int64 = 2 +const version int64 = 3 type variable struct { value value // The actual value @@ -51,157 +49,8 @@ type Auth0Tenant struct { Users []string `json:"users"` } -// Data is the actual configuration data for the app -type Data struct { - CreatedAt time.Time `json:"created_at"` - LoadedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - Version int64 `json:"version" jsonschema:"minimum=1,maximum=1"` - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - CheckForUpdates bool `json:"update_check"` - Log struct { - Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` - Topics []string `json:"topics"` - MaxLines int `json:"max_lines"` - } `json:"log"` - DB struct { - Dir string `json:"dir"` - } `json:"db"` - Host struct { - Name []string `json:"name"` - Auto bool `json:"auto"` - } `json:"host"` - API struct { - ReadOnly bool `json:"read_only"` - Access struct { - HTTP struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"http"` - HTTPS struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"https"` - } `json:"access"` - Auth struct { - Enable bool `json:"enable"` - DisableLocalhost bool `json:"disable_localhost"` - Username string `json:"username"` - Password string `json:"password"` - JWT struct { - Secret string `json:"secret"` - } `json:"jwt"` - Auth0 struct { - Enable bool `json:"enable"` - Tenants []Auth0Tenant `json:"tenants"` - } `json:"auth0"` - } `json:"auth"` - } `json:"api"` - TLS struct { - Address string `json:"address"` - Enable bool `json:"enable"` - Auto bool `json:"auto"` - CertFile string `json:"cert_file"` - KeyFile string `json:"key_file"` - } `json:"tls"` - Storage struct { - Disk struct { - Dir string `json:"dir"` - Size int64 `json:"max_size_mbytes"` - Cache struct { - Enable bool `json:"enable"` - Size uint64 `json:"max_size_mbytes"` - TTL int64 `json:"ttl_seconds"` - FileSize uint64 `json:"max_file_size_mbytes"` - Types []string `json:"types"` - } `json:"cache"` - } `json:"disk"` - Memory struct { - Auth struct { - Enable bool `json:"enable"` - Username string `json:"username"` - Password string `json:"password"` - } `json:"auth"` - Size int64 `json:"max_size_mbytes"` - Purge bool `json:"purge"` - } `json:"memory"` - CORS struct { - Origins []string `json:"origins"` - } `json:"cors"` - MimeTypes string `json:"mimetypes_file"` - } `json:"storage"` - RTMP struct { - Enable bool `json:"enable"` - EnableTLS bool `json:"enable_tls"` - Address string `json:"address"` - AddressTLS string `json:"address_tls"` - App string `json:"app"` - Token string `json:"token"` - } `json:"rtmp"` - SRT struct { - Enable bool `json:"enable"` - Address string `json:"address"` - Passphrase string `json:"passphrase"` - Token string `json:"token"` - Log struct { - Enable bool `json:"enable"` - Topics []string `json:"topics"` - } `json:"log"` - } `json:"srt"` - FFmpeg struct { - Binary string `json:"binary"` - MaxProcesses int64 `json:"max_processes"` - Access struct { - Input struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"input"` - Output struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"output"` - } `json:"access"` - Log struct { - MaxLines int `json:"max_lines"` - MaxHistory int `json:"max_history"` - } `json:"log"` - } `json:"ffmpeg"` - Playout struct { - Enable bool `json:"enable"` - MinPort int `json:"min_port"` - MaxPort int `json:"max_port"` - } `json:"playout"` - Debug struct { - Profiling bool `json:"profiling"` - ForceGC int `json:"force_gc"` - } `json:"debug"` - Metrics struct { - Enable bool `json:"enable"` - EnablePrometheus bool `json:"enable_prometheus"` - Range int64 `json:"range_sec"` // seconds - Interval int64 `json:"interval_sec"` // seconds - } `json:"metrics"` - Sessions struct { - Enable bool `json:"enable"` - IPIgnoreList []string `json:"ip_ignorelist"` - SessionTimeout int `json:"session_timeout_sec"` - Persist bool `json:"persist"` - PersistInterval int `json:"persist_interval_sec"` - MaxBitrate uint64 `json:"max_bitrate_mbit"` - MaxSessions uint64 `json:"max_sessions"` - } `json:"sessions"` - Service struct { - Enable bool `json:"enable"` - Token string `json:"token"` - URL string `json:"url"` - } `json:"service"` - Router struct { - BlockedPrefixes []string `json:"blocked_prefixes"` - Routes map[string]string `json:"routes"` - UIPath string `json:"ui_path"` - } `json:"router"` +type DataVersion struct { + Version int64 `json:"version"` } // Config is a wrapper for Data @@ -214,11 +63,11 @@ type Config struct { // New returns a Config which is initialized with its default values func New() *Config { - data := &Config{} + config := &Config{} - data.init() + config.init() - return data + return config } // NewConfigFrom returns a clone of a Config @@ -263,6 +112,8 @@ func NewConfigFrom(d *Config) *Config { data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants) data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins) + data.Storage.Disk.Cache.Types.Allow = copyStringSlice(d.Storage.Disk.Cache.Types.Allow) + data.Storage.Disk.Cache.Types.Block = copyStringSlice(d.Storage.Disk.Cache.Types.Block) data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow) data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block) @@ -338,7 +189,8 @@ func (d *Config) init() { d.val(newUint64Value(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false) d.val(newInt64Value(&d.Storage.Disk.Cache.TTL, 300), "storage.disk.cache.ttl_seconds", "CORE_STORAGE_DISK_CACHE_TTLSECONDS", nil, "Seconds to keep files in cache", false, false) d.val(newUint64Value(&d.Storage.Disk.Cache.FileSize, 1), "storage.disk.cache.max_file_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES", nil, "Max. file size to put in cache", false, false) - d.val(newStringListValue(&d.Storage.Disk.Cache.Types, []string{}, " "), "storage.disk.cache.types", "CORE_STORAGE_DISK_CACHE_TYPES", nil, "File extensions to cache, empty for all", false, false) + d.val(newStringListValue(&d.Storage.Disk.Cache.Types.Allow, []string{}, " "), "storage.disk.cache.type.allow", "CORE_STORAGE_DISK_CACHE_TYPES_ALLOW", []string{"CORE_STORAGE_DISK_CACHE_TYPES"}, "File extensions to cache, empty for all", false, false) + d.val(newStringListValue(&d.Storage.Disk.Cache.Types.Block, []string{}, " "), "storage.disk.cache.type.block", "CORE_STORAGE_DISK_CACHE_TYPES_BLOCK", nil, "File extensions not to cache, empty for none", false, false) // Storage (Memory) d.val(newBoolValue(&d.Storage.Memory.Auth.Enable, true), "storage.memory.auth.enable", "CORE_STORAGE_MEMORY_AUTH_ENABLE", nil, "Enable basic auth for PUT,POST, and DELETE on /memfs", false, false) @@ -483,36 +335,6 @@ func (d *Config) Merge() { } } -// Migrate will migrate some settings, depending on the version it finds. Migrations -// are only going upwards,i.e. from a lower version to a higher version. -func (d *Config) Migrate() error { - if d.Version == 1 { - if !strings.HasPrefix(d.RTMP.App, "/") { - d.RTMP.App = "/" + d.RTMP.App - } - - if d.RTMP.EnableTLS { - d.RTMP.Enable = true - d.RTMP.AddressTLS = d.RTMP.Address - host, sport, err := net.SplitHostPort(d.RTMP.Address) - if err != nil { - return fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err) - } - - port, err := strconv.Atoi(sport) - if err != nil { - return fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err) - } - - d.RTMP.Address = net.JoinHostPort(host, strconv.Itoa(port-1)) - } - - d.Version = 2 - } - - return nil -} - // Validate validates the current state of the Config for completeness and sanity. Errors are // written to the log. Use resetLogs to indicate to reset the logs prior validation. func (d *Config) Validate(resetLogs bool) { diff --git a/config/data.go b/config/data.go new file mode 100644 index 00000000..8dd82822 --- /dev/null +++ b/config/data.go @@ -0,0 +1,233 @@ +package config + +import "time" + +// Data is the actual configuration data for the app +type Data struct { + CreatedAt time.Time `json:"created_at"` + LoadedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` + Version int64 `json:"version" jsonschema:"minimum=3,maximum=3"` + ID string `json:"id"` + Name string `json:"name"` + Address string `json:"address"` + CheckForUpdates bool `json:"update_check"` + Log struct { + Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` + Topics []string `json:"topics"` + MaxLines int `json:"max_lines"` + } `json:"log"` + DB struct { + Dir string `json:"dir"` + } `json:"db"` + Host struct { + Name []string `json:"name"` + Auto bool `json:"auto"` + } `json:"host"` + API struct { + ReadOnly bool `json:"read_only"` + Access struct { + HTTP struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"http"` + HTTPS struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"https"` + } `json:"access"` + Auth struct { + Enable bool `json:"enable"` + DisableLocalhost bool `json:"disable_localhost"` + Username string `json:"username"` + Password string `json:"password"` + JWT struct { + Secret string `json:"secret"` + } `json:"jwt"` + Auth0 struct { + Enable bool `json:"enable"` + Tenants []Auth0Tenant `json:"tenants"` + } `json:"auth0"` + } `json:"auth"` + } `json:"api"` + TLS struct { + Address string `json:"address"` + Enable bool `json:"enable"` + Auto bool `json:"auto"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + } `json:"tls"` + Storage struct { + Disk struct { + Dir string `json:"dir"` + Size int64 `json:"max_size_mbytes"` + Cache struct { + Enable bool `json:"enable"` + Size uint64 `json:"max_size_mbytes"` + TTL int64 `json:"ttl_seconds"` + FileSize uint64 `json:"max_file_size_mbytes"` + Types struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"types"` + } `json:"cache"` + } `json:"disk"` + Memory struct { + Auth struct { + Enable bool `json:"enable"` + Username string `json:"username"` + Password string `json:"password"` + } `json:"auth"` + Size int64 `json:"max_size_mbytes"` + Purge bool `json:"purge"` + } `json:"memory"` + CORS struct { + Origins []string `json:"origins"` + } `json:"cors"` + MimeTypes string `json:"mimetypes_file"` + } `json:"storage"` + RTMP struct { + Enable bool `json:"enable"` + EnableTLS bool `json:"enable_tls"` + Address string `json:"address"` + AddressTLS string `json:"address_tls"` + App string `json:"app"` + Token string `json:"token"` + } `json:"rtmp"` + SRT struct { + Enable bool `json:"enable"` + Address string `json:"address"` + Passphrase string `json:"passphrase"` + Token string `json:"token"` + Log struct { + Enable bool `json:"enable"` + Topics []string `json:"topics"` + } `json:"log"` + } `json:"srt"` + FFmpeg struct { + Binary string `json:"binary"` + MaxProcesses int64 `json:"max_processes"` + Access struct { + Input struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"input"` + Output struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"output"` + } `json:"access"` + Log struct { + MaxLines int `json:"max_lines"` + MaxHistory int `json:"max_history"` + } `json:"log"` + } `json:"ffmpeg"` + Playout struct { + Enable bool `json:"enable"` + MinPort int `json:"min_port"` + MaxPort int `json:"max_port"` + } `json:"playout"` + Debug struct { + Profiling bool `json:"profiling"` + ForceGC int `json:"force_gc"` + } `json:"debug"` + Metrics struct { + Enable bool `json:"enable"` + EnablePrometheus bool `json:"enable_prometheus"` + Range int64 `json:"range_sec"` // seconds + Interval int64 `json:"interval_sec"` // seconds + } `json:"metrics"` + Sessions struct { + Enable bool `json:"enable"` + IPIgnoreList []string `json:"ip_ignorelist"` + SessionTimeout int `json:"session_timeout_sec"` + Persist bool `json:"persist"` + PersistInterval int `json:"persist_interval_sec"` + MaxBitrate uint64 `json:"max_bitrate_mbit"` + MaxSessions uint64 `json:"max_sessions"` + } `json:"sessions"` + Service struct { + Enable bool `json:"enable"` + Token string `json:"token"` + URL string `json:"url"` + } `json:"service"` + Router struct { + BlockedPrefixes []string `json:"blocked_prefixes"` + Routes map[string]string `json:"routes"` + UIPath string `json:"ui_path"` + } `json:"router"` +} + +func NewV3FromV2(d *dataV2) (*Data, error) { + data := &Data{} + + data.CreatedAt = d.CreatedAt + data.LoadedAt = d.LoadedAt + data.UpdatedAt = d.UpdatedAt + + data.ID = d.ID + data.Name = d.Name + data.Address = d.Address + data.CheckForUpdates = d.CheckForUpdates + + data.Log = d.Log + data.DB = d.DB + data.Host = d.Host + data.API = d.API + data.TLS = d.TLS + data.RTMP = d.RTMP + data.SRT = d.SRT + data.FFmpeg = d.FFmpeg + data.Playout = d.Playout + data.Debug = d.Debug + data.Metrics = d.Metrics + data.Sessions = d.Sessions + data.Service = d.Service + data.Router = d.Router + + data.Log.Topics = copyStringSlice(d.Log.Topics) + + data.Host.Name = copyStringSlice(d.Host.Name) + + data.API.Access.HTTP.Allow = copyStringSlice(d.API.Access.HTTP.Allow) + data.API.Access.HTTP.Block = copyStringSlice(d.API.Access.HTTP.Block) + data.API.Access.HTTPS.Allow = copyStringSlice(d.API.Access.HTTPS.Allow) + data.API.Access.HTTPS.Block = copyStringSlice(d.API.Access.HTTPS.Block) + + data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants) + + data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins) + + data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow) + data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block) + data.FFmpeg.Access.Output.Allow = copyStringSlice(d.FFmpeg.Access.Output.Allow) + data.FFmpeg.Access.Output.Block = copyStringSlice(d.FFmpeg.Access.Output.Block) + + data.Sessions.IPIgnoreList = copyStringSlice(d.Sessions.IPIgnoreList) + + data.SRT.Log.Topics = copyStringSlice(d.SRT.Log.Topics) + + data.Router.BlockedPrefixes = copyStringSlice(d.Router.BlockedPrefixes) + data.Router.Routes = copyStringMap(d.Router.Routes) + + // Actual changes + data.Storage.MimeTypes = d.Storage.MimeTypes + + data.Storage.CORS = d.Storage.CORS + data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins) + + data.Storage.Memory = d.Storage.Memory + + data.Storage.Disk.Dir = d.Storage.Disk.Dir + data.Storage.Disk.Size = d.Storage.Disk.Size + data.Storage.Disk.Cache.Enable = d.Storage.Disk.Cache.Enable + data.Storage.Disk.Cache.Size = d.Storage.Disk.Cache.Size + data.Storage.Disk.Cache.FileSize = d.Storage.Disk.Cache.FileSize + data.Storage.Disk.Cache.TTL = d.Storage.Disk.Cache.TTL + data.Storage.Disk.Cache.Types.Allow = copyStringSlice(d.Storage.Disk.Cache.Types) + data.Storage.Disk.Cache.Types.Block = []string{} + + data.Version = 3 + + return data, nil +} diff --git a/config/data_v1.go b/config/data_v1.go new file mode 100644 index 00000000..bfd77a64 --- /dev/null +++ b/config/data_v1.go @@ -0,0 +1,154 @@ +package config + +import "time" + +type dataV1 struct { + CreatedAt time.Time `json:"created_at"` + LoadedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` + Version int64 `json:"version" jsonschema:"minimum=1,maximum=1"` + ID string `json:"id"` + Name string `json:"name"` + Address string `json:"address"` + CheckForUpdates bool `json:"update_check"` + Log struct { + Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` + Topics []string `json:"topics"` + MaxLines int `json:"max_lines"` + } `json:"log"` + DB struct { + Dir string `json:"dir"` + } `json:"db"` + Host struct { + Name []string `json:"name"` + Auto bool `json:"auto"` + } `json:"host"` + API struct { + ReadOnly bool `json:"read_only"` + Access struct { + HTTP struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"http"` + HTTPS struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"https"` + } `json:"access"` + Auth struct { + Enable bool `json:"enable"` + DisableLocalhost bool `json:"disable_localhost"` + Username string `json:"username"` + Password string `json:"password"` + JWT struct { + Secret string `json:"secret"` + } `json:"jwt"` + Auth0 struct { + Enable bool `json:"enable"` + Tenants []Auth0Tenant `json:"tenants"` + } `json:"auth0"` + } `json:"auth"` + } `json:"api"` + TLS struct { + Address string `json:"address"` + Enable bool `json:"enable"` + Auto bool `json:"auto"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + } `json:"tls"` + Storage struct { + Disk struct { + Dir string `json:"dir"` + Size int64 `json:"max_size_mbytes"` + Cache struct { + Enable bool `json:"enable"` + Size uint64 `json:"max_size_mbytes"` + TTL int64 `json:"ttl_seconds"` + FileSize uint64 `json:"max_file_size_mbytes"` + Types []string `json:"types"` + } `json:"cache"` + } `json:"disk"` + Memory struct { + Auth struct { + Enable bool `json:"enable"` + Username string `json:"username"` + Password string `json:"password"` + } `json:"auth"` + Size int64 `json:"max_size_mbytes"` + Purge bool `json:"purge"` + } `json:"memory"` + CORS struct { + Origins []string `json:"origins"` + } `json:"cors"` + MimeTypes string `json:"mimetypes_file"` + } `json:"storage"` + RTMP struct { + Enable bool `json:"enable"` + EnableTLS bool `json:"enable_tls"` + Address string `json:"address"` + App string `json:"app"` + Token string `json:"token"` + } `json:"rtmp"` + SRT struct { + Enable bool `json:"enable"` + Address string `json:"address"` + Passphrase string `json:"passphrase"` + Token string `json:"token"` + Log struct { + Enable bool `json:"enable"` + Topics []string `json:"topics"` + } `json:"log"` + } `json:"srt"` + FFmpeg struct { + Binary string `json:"binary"` + MaxProcesses int64 `json:"max_processes"` + Access struct { + Input struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"input"` + Output struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"output"` + } `json:"access"` + Log struct { + MaxLines int `json:"max_lines"` + MaxHistory int `json:"max_history"` + } `json:"log"` + } `json:"ffmpeg"` + Playout struct { + Enable bool `json:"enable"` + MinPort int `json:"min_port"` + MaxPort int `json:"max_port"` + } `json:"playout"` + Debug struct { + Profiling bool `json:"profiling"` + ForceGC int `json:"force_gc"` + } `json:"debug"` + Metrics struct { + Enable bool `json:"enable"` + EnablePrometheus bool `json:"enable_prometheus"` + Range int64 `json:"range_sec"` // seconds + Interval int64 `json:"interval_sec"` // seconds + } `json:"metrics"` + Sessions struct { + Enable bool `json:"enable"` + IPIgnoreList []string `json:"ip_ignorelist"` + SessionTimeout int `json:"session_timeout_sec"` + Persist bool `json:"persist"` + PersistInterval int `json:"persist_interval_sec"` + MaxBitrate uint64 `json:"max_bitrate_mbit"` + MaxSessions uint64 `json:"max_sessions"` + } `json:"sessions"` + Service struct { + Enable bool `json:"enable"` + Token string `json:"token"` + URL string `json:"url"` + } `json:"service"` + Router struct { + BlockedPrefixes []string `json:"blocked_prefixes"` + Routes map[string]string `json:"routes"` + UIPath string `json:"ui_path"` + } `json:"router"` +} diff --git a/config/data_v2.go b/config/data_v2.go new file mode 100644 index 00000000..0249429d --- /dev/null +++ b/config/data_v2.go @@ -0,0 +1,247 @@ +package config + +import ( + "fmt" + "net" + "strconv" + "strings" + "time" +) + +type dataV2 struct { + CreatedAt time.Time `json:"created_at"` + LoadedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` + Version int64 `json:"version" jsonschema:"minimum=2,maximum=2"` + ID string `json:"id"` + Name string `json:"name"` + Address string `json:"address"` + CheckForUpdates bool `json:"update_check"` + Log struct { + Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` + Topics []string `json:"topics"` + MaxLines int `json:"max_lines"` + } `json:"log"` + DB struct { + Dir string `json:"dir"` + } `json:"db"` + Host struct { + Name []string `json:"name"` + Auto bool `json:"auto"` + } `json:"host"` + API struct { + ReadOnly bool `json:"read_only"` + Access struct { + HTTP struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"http"` + HTTPS struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"https"` + } `json:"access"` + Auth struct { + Enable bool `json:"enable"` + DisableLocalhost bool `json:"disable_localhost"` + Username string `json:"username"` + Password string `json:"password"` + JWT struct { + Secret string `json:"secret"` + } `json:"jwt"` + Auth0 struct { + Enable bool `json:"enable"` + Tenants []Auth0Tenant `json:"tenants"` + } `json:"auth0"` + } `json:"auth"` + } `json:"api"` + TLS struct { + Address string `json:"address"` + Enable bool `json:"enable"` + Auto bool `json:"auto"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + } `json:"tls"` + Storage struct { + Disk struct { + Dir string `json:"dir"` + Size int64 `json:"max_size_mbytes"` + Cache struct { + Enable bool `json:"enable"` + Size uint64 `json:"max_size_mbytes"` + TTL int64 `json:"ttl_seconds"` + FileSize uint64 `json:"max_file_size_mbytes"` + Types []string `json:"types"` + } `json:"cache"` + } `json:"disk"` + Memory struct { + Auth struct { + Enable bool `json:"enable"` + Username string `json:"username"` + Password string `json:"password"` + } `json:"auth"` + Size int64 `json:"max_size_mbytes"` + Purge bool `json:"purge"` + } `json:"memory"` + CORS struct { + Origins []string `json:"origins"` + } `json:"cors"` + MimeTypes string `json:"mimetypes_file"` + } `json:"storage"` + RTMP struct { + Enable bool `json:"enable"` + EnableTLS bool `json:"enable_tls"` + Address string `json:"address"` + AddressTLS string `json:"address_tls"` + App string `json:"app"` + Token string `json:"token"` + } `json:"rtmp"` + SRT struct { + Enable bool `json:"enable"` + Address string `json:"address"` + Passphrase string `json:"passphrase"` + Token string `json:"token"` + Log struct { + Enable bool `json:"enable"` + Topics []string `json:"topics"` + } `json:"log"` + } `json:"srt"` + FFmpeg struct { + Binary string `json:"binary"` + MaxProcesses int64 `json:"max_processes"` + Access struct { + Input struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"input"` + Output struct { + Allow []string `json:"allow"` + Block []string `json:"block"` + } `json:"output"` + } `json:"access"` + Log struct { + MaxLines int `json:"max_lines"` + MaxHistory int `json:"max_history"` + } `json:"log"` + } `json:"ffmpeg"` + Playout struct { + Enable bool `json:"enable"` + MinPort int `json:"min_port"` + MaxPort int `json:"max_port"` + } `json:"playout"` + Debug struct { + Profiling bool `json:"profiling"` + ForceGC int `json:"force_gc"` + } `json:"debug"` + Metrics struct { + Enable bool `json:"enable"` + EnablePrometheus bool `json:"enable_prometheus"` + Range int64 `json:"range_sec"` // seconds + Interval int64 `json:"interval_sec"` // seconds + } `json:"metrics"` + Sessions struct { + Enable bool `json:"enable"` + IPIgnoreList []string `json:"ip_ignorelist"` + SessionTimeout int `json:"session_timeout_sec"` + Persist bool `json:"persist"` + PersistInterval int `json:"persist_interval_sec"` + MaxBitrate uint64 `json:"max_bitrate_mbit"` + MaxSessions uint64 `json:"max_sessions"` + } `json:"sessions"` + Service struct { + Enable bool `json:"enable"` + Token string `json:"token"` + URL string `json:"url"` + } `json:"service"` + Router struct { + BlockedPrefixes []string `json:"blocked_prefixes"` + Routes map[string]string `json:"routes"` + UIPath string `json:"ui_path"` + } `json:"router"` +} + +// Migrate will migrate some settings, depending on the version it finds. Migrations +// are only going upwards,i.e. from a lower version to a higher version. +func NewV2FromV1(d *dataV1) (*dataV2, error) { + data := &dataV2{} + + data.CreatedAt = d.CreatedAt + data.LoadedAt = d.LoadedAt + data.UpdatedAt = d.UpdatedAt + + data.ID = d.ID + data.Name = d.Name + data.Address = d.Address + data.CheckForUpdates = d.CheckForUpdates + + data.Log = d.Log + data.DB = d.DB + data.Host = d.Host + data.API = d.API + data.TLS = d.TLS + data.Storage = d.Storage + data.SRT = d.SRT + data.FFmpeg = d.FFmpeg + data.Playout = d.Playout + data.Debug = d.Debug + data.Metrics = d.Metrics + data.Sessions = d.Sessions + data.Service = d.Service + data.Router = d.Router + + data.Log.Topics = copyStringSlice(d.Log.Topics) + + data.Host.Name = copyStringSlice(d.Host.Name) + + data.API.Access.HTTP.Allow = copyStringSlice(d.API.Access.HTTP.Allow) + data.API.Access.HTTP.Block = copyStringSlice(d.API.Access.HTTP.Block) + data.API.Access.HTTPS.Allow = copyStringSlice(d.API.Access.HTTPS.Allow) + data.API.Access.HTTPS.Block = copyStringSlice(d.API.Access.HTTPS.Block) + + data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants) + + data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins) + + data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow) + data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block) + data.FFmpeg.Access.Output.Allow = copyStringSlice(d.FFmpeg.Access.Output.Allow) + data.FFmpeg.Access.Output.Block = copyStringSlice(d.FFmpeg.Access.Output.Block) + + data.Sessions.IPIgnoreList = copyStringSlice(d.Sessions.IPIgnoreList) + + data.SRT.Log.Topics = copyStringSlice(d.SRT.Log.Topics) + + data.Router.BlockedPrefixes = copyStringSlice(d.Router.BlockedPrefixes) + data.Router.Routes = copyStringMap(d.Router.Routes) + + // Actual changes + data.RTMP.Enable = d.RTMP.Enable + data.RTMP.EnableTLS = d.RTMP.EnableTLS + data.RTMP.Address = d.RTMP.Address + data.RTMP.App = d.RTMP.App + data.RTMP.Token = d.RTMP.Token + + if !strings.HasPrefix(data.RTMP.App, "/") { + data.RTMP.App = "/" + data.RTMP.App + } + + if d.RTMP.EnableTLS { + data.RTMP.Enable = true + data.RTMP.AddressTLS = data.RTMP.Address + host, sport, err := net.SplitHostPort(data.RTMP.Address) + if err != nil { + return nil, fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err) + } + + port, err := strconv.Atoi(sport) + if err != nil { + return nil, fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err) + } + + data.RTMP.Address = net.JoinHostPort(host, strconv.Itoa(port-1)) + } + + data.Version = 2 + + return data, nil +} diff --git a/config/json.go b/config/json.go index 1f6aabe7..3b185d0e 100644 --- a/config/json.go +++ b/config/json.go @@ -3,7 +3,6 @@ package config import ( gojson "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "time" @@ -102,7 +101,7 @@ func (c *jsonStore) Reload() error { return nil } -func (c *jsonStore) load(data *Config) error { +func (c *jsonStore) load(config *Config) error { if len(c.path) == 0 { return nil } @@ -111,17 +110,56 @@ func (c *jsonStore) load(data *Config) error { return nil } - jsondata, err := ioutil.ReadFile(c.path) + jsondata, err := os.ReadFile(c.path) if err != nil { return err } - if err = gojson.Unmarshal(jsondata, data); err != nil { + dataV3 := &Data{} + + version := DataVersion{} + + if err = gojson.Unmarshal(jsondata, &version); err != nil { return json.FormatError(jsondata, err) } - data.LoadedAt = time.Now() - data.UpdatedAt = data.LoadedAt + if version.Version == 1 { + dataV1 := &dataV1{} + + if err = gojson.Unmarshal(jsondata, dataV1); err != nil { + return json.FormatError(jsondata, err) + } + + dataV2, err := NewV2FromV1(dataV1) + if err != nil { + return err + } + + dataV3, err = NewV3FromV2(dataV2) + if err != nil { + return err + } + } else if version.Version == 2 { + dataV2 := &dataV2{} + + if err = gojson.Unmarshal(jsondata, dataV2); err != nil { + return json.FormatError(jsondata, err) + } + + dataV3, err = NewV3FromV2(dataV2) + if err != nil { + return err + } + } else if version.Version == 3 { + if err = gojson.Unmarshal(jsondata, dataV3); err != nil { + return json.FormatError(jsondata, err) + } + } + + config.Data = *dataV3 + + config.LoadedAt = time.Now() + config.UpdatedAt = config.LoadedAt return nil } @@ -140,7 +178,7 @@ func (c *jsonStore) store(data *Config) error { dir, filename := filepath.Split(c.path) - tmpfile, err := ioutil.TempFile(dir, filename) + tmpfile, err := os.CreateTemp(dir, filename) if err != nil { return err } diff --git a/http/cache/lru.go b/http/cache/lru.go index 67764438..a5b1f93a 100644 --- a/http/cache/lru.go +++ b/http/cache/lru.go @@ -11,23 +11,25 @@ import ( // LRUConfig is the configuration for a new LRU cache type LRUConfig struct { - TTL time.Duration // For how long the object should stay in cache - MaxSize uint64 // Max. size of the cache, 0 for unlimited, bytes - MaxFileSize uint64 // Max. file size allowed to put in cache, 0 for unlimited, bytes - Extensions []string // List of file extension allowed to cache, empty list for all files - Logger log.Logger + TTL time.Duration // For how long the object should stay in cache + MaxSize uint64 // Max. size of the cache, 0 for unlimited, bytes + MaxFileSize uint64 // Max. file size allowed to put in cache, 0 for unlimited, bytes + AllowExtensions []string // List of file extension allowed to cache, empty list for all files + BlockExtensions []string // List of file extensions not allowed to cache, empty list for none + Logger log.Logger } type lrucache struct { - ttl time.Duration - maxSize uint64 - maxFileSize uint64 - extensions []string - objects map[string]*list.Element - list *list.List - size uint64 - lock sync.Mutex - logger log.Logger + ttl time.Duration + maxSize uint64 + maxFileSize uint64 + allowExtensions []string + blockExtensions []string + objects map[string]*list.Element + list *list.List + size uint64 + lock sync.Mutex + logger log.Logger } type value struct { @@ -53,11 +55,14 @@ func NewLRUCache(config LRUConfig) (Cacher, error) { } if cache.logger == nil { - cache.logger = log.New("HTTPCache") + cache.logger = log.New("") } - cache.extensions = make([]string, len(config.Extensions)) - copy(cache.extensions, config.Extensions) + cache.allowExtensions = make([]string, len(config.AllowExtensions)) + copy(cache.allowExtensions, config.AllowExtensions) + + cache.blockExtensions = make([]string, len(config.BlockExtensions)) + copy(cache.blockExtensions, config.BlockExtensions) return cache, nil } @@ -199,19 +204,27 @@ func (c *lrucache) TTL() time.Duration { } func (c *lrucache) IsExtensionCacheable(extension string) bool { - if len(c.extensions) == 0 { + if len(c.allowExtensions) == 0 && len(c.blockExtensions) == 0 { return true } - cacheable := false - for _, e := range c.extensions { + for _, e := range c.blockExtensions { if extension == e { - cacheable = true - break + return false } } - return cacheable + if len(c.allowExtensions) == 0 { + return true + } + + for _, e := range c.allowExtensions { + if extension == e { + return true + } + } + + return false } func (c *lrucache) IsSizeCacheable(size uint64) bool { diff --git a/http/cache/lru_test.go b/http/cache/lru_test.go index 3a402bf2..5b1c5f67 100644 --- a/http/cache/lru_test.go +++ b/http/cache/lru_test.go @@ -8,11 +8,12 @@ import ( ) var defaultConfig = LRUConfig{ - TTL: time.Hour, - MaxSize: 128, - MaxFileSize: 0, - Extensions: []string{".html", ".js", ".jpg"}, - Logger: nil, + TTL: time.Hour, + MaxSize: 128, + MaxFileSize: 0, + AllowExtensions: []string{".html", ".js", ".jpg"}, + BlockExtensions: []string{".m3u8"}, + Logger: nil, } func getCache(t *testing.T) *lrucache { @@ -27,8 +28,6 @@ func TestNew(t *testing.T) { TTL: time.Hour, MaxSize: 128, MaxFileSize: 129, - Extensions: []string{}, - Logger: nil, }) require.NotEqual(t, nil, err) @@ -36,8 +35,6 @@ func TestNew(t *testing.T) { TTL: time.Hour, MaxSize: 0, MaxFileSize: 129, - Extensions: []string{}, - Logger: nil, }) require.Equal(t, nil, err) @@ -45,8 +42,6 @@ func TestNew(t *testing.T) { TTL: time.Hour, MaxSize: 128, MaxFileSize: 127, - Extensions: []string{}, - Logger: nil, }) require.Equal(t, nil, err) } @@ -144,7 +139,7 @@ func TestLRU(t *testing.T) { require.NotEqual(t, nil, data) } -func TestExtension(t *testing.T) { +func TestAllowExtension(t *testing.T) { cache := getCache(t) r := cache.IsExtensionCacheable(".html") @@ -154,6 +149,17 @@ func TestExtension(t *testing.T) { require.Equal(t, false, r) } +func TestBlockExtension(t *testing.T) { + cache := getCache(t) + cache.allowExtensions = []string{} + + r := cache.IsExtensionCacheable(".html") + require.Equal(t, true, r) + + r = cache.IsExtensionCacheable(".m3u8") + require.Equal(t, false, r) +} + func TestSize(t *testing.T) { cache := getCache(t)