apps/core
1
0
Fork 0
mirror of https://github.com/datarhei/core.git synced 2025-09-26 20:11:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add cache block list for extensions not to cache

Browse Source
This commit is contained in:
Ingo Oppermann
2022-08-02 19:10:28 +02:00
parent 6af226aea7
commit 273ca0abbc
12 changed files with 760 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -1,7 +1,8 @@
# Core # Core
The cloud-native audio/video processing API. The cloud-native audio/video processing API.
[![License: MIT](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)]([https://opensource.org/licenses/MI](https://www.apache.org/licenses/LICENSE-2.0)) [![License: MIT](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)](<[https://opensource.org/licenses/MI](https://www.apache.org/licenses/LICENSE-2.0)>)
[![CodeQL](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml) [![CodeQL](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/codeql-analysis.yml)
[![tests](https://github.com/datarhei/core/actions/workflows/go-tests.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/go-tests.yml) [![tests](https://github.com/datarhei/core/actions/workflows/go-tests.yml/badge.svg)](https://github.com/datarhei/core/actions/workflows/go-tests.yml)
[![codecov](https://codecov.io/gh/datarhei/core/branch/main/graph/badge.svg?token=90YMPZRAFK)](https://codecov.io/gh/datarhei/core) [![codecov](https://codecov.io/gh/datarhei/core/branch/main/graph/badge.svg?token=90YMPZRAFK)](https://codecov.io/gh/datarhei/core)
@@ -119,7 +120,8 @@ The currently known environment variables (but not all will be respected) are:
| CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES | `0` | Max. allowed cache size, 0 for unlimited. | | CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES | `0` | Max. allowed cache size, 0 for unlimited. |
| CORE_STORAGE_DISK_CACHE_TTLSECONDS | `300` | Seconds to keep files in cache. | | CORE_STORAGE_DISK_CACHE_TTLSECONDS | `300` | Seconds to keep files in cache. |
| CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES | `1` | Max. file size to put in cache. | | CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES | `1` | Max. file size to put in cache. |
| CORE_STORAGE_DISK_CACHE_TYPES | (not set) | List of file extensions to cache (space-separated, e.g. ".html .js"), empty for all. | | CORE_STORAGE_DISK_CACHE_TYPES_ALLOW | (not set) | List of file extensions to cache (space-separated, e.g. ".html .js"), empty for all. |
| CORE_STORAGE_DISK_CACHE_TYPES_BLOCK | (not set) | List of file extensions not to cache (space-separated, e.g. ".m3u8 .mpd"), empty for none. |
| CORE_STORAGE_MEMORY_AUTH_ENABLE | `true` | Enable basic auth for PUT,POST, and DELETE on /memfs. | | CORE_STORAGE_MEMORY_AUTH_ENABLE | `true` | Enable basic auth for PUT,POST, and DELETE on /memfs. |
| CORE_STORAGE_MEMORY_AUTH_USERNAME | (not set) | Username for Basic-Auth of `/memfs`. Required if auth is enabled. | | CORE_STORAGE_MEMORY_AUTH_USERNAME | (not set) | Username for Basic-Auth of `/memfs`. Required if auth is enabled. |
| CORE_STORAGE_MEMORY_AUTH_PASSWORD | (not set) | Password for Basic-Auth of `/memfs`. Required if auth is enabled. | | CORE_STORAGE_MEMORY_AUTH_PASSWORD | (not set) | Password for Basic-Auth of `/memfs`. Required if auth is enabled. |
@@ -180,7 +182,7 @@ All other values will be filled with default values and persisted on disk. The e
``` ```
{ {
"version": 1, "version": 3,
"id": "[will be generated if not given]", "id": "[will be generated if not given]",
"name": "[will be generated if not given]", "name": "[will be generated if not given]",
"address": ":8080", "address": ":8080",
@@ -238,7 +240,10 @@ All other values will be filled with default values and persisted on disk. The e
"max_size_mbytes": 0, "max_size_mbytes": 0,
"ttl_seconds": 300, "ttl_seconds": 300,
"max_file_size_mbytes": 1, "max_file_size_mbytes": 1,
"types": [] "types": {
"allow": [],
"block": []
}
} }
}, },
"memory": { "memory": {

16
app/api/api.go
View File

@@ -154,11 +154,6 @@ func (a *api) Reload() error {
} }
cfg := store.Get() cfg := store.Get()
if err := cfg.Migrate(); err == nil {
store.Set(cfg)
} else {
return err
}
cfg.Merge() cfg.Merge()
@@ -631,11 +626,12 @@ func (a *api) start() error {
if cfg.Storage.Disk.Cache.Enable { if cfg.Storage.Disk.Cache.Enable {
diskCache, err := cache.NewLRUCache(cache.LRUConfig{ diskCache, err := cache.NewLRUCache(cache.LRUConfig{
TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second, TTL: time.Duration(cfg.Storage.Disk.Cache.TTL) * time.Second,
MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024, MaxSize: cfg.Storage.Disk.Cache.Size * 1024 * 1024,
MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024, MaxFileSize: cfg.Storage.Disk.Cache.FileSize * 1024 * 1024,
Extensions: cfg.Storage.Disk.Cache.Types, AllowExtensions: cfg.Storage.Disk.Cache.Types.Allow,
Logger: a.log.logger.core.WithComponent("HTTPCache"), BlockExtensions: cfg.Storage.Disk.Cache.Types.Block,
Logger: a.log.logger.core.WithComponent("HTTPCache"),
}) })
if err != nil { if err != nil {

2
app/import/main.go
View File

@@ -33,7 +33,6 @@ func doImport(logger log.Logger, configstore config.Store) error {
logger.Info().Log("Database import") logger.Info().Log("Database import")
cfg := configstore.Get() cfg := configstore.Get()
cfg.Migrate()
// Merging the persisted config with the environment variables // Merging the persisted config with the environment variables
cfg.Merge() cfg.Merge()
@@ -117,7 +116,6 @@ func doImport(logger log.Logger, configstore config.Store) error {
// Get the unmerged config for persisting // Get the unmerged config for persisting
cfg = configstore.Get() cfg = configstore.Get()
cfg.Migrate()
// Add static routes to mimic the old URLs // Add static routes to mimic the old URLs
cfg.Router.Routes["/hls/live.stream.m3u8"] = "/memfs/" + importConfig.id + ".m3u8" cfg.Router.Routes["/hls/live.stream.m3u8"] = "/memfs/" + importConfig.id + ".m3u8"

2
app/import/main_test.go
View File

@@ -11,8 +11,6 @@ func TestImport(t *testing.T) {
configstore := config.NewDummyStore() configstore := config.NewDummyStore()
cfg := configstore.Get() cfg := configstore.Get()
cfg.Version = 1
cfg.Migrate()
err := configstore.Set(cfg) err := configstore.Set(cfg)
require.NoError(t, err) require.NoError(t, err)

4
app/version.go
View File

@@ -29,8 +29,8 @@ func (v versionInfo) MinorString() string {
// Version of the app // Version of the app
var Version = versionInfo{ var Version = versionInfo{
Major: 16, Major: 16,
Minor: 9, Minor: 10,
Patch: 1, Patch: 0,
} }
// Commit is the git commit the app is build from. It should be filled in during compilation // Commit is the git commit the app is build from. It should be filled in during compilation

198
config/config.go
View File

@@ -6,8 +6,6 @@ import (
"fmt" "fmt"
"net" "net"
"os" "os"
"strconv"
"strings"
"time" "time"
"github.com/datarhei/core/v16/math/rand" "github.com/datarhei/core/v16/math/rand"
@@ -16,7 +14,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
) )
const version int64 = 2 const version int64 = 3
type variable struct { type variable struct {
value value // The actual value value value // The actual value
@@ -51,157 +49,8 @@ type Auth0Tenant struct {
Users []string `json:"users"` Users []string `json:"users"`
} }
// Data is the actual configuration data for the app type DataVersion struct {
type Data struct { Version int64 `json:"version"`
CreatedAt time.Time `json:"created_at"`
LoadedAt time.Time `json:"-"`
UpdatedAt time.Time `json:"-"`
Version int64 `json:"version" jsonschema:"minimum=1,maximum=1"`
ID string `json:"id"`
Name string `json:"name"`
Address string `json:"address"`
CheckForUpdates bool `json:"update_check"`
Log struct {
Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"`
Topics []string `json:"topics"`
MaxLines int `json:"max_lines"`
} `json:"log"`
DB struct {
Dir string `json:"dir"`
} `json:"db"`
Host struct {
Name []string `json:"name"`
Auto bool `json:"auto"`
} `json:"host"`
API struct {
ReadOnly bool `json:"read_only"`
Access struct {
HTTP struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"http"`
HTTPS struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"https"`
} `json:"access"`
Auth struct {
Enable bool `json:"enable"`
DisableLocalhost bool `json:"disable_localhost"`
Username string `json:"username"`
Password string `json:"password"`
JWT struct {
Secret string `json:"secret"`
} `json:"jwt"`
Auth0 struct {
Enable bool `json:"enable"`
Tenants []Auth0Tenant `json:"tenants"`
} `json:"auth0"`
} `json:"auth"`
} `json:"api"`
TLS struct {
Address string `json:"address"`
Enable bool `json:"enable"`
Auto bool `json:"auto"`
CertFile string `json:"cert_file"`
KeyFile string `json:"key_file"`
} `json:"tls"`
Storage struct {
Disk struct {
Dir string `json:"dir"`
Size int64 `json:"max_size_mbytes"`
Cache struct {
Enable bool `json:"enable"`
Size uint64 `json:"max_size_mbytes"`
TTL int64 `json:"ttl_seconds"`
FileSize uint64 `json:"max_file_size_mbytes"`
Types []string `json:"types"`
} `json:"cache"`
} `json:"disk"`
Memory struct {
Auth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
} `json:"auth"`
Size int64 `json:"max_size_mbytes"`
Purge bool `json:"purge"`
} `json:"memory"`
CORS struct {
Origins []string `json:"origins"`
} `json:"cors"`
MimeTypes string `json:"mimetypes_file"`
} `json:"storage"`
RTMP struct {
Enable bool `json:"enable"`
EnableTLS bool `json:"enable_tls"`
Address string `json:"address"`
AddressTLS string `json:"address_tls"`
App string `json:"app"`
Token string `json:"token"`
} `json:"rtmp"`
SRT struct {
Enable bool `json:"enable"`
Address string `json:"address"`
Passphrase string `json:"passphrase"`
Token string `json:"token"`
Log struct {
Enable bool `json:"enable"`
Topics []string `json:"topics"`
} `json:"log"`
} `json:"srt"`
FFmpeg struct {
Binary string `json:"binary"`
MaxProcesses int64 `json:"max_processes"`
Access struct {
Input struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"input"`
Output struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"output"`
} `json:"access"`
Log struct {
MaxLines int `json:"max_lines"`
MaxHistory int `json:"max_history"`
} `json:"log"`
} `json:"ffmpeg"`
Playout struct {
Enable bool `json:"enable"`
MinPort int `json:"min_port"`
MaxPort int `json:"max_port"`
} `json:"playout"`
Debug struct {
Profiling bool `json:"profiling"`
ForceGC int `json:"force_gc"`
} `json:"debug"`
Metrics struct {
Enable bool `json:"enable"`
EnablePrometheus bool `json:"enable_prometheus"`
Range int64 `json:"range_sec"` // seconds
Interval int64 `json:"interval_sec"` // seconds
} `json:"metrics"`
Sessions struct {
Enable bool `json:"enable"`
IPIgnoreList []string `json:"ip_ignorelist"`
SessionTimeout int `json:"session_timeout_sec"`
Persist bool `json:"persist"`
PersistInterval int `json:"persist_interval_sec"`
MaxBitrate uint64 `json:"max_bitrate_mbit"`
MaxSessions uint64 `json:"max_sessions"`
} `json:"sessions"`
Service struct {
Enable bool `json:"enable"`
Token string `json:"token"`
URL string `json:"url"`
} `json:"service"`
Router struct {
BlockedPrefixes []string `json:"blocked_prefixes"`
Routes map[string]string `json:"routes"`
UIPath string `json:"ui_path"`
} `json:"router"`
} }
// Config is a wrapper for Data // Config is a wrapper for Data
@@ -214,11 +63,11 @@ type Config struct {
// New returns a Config which is initialized with its default values // New returns a Config which is initialized with its default values
func New() *Config { func New() *Config {
data := &Config{} config := &Config{}
data.init() config.init()
return data return config
} }
// NewConfigFrom returns a clone of a Config // NewConfigFrom returns a clone of a Config
@@ -263,6 +112,8 @@ func NewConfigFrom(d *Config) *Config {
data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants) data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants)
data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins) data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins)
data.Storage.Disk.Cache.Types.Allow = copyStringSlice(d.Storage.Disk.Cache.Types.Allow)
data.Storage.Disk.Cache.Types.Block = copyStringSlice(d.Storage.Disk.Cache.Types.Block)
data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow) data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow)
data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block) data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block)
@@ -338,7 +189,8 @@ func (d *Config) init() {
d.val(newUint64Value(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false) d.val(newUint64Value(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false)
d.val(newInt64Value(&d.Storage.Disk.Cache.TTL, 300), "storage.disk.cache.ttl_seconds", "CORE_STORAGE_DISK_CACHE_TTLSECONDS", nil, "Seconds to keep files in cache", false, false) d.val(newInt64Value(&d.Storage.Disk.Cache.TTL, 300), "storage.disk.cache.ttl_seconds", "CORE_STORAGE_DISK_CACHE_TTLSECONDS", nil, "Seconds to keep files in cache", false, false)
d.val(newUint64Value(&d.Storage.Disk.Cache.FileSize, 1), "storage.disk.cache.max_file_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES", nil, "Max. file size to put in cache", false, false) d.val(newUint64Value(&d.Storage.Disk.Cache.FileSize, 1), "storage.disk.cache.max_file_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXFILESIZEMBYTES", nil, "Max. file size to put in cache", false, false)
d.val(newStringListValue(&d.Storage.Disk.Cache.Types, []string{}, " "), "storage.disk.cache.types", "CORE_STORAGE_DISK_CACHE_TYPES", nil, "File extensions to cache, empty for all", false, false) d.val(newStringListValue(&d.Storage.Disk.Cache.Types.Allow, []string{}, " "), "storage.disk.cache.type.allow", "CORE_STORAGE_DISK_CACHE_TYPES_ALLOW", []string{"CORE_STORAGE_DISK_CACHE_TYPES"}, "File extensions to cache, empty for all", false, false)
d.val(newStringListValue(&d.Storage.Disk.Cache.Types.Block, []string{}, " "), "storage.disk.cache.type.block", "CORE_STORAGE_DISK_CACHE_TYPES_BLOCK", nil, "File extensions not to cache, empty for none", false, false)
// Storage (Memory) // Storage (Memory)
d.val(newBoolValue(&d.Storage.Memory.Auth.Enable, true), "storage.memory.auth.enable", "CORE_STORAGE_MEMORY_AUTH_ENABLE", nil, "Enable basic auth for PUT,POST, and DELETE on /memfs", false, false) d.val(newBoolValue(&d.Storage.Memory.Auth.Enable, true), "storage.memory.auth.enable", "CORE_STORAGE_MEMORY_AUTH_ENABLE", nil, "Enable basic auth for PUT,POST, and DELETE on /memfs", false, false)
@@ -483,36 +335,6 @@ func (d *Config) Merge() {
} }
} }
// Migrate will migrate some settings, depending on the version it finds. Migrations
// are only going upwards,i.e. from a lower version to a higher version.
func (d *Config) Migrate() error {
if d.Version == 1 {
if !strings.HasPrefix(d.RTMP.App, "/") {
d.RTMP.App = "/" + d.RTMP.App
}
if d.RTMP.EnableTLS {
d.RTMP.Enable = true
d.RTMP.AddressTLS = d.RTMP.Address
host, sport, err := net.SplitHostPort(d.RTMP.Address)
if err != nil {
return fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err)
}
port, err := strconv.Atoi(sport)
if err != nil {
return fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err)
}
d.RTMP.Address = net.JoinHostPort(host, strconv.Itoa(port-1))
}
d.Version = 2
}
return nil
}
// Validate validates the current state of the Config for completeness and sanity. Errors are // Validate validates the current state of the Config for completeness and sanity. Errors are
// written to the log. Use resetLogs to indicate to reset the logs prior validation. // written to the log. Use resetLogs to indicate to reset the logs prior validation.
func (d *Config) Validate(resetLogs bool) { func (d *Config) Validate(resetLogs bool) {

233
config/data.go Normal file
View File

@@ -0,0 +1,233 @@
package config
import "time"
// Data is the actual configuration data for the app
type Data struct {
CreatedAt time.Time `json:"created_at"`
LoadedAt time.Time `json:"-"`
UpdatedAt time.Time `json:"-"`
Version int64 `json:"version" jsonschema:"minimum=3,maximum=3"`
ID string `json:"id"`
Name string `json:"name"`
Address string `json:"address"`
CheckForUpdates bool `json:"update_check"`
Log struct {
Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"`
Topics []string `json:"topics"`
MaxLines int `json:"max_lines"`
} `json:"log"`
DB struct {
Dir string `json:"dir"`
} `json:"db"`
Host struct {
Name []string `json:"name"`
Auto bool `json:"auto"`
} `json:"host"`
API struct {
ReadOnly bool `json:"read_only"`
Access struct {
HTTP struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"http"`
HTTPS struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"https"`
} `json:"access"`
Auth struct {
Enable bool `json:"enable"`
DisableLocalhost bool `json:"disable_localhost"`
Username string `json:"username"`
Password string `json:"password"`
JWT struct {
Secret string `json:"secret"`
} `json:"jwt"`
Auth0 struct {
Enable bool `json:"enable"`
Tenants []Auth0Tenant `json:"tenants"`
} `json:"auth0"`
} `json:"auth"`
} `json:"api"`
TLS struct {
Address string `json:"address"`
Enable bool `json:"enable"`
Auto bool `json:"auto"`
CertFile string `json:"cert_file"`
KeyFile string `json:"key_file"`
} `json:"tls"`
Storage struct {
Disk struct {
Dir string `json:"dir"`
Size int64 `json:"max_size_mbytes"`
Cache struct {
Enable bool `json:"enable"`
Size uint64 `json:"max_size_mbytes"`
TTL int64 `json:"ttl_seconds"`
FileSize uint64 `json:"max_file_size_mbytes"`
Types struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"types"`
} `json:"cache"`
} `json:"disk"`
Memory struct {
Auth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
} `json:"auth"`
Size int64 `json:"max_size_mbytes"`
Purge bool `json:"purge"`
} `json:"memory"`
CORS struct {
Origins []string `json:"origins"`
} `json:"cors"`
MimeTypes string `json:"mimetypes_file"`
} `json:"storage"`
RTMP struct {
Enable bool `json:"enable"`
EnableTLS bool `json:"enable_tls"`
Address string `json:"address"`
AddressTLS string `json:"address_tls"`
App string `json:"app"`
Token string `json:"token"`
} `json:"rtmp"`
SRT struct {
Enable bool `json:"enable"`
Address string `json:"address"`
Passphrase string `json:"passphrase"`
Token string `json:"token"`
Log struct {
Enable bool `json:"enable"`
Topics []string `json:"topics"`
} `json:"log"`
} `json:"srt"`
FFmpeg struct {
Binary string `json:"binary"`
MaxProcesses int64 `json:"max_processes"`
Access struct {
Input struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"input"`
Output struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"output"`
} `json:"access"`
Log struct {
MaxLines int `json:"max_lines"`
MaxHistory int `json:"max_history"`
} `json:"log"`
} `json:"ffmpeg"`
Playout struct {
Enable bool `json:"enable"`
MinPort int `json:"min_port"`
MaxPort int `json:"max_port"`
} `json:"playout"`
Debug struct {
Profiling bool `json:"profiling"`
ForceGC int `json:"force_gc"`
} `json:"debug"`
Metrics struct {
Enable bool `json:"enable"`
EnablePrometheus bool `json:"enable_prometheus"`
Range int64 `json:"range_sec"` // seconds
Interval int64 `json:"interval_sec"` // seconds
} `json:"metrics"`
Sessions struct {
Enable bool `json:"enable"`
IPIgnoreList []string `json:"ip_ignorelist"`
SessionTimeout int `json:"session_timeout_sec"`
Persist bool `json:"persist"`
PersistInterval int `json:"persist_interval_sec"`
MaxBitrate uint64 `json:"max_bitrate_mbit"`
MaxSessions uint64 `json:"max_sessions"`
} `json:"sessions"`
Service struct {
Enable bool `json:"enable"`
Token string `json:"token"`
URL string `json:"url"`
} `json:"service"`
Router struct {
BlockedPrefixes []string `json:"blocked_prefixes"`
Routes map[string]string `json:"routes"`
UIPath string `json:"ui_path"`
} `json:"router"`
}
func NewV3FromV2(d *dataV2) (*Data, error) {
data := &Data{}
data.CreatedAt = d.CreatedAt
data.LoadedAt = d.LoadedAt
data.UpdatedAt = d.UpdatedAt
data.ID = d.ID
data.Name = d.Name
data.Address = d.Address
data.CheckForUpdates = d.CheckForUpdates
data.Log = d.Log
data.DB = d.DB
data.Host = d.Host
data.API = d.API
data.TLS = d.TLS
data.RTMP = d.RTMP
data.SRT = d.SRT
data.FFmpeg = d.FFmpeg
data.Playout = d.Playout
data.Debug = d.Debug
data.Metrics = d.Metrics
data.Sessions = d.Sessions
data.Service = d.Service
data.Router = d.Router
data.Log.Topics = copyStringSlice(d.Log.Topics)
data.Host.Name = copyStringSlice(d.Host.Name)
data.API.Access.HTTP.Allow = copyStringSlice(d.API.Access.HTTP.Allow)
data.API.Access.HTTP.Block = copyStringSlice(d.API.Access.HTTP.Block)
data.API.Access.HTTPS.Allow = copyStringSlice(d.API.Access.HTTPS.Allow)
data.API.Access.HTTPS.Block = copyStringSlice(d.API.Access.HTTPS.Block)
data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants)
data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins)
data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow)
data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block)
data.FFmpeg.Access.Output.Allow = copyStringSlice(d.FFmpeg.Access.Output.Allow)
data.FFmpeg.Access.Output.Block = copyStringSlice(d.FFmpeg.Access.Output.Block)
data.Sessions.IPIgnoreList = copyStringSlice(d.Sessions.IPIgnoreList)
data.SRT.Log.Topics = copyStringSlice(d.SRT.Log.Topics)
data.Router.BlockedPrefixes = copyStringSlice(d.Router.BlockedPrefixes)
data.Router.Routes = copyStringMap(d.Router.Routes)
// Actual changes
data.Storage.MimeTypes = d.Storage.MimeTypes
data.Storage.CORS = d.Storage.CORS
data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins)
data.Storage.Memory = d.Storage.Memory
data.Storage.Disk.Dir = d.Storage.Disk.Dir
data.Storage.Disk.Size = d.Storage.Disk.Size
data.Storage.Disk.Cache.Enable = d.Storage.Disk.Cache.Enable
data.Storage.Disk.Cache.Size = d.Storage.Disk.Cache.Size
data.Storage.Disk.Cache.FileSize = d.Storage.Disk.Cache.FileSize
data.Storage.Disk.Cache.TTL = d.Storage.Disk.Cache.TTL
data.Storage.Disk.Cache.Types.Allow = copyStringSlice(d.Storage.Disk.Cache.Types)
data.Storage.Disk.Cache.Types.Block = []string{}
data.Version = 3
return data, nil
}

154
config/data_v1.go Normal file
View File

@@ -0,0 +1,154 @@
package config
import "time"
type dataV1 struct {
CreatedAt time.Time `json:"created_at"`
LoadedAt time.Time `json:"-"`
UpdatedAt time.Time `json:"-"`
Version int64 `json:"version" jsonschema:"minimum=1,maximum=1"`
ID string `json:"id"`
Name string `json:"name"`
Address string `json:"address"`
CheckForUpdates bool `json:"update_check"`
Log struct {
Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"`
Topics []string `json:"topics"`
MaxLines int `json:"max_lines"`
} `json:"log"`
DB struct {
Dir string `json:"dir"`
} `json:"db"`
Host struct {
Name []string `json:"name"`
Auto bool `json:"auto"`
} `json:"host"`
API struct {
ReadOnly bool `json:"read_only"`
Access struct {
HTTP struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"http"`
HTTPS struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"https"`
} `json:"access"`
Auth struct {
Enable bool `json:"enable"`
DisableLocalhost bool `json:"disable_localhost"`
Username string `json:"username"`
Password string `json:"password"`
JWT struct {
Secret string `json:"secret"`
} `json:"jwt"`
Auth0 struct {
Enable bool `json:"enable"`
Tenants []Auth0Tenant `json:"tenants"`
} `json:"auth0"`
} `json:"auth"`
} `json:"api"`
TLS struct {
Address string `json:"address"`
Enable bool `json:"enable"`
Auto bool `json:"auto"`
CertFile string `json:"cert_file"`
KeyFile string `json:"key_file"`
} `json:"tls"`
Storage struct {
Disk struct {
Dir string `json:"dir"`
Size int64 `json:"max_size_mbytes"`
Cache struct {
Enable bool `json:"enable"`
Size uint64 `json:"max_size_mbytes"`
TTL int64 `json:"ttl_seconds"`
FileSize uint64 `json:"max_file_size_mbytes"`
Types []string `json:"types"`
} `json:"cache"`
} `json:"disk"`
Memory struct {
Auth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
} `json:"auth"`
Size int64 `json:"max_size_mbytes"`
Purge bool `json:"purge"`
} `json:"memory"`
CORS struct {
Origins []string `json:"origins"`
} `json:"cors"`
MimeTypes string `json:"mimetypes_file"`
} `json:"storage"`
RTMP struct {
Enable bool `json:"enable"`
EnableTLS bool `json:"enable_tls"`
Address string `json:"address"`
App string `json:"app"`
Token string `json:"token"`
} `json:"rtmp"`
SRT struct {
Enable bool `json:"enable"`
Address string `json:"address"`
Passphrase string `json:"passphrase"`
Token string `json:"token"`
Log struct {
Enable bool `json:"enable"`
Topics []string `json:"topics"`
} `json:"log"`
} `json:"srt"`
FFmpeg struct {
Binary string `json:"binary"`
MaxProcesses int64 `json:"max_processes"`
Access struct {
Input struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"input"`
Output struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"output"`
} `json:"access"`
Log struct {
MaxLines int `json:"max_lines"`
MaxHistory int `json:"max_history"`
} `json:"log"`
} `json:"ffmpeg"`
Playout struct {
Enable bool `json:"enable"`
MinPort int `json:"min_port"`
MaxPort int `json:"max_port"`
} `json:"playout"`
Debug struct {
Profiling bool `json:"profiling"`
ForceGC int `json:"force_gc"`
} `json:"debug"`
Metrics struct {
Enable bool `json:"enable"`
EnablePrometheus bool `json:"enable_prometheus"`
Range int64 `json:"range_sec"` // seconds
Interval int64 `json:"interval_sec"` // seconds
} `json:"metrics"`
Sessions struct {
Enable bool `json:"enable"`
IPIgnoreList []string `json:"ip_ignorelist"`
SessionTimeout int `json:"session_timeout_sec"`
Persist bool `json:"persist"`
PersistInterval int `json:"persist_interval_sec"`
MaxBitrate uint64 `json:"max_bitrate_mbit"`
MaxSessions uint64 `json:"max_sessions"`
} `json:"sessions"`
Service struct {
Enable bool `json:"enable"`
Token string `json:"token"`
URL string `json:"url"`
} `json:"service"`
Router struct {
BlockedPrefixes []string `json:"blocked_prefixes"`
Routes map[string]string `json:"routes"`
UIPath string `json:"ui_path"`
} `json:"router"`
}

247
config/data_v2.go Normal file
View File

@@ -0,0 +1,247 @@
package config
import (
"fmt"
"net"
"strconv"
"strings"
"time"
)
type dataV2 struct {
CreatedAt time.Time `json:"created_at"`
LoadedAt time.Time `json:"-"`
UpdatedAt time.Time `json:"-"`
Version int64 `json:"version" jsonschema:"minimum=2,maximum=2"`
ID string `json:"id"`
Name string `json:"name"`
Address string `json:"address"`
CheckForUpdates bool `json:"update_check"`
Log struct {
Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"`
Topics []string `json:"topics"`
MaxLines int `json:"max_lines"`
} `json:"log"`
DB struct {
Dir string `json:"dir"`
} `json:"db"`
Host struct {
Name []string `json:"name"`
Auto bool `json:"auto"`
} `json:"host"`
API struct {
ReadOnly bool `json:"read_only"`
Access struct {
HTTP struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"http"`
HTTPS struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"https"`
} `json:"access"`
Auth struct {
Enable bool `json:"enable"`
DisableLocalhost bool `json:"disable_localhost"`
Username string `json:"username"`
Password string `json:"password"`
JWT struct {
Secret string `json:"secret"`
} `json:"jwt"`
Auth0 struct {
Enable bool `json:"enable"`
Tenants []Auth0Tenant `json:"tenants"`
} `json:"auth0"`
} `json:"auth"`
} `json:"api"`
TLS struct {
Address string `json:"address"`
Enable bool `json:"enable"`
Auto bool `json:"auto"`
CertFile string `json:"cert_file"`
KeyFile string `json:"key_file"`
} `json:"tls"`
Storage struct {
Disk struct {
Dir string `json:"dir"`
Size int64 `json:"max_size_mbytes"`
Cache struct {
Enable bool `json:"enable"`
Size uint64 `json:"max_size_mbytes"`
TTL int64 `json:"ttl_seconds"`
FileSize uint64 `json:"max_file_size_mbytes"`
Types []string `json:"types"`
} `json:"cache"`
} `json:"disk"`
Memory struct {
Auth struct {
Enable bool `json:"enable"`
Username string `json:"username"`
Password string `json:"password"`
} `json:"auth"`
Size int64 `json:"max_size_mbytes"`
Purge bool `json:"purge"`
} `json:"memory"`
CORS struct {
Origins []string `json:"origins"`
} `json:"cors"`
MimeTypes string `json:"mimetypes_file"`
} `json:"storage"`
RTMP struct {
Enable bool `json:"enable"`
EnableTLS bool `json:"enable_tls"`
Address string `json:"address"`
AddressTLS string `json:"address_tls"`
App string `json:"app"`
Token string `json:"token"`
} `json:"rtmp"`
SRT struct {
Enable bool `json:"enable"`
Address string `json:"address"`
Passphrase string `json:"passphrase"`
Token string `json:"token"`
Log struct {
Enable bool `json:"enable"`
Topics []string `json:"topics"`
} `json:"log"`
} `json:"srt"`
FFmpeg struct {
Binary string `json:"binary"`
MaxProcesses int64 `json:"max_processes"`
Access struct {
Input struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"input"`
Output struct {
Allow []string `json:"allow"`
Block []string `json:"block"`
} `json:"output"`
} `json:"access"`
Log struct {
MaxLines int `json:"max_lines"`
MaxHistory int `json:"max_history"`
} `json:"log"`
} `json:"ffmpeg"`
Playout struct {
Enable bool `json:"enable"`
MinPort int `json:"min_port"`
MaxPort int `json:"max_port"`
} `json:"playout"`
Debug struct {
Profiling bool `json:"profiling"`
ForceGC int `json:"force_gc"`
} `json:"debug"`
Metrics struct {
Enable bool `json:"enable"`
EnablePrometheus bool `json:"enable_prometheus"`
Range int64 `json:"range_sec"` // seconds
Interval int64 `json:"interval_sec"` // seconds
} `json:"metrics"`
Sessions struct {
Enable bool `json:"enable"`
IPIgnoreList []string `json:"ip_ignorelist"`
SessionTimeout int `json:"session_timeout_sec"`
Persist bool `json:"persist"`
PersistInterval int `json:"persist_interval_sec"`
MaxBitrate uint64 `json:"max_bitrate_mbit"`
MaxSessions uint64 `json:"max_sessions"`
} `json:"sessions"`
Service struct {
Enable bool `json:"enable"`
Token string `json:"token"`
URL string `json:"url"`
} `json:"service"`
Router struct {
BlockedPrefixes []string `json:"blocked_prefixes"`
Routes map[string]string `json:"routes"`
UIPath string `json:"ui_path"`
} `json:"router"`
}
// Migrate will migrate some settings, depending on the version it finds. Migrations
// are only going upwards,i.e. from a lower version to a higher version.
func NewV2FromV1(d *dataV1) (*dataV2, error) {
data := &dataV2{}
data.CreatedAt = d.CreatedAt
data.LoadedAt = d.LoadedAt
data.UpdatedAt = d.UpdatedAt
data.ID = d.ID
data.Name = d.Name
data.Address = d.Address
data.CheckForUpdates = d.CheckForUpdates
data.Log = d.Log
data.DB = d.DB
data.Host = d.Host
data.API = d.API
data.TLS = d.TLS
data.Storage = d.Storage
data.SRT = d.SRT
data.FFmpeg = d.FFmpeg
data.Playout = d.Playout
data.Debug = d.Debug
data.Metrics = d.Metrics
data.Sessions = d.Sessions
data.Service = d.Service
data.Router = d.Router
data.Log.Topics = copyStringSlice(d.Log.Topics)
data.Host.Name = copyStringSlice(d.Host.Name)
data.API.Access.HTTP.Allow = copyStringSlice(d.API.Access.HTTP.Allow)
data.API.Access.HTTP.Block = copyStringSlice(d.API.Access.HTTP.Block)
data.API.Access.HTTPS.Allow = copyStringSlice(d.API.Access.HTTPS.Allow)
data.API.Access.HTTPS.Block = copyStringSlice(d.API.Access.HTTPS.Block)
data.API.Auth.Auth0.Tenants = copyTenantSlice(d.API.Auth.Auth0.Tenants)
data.Storage.CORS.Origins = copyStringSlice(d.Storage.CORS.Origins)
data.FFmpeg.Access.Input.Allow = copyStringSlice(d.FFmpeg.Access.Input.Allow)
data.FFmpeg.Access.Input.Block = copyStringSlice(d.FFmpeg.Access.Input.Block)
data.FFmpeg.Access.Output.Allow = copyStringSlice(d.FFmpeg.Access.Output.Allow)
data.FFmpeg.Access.Output.Block = copyStringSlice(d.FFmpeg.Access.Output.Block)
data.Sessions.IPIgnoreList = copyStringSlice(d.Sessions.IPIgnoreList)
data.SRT.Log.Topics = copyStringSlice(d.SRT.Log.Topics)
data.Router.BlockedPrefixes = copyStringSlice(d.Router.BlockedPrefixes)
data.Router.Routes = copyStringMap(d.Router.Routes)
// Actual changes
data.RTMP.Enable = d.RTMP.Enable
data.RTMP.EnableTLS = d.RTMP.EnableTLS
data.RTMP.Address = d.RTMP.Address
data.RTMP.App = d.RTMP.App
data.RTMP.Token = d.RTMP.Token
if !strings.HasPrefix(data.RTMP.App, "/") {
data.RTMP.App = "/" + data.RTMP.App
}
if d.RTMP.EnableTLS {
data.RTMP.Enable = true
data.RTMP.AddressTLS = data.RTMP.Address
host, sport, err := net.SplitHostPort(data.RTMP.Address)
if err != nil {
return nil, fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err)
}
port, err := strconv.Atoi(sport)
if err != nil {
return nil, fmt.Errorf("migrating rtmp.address to rtmp.address_tls failed: %w", err)
}
data.RTMP.Address = net.JoinHostPort(host, strconv.Itoa(port-1))
}
data.Version = 2
return data, nil
}

52
config/json.go
View File

@@ -3,7 +3,6 @@ package config
import ( import (
gojson "encoding/json" gojson "encoding/json"
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
@@ -102,7 +101,7 @@ func (c *jsonStore) Reload() error {
return nil return nil
} }
func (c *jsonStore) load(data *Config) error { func (c *jsonStore) load(config *Config) error {
if len(c.path) == 0 { if len(c.path) == 0 {
return nil return nil
} }
@@ -111,17 +110,56 @@ func (c *jsonStore) load(data *Config) error {
return nil return nil
} }
jsondata, err := ioutil.ReadFile(c.path) jsondata, err := os.ReadFile(c.path)
if err != nil { if err != nil {
return err return err
} }
if err = gojson.Unmarshal(jsondata, data); err != nil { dataV3 := &Data{}
version := DataVersion{}
if err = gojson.Unmarshal(jsondata, &version); err != nil {
return json.FormatError(jsondata, err) return json.FormatError(jsondata, err)
} }
data.LoadedAt = time.Now() if version.Version == 1 {
data.UpdatedAt = data.LoadedAt dataV1 := &dataV1{}
if err = gojson.Unmarshal(jsondata, dataV1); err != nil {
return json.FormatError(jsondata, err)
}
dataV2, err := NewV2FromV1(dataV1)
if err != nil {
return err
}
dataV3, err = NewV3FromV2(dataV2)
if err != nil {
return err
}
} else if version.Version == 2 {
dataV2 := &dataV2{}
if err = gojson.Unmarshal(jsondata, dataV2); err != nil {
return json.FormatError(jsondata, err)
}
dataV3, err = NewV3FromV2(dataV2)
if err != nil {
return err
}
} else if version.Version == 3 {
if err = gojson.Unmarshal(jsondata, dataV3); err != nil {
return json.FormatError(jsondata, err)
}
}
config.Data = *dataV3
config.LoadedAt = time.Now()
config.UpdatedAt = config.LoadedAt
return nil return nil
} }
@@ -140,7 +178,7 @@ func (c *jsonStore) store(data *Config) error {
dir, filename := filepath.Split(c.path) dir, filename := filepath.Split(c.path)
tmpfile, err := ioutil.TempFile(dir, filename) tmpfile, err := os.CreateTemp(dir, filename)
if err != nil { if err != nil {
return err return err
} }

59
http/cache/lru.go vendored
View File

@@ -11,23 +11,25 @@ import (
// LRUConfig is the configuration for a new LRU cache // LRUConfig is the configuration for a new LRU cache
type LRUConfig struct { type LRUConfig struct {
TTL time.Duration // For how long the object should stay in cache TTL time.Duration // For how long the object should stay in cache
MaxSize uint64 // Max. size of the cache, 0 for unlimited, bytes MaxSize uint64 // Max. size of the cache, 0 for unlimited, bytes
MaxFileSize uint64 // Max. file size allowed to put in cache, 0 for unlimited, bytes MaxFileSize uint64 // Max. file size allowed to put in cache, 0 for unlimited, bytes
Extensions []string // List of file extension allowed to cache, empty list for all files AllowExtensions []string // List of file extension allowed to cache, empty list for all files
Logger log.Logger BlockExtensions []string // List of file extensions not allowed to cache, empty list for none
Logger log.Logger
} }
type lrucache struct { type lrucache struct {
ttl time.Duration ttl time.Duration
maxSize uint64 maxSize uint64
maxFileSize uint64 maxFileSize uint64
extensions []string allowExtensions []string
objects map[string]*list.Element blockExtensions []string
list *list.List objects map[string]*list.Element
size uint64 list *list.List
lock sync.Mutex size uint64
logger log.Logger lock sync.Mutex
logger log.Logger
} }
type value struct { type value struct {
@@ -53,11 +55,14 @@ func NewLRUCache(config LRUConfig) (Cacher, error) {
} }
if cache.logger == nil { if cache.logger == nil {
cache.logger = log.New("HTTPCache") cache.logger = log.New("")
} }
cache.extensions = make([]string, len(config.Extensions)) cache.allowExtensions = make([]string, len(config.AllowExtensions))
copy(cache.extensions, config.Extensions) copy(cache.allowExtensions, config.AllowExtensions)
cache.blockExtensions = make([]string, len(config.BlockExtensions))
copy(cache.blockExtensions, config.BlockExtensions)
return cache, nil return cache, nil
} }
@@ -199,19 +204,27 @@ func (c *lrucache) TTL() time.Duration {
} }
func (c *lrucache) IsExtensionCacheable(extension string) bool { func (c *lrucache) IsExtensionCacheable(extension string) bool {
if len(c.extensions) == 0 { if len(c.allowExtensions) == 0 && len(c.blockExtensions) == 0 {
return true return true
} }
cacheable := false for _, e := range c.blockExtensions {
for _, e := range c.extensions {
if extension == e { if extension == e {
cacheable = true return false
break
} }
} }
return cacheable if len(c.allowExtensions) == 0 {
return true
}
for _, e := range c.allowExtensions {
if extension == e {
return true
}
}
return false
} }
func (c *lrucache) IsSizeCacheable(size uint64) bool { func (c *lrucache) IsSizeCacheable(size uint64) bool {

30
http/cache/lru_test.go vendored
View File

@@ -8,11 +8,12 @@ import (
) )
var defaultConfig = LRUConfig{ var defaultConfig = LRUConfig{
TTL: time.Hour, TTL: time.Hour,
MaxSize: 128, MaxSize: 128,
MaxFileSize: 0, MaxFileSize: 0,
Extensions: []string{".html", ".js", ".jpg"}, AllowExtensions: []string{".html", ".js", ".jpg"},
Logger: nil, BlockExtensions: []string{".m3u8"},
Logger: nil,
} }
func getCache(t *testing.T) *lrucache { func getCache(t *testing.T) *lrucache {
@@ -27,8 +28,6 @@ func TestNew(t *testing.T) {
TTL: time.Hour, TTL: time.Hour,
MaxSize: 128, MaxSize: 128,
MaxFileSize: 129, MaxFileSize: 129,
Extensions: []string{},
Logger: nil,
}) })
require.NotEqual(t, nil, err) require.NotEqual(t, nil, err)
@@ -36,8 +35,6 @@ func TestNew(t *testing.T) {
TTL: time.Hour, TTL: time.Hour,
MaxSize: 0, MaxSize: 0,
MaxFileSize: 129, MaxFileSize: 129,
Extensions: []string{},
Logger: nil,
}) })
require.Equal(t, nil, err) require.Equal(t, nil, err)
@@ -45,8 +42,6 @@ func TestNew(t *testing.T) {
TTL: time.Hour, TTL: time.Hour,
MaxSize: 128, MaxSize: 128,
MaxFileSize: 127, MaxFileSize: 127,
Extensions: []string{},
Logger: nil,
}) })
require.Equal(t, nil, err) require.Equal(t, nil, err)
} }
@@ -144,7 +139,7 @@ func TestLRU(t *testing.T) {
require.NotEqual(t, nil, data) require.NotEqual(t, nil, data)
} }
func TestExtension(t *testing.T) { func TestAllowExtension(t *testing.T) {
cache := getCache(t) cache := getCache(t)
r := cache.IsExtensionCacheable(".html") r := cache.IsExtensionCacheable(".html")
@@ -154,6 +149,17 @@ func TestExtension(t *testing.T) {
require.Equal(t, false, r) require.Equal(t, false, r)
} }
func TestBlockExtension(t *testing.T) {
cache := getCache(t)
cache.allowExtensions = []string{}
r := cache.IsExtensionCacheable(".html")
require.Equal(t, true, r)
r = cache.IsExtensionCacheable(".m3u8")
require.Equal(t, false, r)
}
func TestSize(t *testing.T) { func TestSize(t *testing.T) {
cache := getCache(t) cache := getCache(t)