package cluster import ( "context" "fmt" "io" "net" "net/http" "net/url" "path/filepath" "strings" "sync" "time" "github.com/datarhei/core/v16/client" httpapi "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/restream/app" ) type Node interface { Connect() error Disconnect() StartFiles(updates chan<- NodeFiles) error StopFiles() GetURL(path string) (string, error) GetFile(path string) (io.ReadCloser, error) ProcessList() ([]ProcessConfig, error) ProcessAdd(*app.Config) error ProcessStart(id string) error ProcessStop(id string) error ProcessDelete(id string) error NodeReader } type NodeReader interface { ID() string Address() string IPs() []string Files() NodeFiles State() NodeState } type NodeFiles struct { ID string Files []string LastUpdate time.Time } type NodeResources struct { NCPU float64 CPU float64 Mem float64 MemTotal float64 } type NodeState struct { ID string State string LastContact time.Time Latency time.Duration Resources NodeResources } type nodeState string func (n nodeState) String() string { return string(n) } const ( stateDisconnected nodeState = "disconnected" stateConnected nodeState = "connected" ) type node struct { address string ips []string peer client.RestClient peerLock sync.RWMutex cancelPing context.CancelFunc lastContact time.Time resources struct { ncpu float64 cpu float64 mem float64 memTotal float64 } state nodeState latency float64 // Seconds stateLock sync.RWMutex updates chan<- NodeFiles filesList []string lastUpdate time.Time cancelFiles context.CancelFunc runningLock sync.Mutex running bool host string secure bool hasRTMP bool rtmpAddress string rtmpToken string hasSRT bool srtAddress string srtPassphrase string srtToken string } func NewNode(address string) Node { n := &node{ address: address, state: stateDisconnected, secure: strings.HasPrefix(address, "https://"), } return n } func (n *node) Connect() error { n.peerLock.Lock() defer n.peerLock.Unlock() if n.peer != nil { return nil } u, err := url.Parse(n.address) if err != nil { return fmt.Errorf("invalid address: %w", err) } host, _, err := net.SplitHostPort(u.Host) if err != nil { return fmt.Errorf("invalid address: %w", err) } addrs, err := net.LookupHost(host) if err != nil { return fmt.Errorf("lookup failed: %w", err) } peer, err := client.New(client.Config{ Address: n.address, Auth0Token: "", Client: &http.Client{ Timeout: 5 * time.Second, }, }) if err != nil { return fmt.Errorf("creating client failed (%s): %w", n.address, err) } config, err := peer.Config() if err != nil { return err } if config.Config.RTMP.Enable { n.hasRTMP = true n.rtmpAddress = "rtmp://" isHostIP := net.ParseIP(host) != nil address := config.Config.RTMP.Address if n.secure && config.Config.RTMP.EnableTLS && !isHostIP { address = config.Config.RTMP.AddressTLS n.rtmpAddress = "rtmps://" } _, port, err := net.SplitHostPort(address) if err != nil { n.hasRTMP = false } else { n.rtmpAddress += host + ":" + port n.rtmpToken = config.Config.RTMP.Token } } if config.Config.SRT.Enable { n.hasSRT = true n.srtAddress = "srt://" _, port, err := net.SplitHostPort(config.Config.SRT.Address) if err != nil { n.hasSRT = false } else { n.srtAddress += host + ":" + port n.srtPassphrase = config.Config.SRT.Passphrase n.srtToken = config.Config.SRT.Token } } n.ips = addrs n.host = host n.peer = peer ctx, cancel := context.WithCancel(context.Background()) n.cancelPing = cancel go func(ctx context.Context) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: // Ping ok, latency := n.peer.Ping() n.stateLock.Lock() if !ok { n.state = stateDisconnected } else { n.lastContact = time.Now() n.state = stateConnected } n.latency = n.latency*0.2 + latency.Seconds()*0.8 n.stateLock.Unlock() case <-ctx.Done(): return } } }(ctx) go func(ctx context.Context) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: // Metrics metrics, err := n.peer.Metrics(httpapi.MetricsQuery{ Metrics: []httpapi.MetricsQueryMetric{ { Name: "cpu_ncpu", }, { Name: "cpu_idle", }, { Name: "mem_total", }, { Name: "mem_free", }, }, }) if err != nil { n.stateLock.Lock() n.resources.cpu = 100 n.resources.ncpu = 1 n.resources.mem = 100 n.resources.memTotal = 1 n.stateLock.Unlock() } cpu_ncpu := .0 cpu_idle := .0 mem_total := .0 mem_free := .0 for _, x := range metrics.Metrics { if x.Name == "cpu_idle" { cpu_idle = x.Values[0].Value } else if x.Name == "cpu_ncpu" { cpu_ncpu = x.Values[0].Value } else if x.Name == "mem_total" { mem_total = x.Values[0].Value } else if x.Name == "mem_free" { mem_free = x.Values[0].Value } } n.stateLock.Lock() n.resources.ncpu = cpu_ncpu n.resources.cpu = 100 - cpu_idle if mem_total != 0 { n.resources.mem = (mem_total - mem_free) / mem_total * 100 n.resources.memTotal = mem_total } else { n.resources.mem = 100 n.resources.memTotal = 1 } n.lastContact = time.Now() n.stateLock.Unlock() case <-ctx.Done(): return } } }(ctx) return nil } func (n *node) Disconnect() { n.peerLock.Lock() defer n.peerLock.Unlock() if n.cancelPing != nil { n.cancelPing() n.cancelPing = nil } n.peer = nil } func (n *node) StartFiles(updates chan<- NodeFiles) error { n.runningLock.Lock() defer n.runningLock.Unlock() if n.running { return nil } n.running = true n.updates = updates ctx, cancel := context.WithCancel(context.Background()) n.cancelFiles = cancel go func(ctx context.Context) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: n.files() select { case n.updates <- n.Files(): default: } } } }(ctx) return nil } func (n *node) StopFiles() { n.runningLock.Lock() defer n.runningLock.Unlock() if !n.running { return } n.running = false n.cancelFiles() } func (n *node) Address() string { return n.address } func (n *node) IPs() []string { return n.ips } func (n *node) ID() string { return n.peer.ID() } func (n *node) Files() NodeFiles { n.stateLock.RLock() defer n.stateLock.RUnlock() state := NodeFiles{ ID: n.peer.ID(), LastUpdate: n.lastUpdate, } if n.state != stateDisconnected && time.Since(n.lastUpdate) <= 2*time.Second { state.Files = make([]string, len(n.filesList)) copy(state.Files, n.filesList) } return state } func (n *node) State() NodeState { n.stateLock.RLock() defer n.stateLock.RUnlock() state := NodeState{ ID: n.peer.ID(), LastContact: n.lastContact, State: n.state.String(), Latency: time.Duration(n.latency * float64(time.Second)), Resources: NodeResources{ NCPU: n.resources.ncpu, CPU: n.resources.cpu, Mem: n.resources.mem, MemTotal: n.resources.memTotal, }, } return state } func (n *node) files() { filesChan := make(chan string, 1024) filesList := []string{} wgList := sync.WaitGroup{} wgList.Add(1) go func() { defer wgList.Done() for file := range filesChan { if len(file) == 0 { return } filesList = append(filesList, file) } }() wg := sync.WaitGroup{} wg.Add(2) go func(f chan<- string) { defer wg.Done() n.peerLock.RLock() defer n.peerLock.RUnlock() files, err := n.peer.MemFSList("name", "asc") if err != nil { return } for _, file := range files { f <- "mem:" + file.Name } }(filesChan) go func(f chan<- string) { defer wg.Done() n.peerLock.RLock() defer n.peerLock.RUnlock() files, err := n.peer.DiskFSList("name", "asc") if err != nil { return } for _, file := range files { f <- "disk:" + file.Name } }(filesChan) if n.hasRTMP { wg.Add(1) go func(f chan<- string) { defer wg.Done() n.peerLock.RLock() defer n.peerLock.RUnlock() files, err := n.peer.RTMPChannels() if err != nil { return } for _, file := range files { f <- "rtmp:" + file.Name } }(filesChan) } if n.hasSRT { wg.Add(1) go func(f chan<- string) { defer wg.Done() n.peerLock.RLock() defer n.peerLock.RUnlock() files, err := n.peer.SRTChannels() if err != nil { return } for _, file := range files { f <- "srt:" + file.Name } }(filesChan) } wg.Wait() filesChan <- "" wgList.Wait() n.stateLock.Lock() n.filesList = make([]string, len(filesList)) copy(n.filesList, filesList) n.lastUpdate = time.Now() n.lastContact = time.Now() n.stateLock.Unlock() } func (n *node) GetURL(path string) (string, error) { prefix, path, found := strings.Cut(path, ":") if !found { return "", fmt.Errorf("no prefix provided") } u := "" if prefix == "mem" { u = n.address + "/" + filepath.Join("memfs", path) } else if prefix == "disk" { u = n.address + path } else if prefix == "rtmp" { u = n.rtmpAddress + path if len(n.rtmpToken) != 0 { u += "?token=" + url.QueryEscape(n.rtmpToken) } } else if prefix == "srt" { u = n.srtAddress + "?mode=caller" if len(n.srtPassphrase) != 0 { u += "&passphrase=" + url.QueryEscape(n.srtPassphrase) } streamid := "#!:m=request,r=" + path if len(n.srtToken) != 0 { streamid += ",token=" + n.srtToken } u += "&streamid=" + url.QueryEscape(streamid) } else { return "", fmt.Errorf("unknown prefix") } return u, nil } func (n *node) GetFile(path string) (io.ReadCloser, error) { prefix, path, found := strings.Cut(path, ":") if !found { return nil, fmt.Errorf("no prefix provided") } n.peerLock.RLock() defer n.peerLock.RUnlock() if prefix == "mem" { return n.peer.MemFSGetFile(path) } else if prefix == "disk" { return n.peer.DiskFSGetFile(path) } return nil, fmt.Errorf("unknown prefix") } func (n *node) ProcessList() ([]ProcessConfig, error) { list, err := n.peer.ProcessList(nil, nil) if err != nil { return nil, err } processes := []ProcessConfig{} for _, p := range list { process := ProcessConfig{ NodeID: n.ID(), Order: p.State.Order, State: p.State.State, Mem: float64(p.State.Memory) / float64(n.resources.memTotal), Runtime: time.Duration(p.State.Runtime) * time.Second, Config: p.Config.Marshal(), } if x, err := p.State.CPU.Float64(); err == nil { process.CPU = x / n.resources.ncpu } else { process.CPU = 100 } processes = append(processes, process) } return processes, nil } func (n *node) ProcessAdd(config *app.Config) error { cfg := httpapi.ProcessConfig{} cfg.Unmarshal(config) return n.peer.ProcessAdd(cfg) } func (n *node) ProcessStart(id string) error { return n.peer.ProcessCommand(id, "start") } func (n *node) ProcessStop(id string) error { return n.peer.ProcessCommand(id, "stop") } func (n *node) ProcessDelete(id string) error { return n.peer.ProcessDelete(id) }