diff --git a/app/api/api.go b/app/api/api.go index 1cdd1f4d..7d734fe6 100644 --- a/app/api/api.go +++ b/app/api/api.go @@ -543,6 +543,7 @@ func (a *api) start(ctx context.Context) error { CoreSkills: a.ffmpeg.Skills(), IPLimiter: a.sessionsLimiter, Logger: a.log.logger.core.WithComponent("Cluster"), + Resources: a.resources, Debug: cluster.DebugConfig{ DisableFFmpegCheck: cfg.Cluster.Debug.DisableFFmpegCheck, }, @@ -1337,7 +1338,7 @@ func (a *api) start(ctx context.Context) error { } if a.cluster != nil { - config.Proxy = a.cluster.ProxyReader() + config.Proxy = a.cluster.Manager() } if cfg.RTMP.EnableTLS { @@ -1368,7 +1369,7 @@ func (a *api) start(ctx context.Context) error { } if a.cluster != nil { - config.Proxy = a.cluster.ProxyReader() + config.Proxy = a.cluster.Manager() } if cfg.SRT.Log.Enable { diff --git a/cluster/about.go b/cluster/about.go new file mode 100644 index 00000000..d5216bdc --- /dev/null +++ b/cluster/about.go @@ -0,0 +1,173 @@ +package cluster + +import ( + "errors" + "time" + + "github.com/datarhei/core/v16/cluster/raft" + "github.com/datarhei/core/v16/slices" +) + +type ClusterRaft struct { + Address string + State string + LastContact time.Duration + NumPeers uint64 + LogTerm uint64 + LogIndex uint64 +} + +type ClusterNodeResources struct { + IsThrottling bool // Whether this core is currently throttling + NCPU float64 // Number of CPU on this node + CPU float64 // Current CPU load, 0-100*ncpu + CPULimit float64 // Defined CPU load limit, 0-100*ncpu + Mem uint64 // Currently used memory in bytes + MemLimit uint64 // Defined memory limit in bytes + Error error +} + +type ClusterNode struct { + ID string + Name string + Version string + State string + Error error + Voter bool + Leader bool + Address string + CreatedAt time.Time + Uptime time.Duration + LastContact time.Duration + Latency time.Duration + Core ClusterNodeCore + Resources ClusterNodeResources +} + +type ClusterNodeCore struct { + Address string + State string + Error error + LastContact time.Duration + Latency time.Duration + Version string +} + +type ClusterAboutLeader struct { + ID string + Address string + ElectedSince time.Duration +} + +type ClusterAbout struct { + ID string + Domains []string + Leader ClusterAboutLeader + State string + Raft ClusterRaft + Nodes []ClusterNode + Version ClusterVersion + Error error +} + +func (c *cluster) About() (ClusterAbout, error) { + c.stateLock.RLock() + hasLeader := c.hasRaftLeader + domains := slices.Copy(c.hostnames) + c.stateLock.RUnlock() + + about := ClusterAbout{ + ID: c.id, + Leader: ClusterAboutLeader{}, + State: "online", + Version: Version, + Domains: domains, + } + + if !hasLeader { + about.State = "offline" + about.Error = errors.New("no raft leader elected") + } + + stats := c.raft.Stats() + + about.Raft.Address = stats.Address + about.Raft.State = stats.State + about.Raft.LastContact = stats.LastContact + about.Raft.NumPeers = stats.NumPeers + about.Raft.LogIndex = stats.LogIndex + about.Raft.LogTerm = stats.LogTerm + + servers, err := c.raft.Servers() + if err != nil { + c.logger.Warn().WithError(err).Log("Raft configuration") + } + + serversMap := map[string]raft.Server{} + + for _, s := range servers { + serversMap[s.ID] = s + + if s.Leader { + about.Leader.ID = s.ID + about.Leader.Address = s.Address + about.Leader.ElectedSince = s.LastChange + } + } + + storeNodes := c.ListNodes() + nodes := c.manager.NodeList() + + for _, node := range nodes { + nodeAbout := node.About() + + node := ClusterNode{ + ID: nodeAbout.ID, + Name: nodeAbout.Name, + Version: nodeAbout.Version, + State: nodeAbout.State, + Error: nodeAbout.Error, + Address: nodeAbout.Address, + LastContact: time.Since(nodeAbout.LastContact), + Latency: nodeAbout.Latency, + CreatedAt: nodeAbout.Core.CreatedAt, + Uptime: nodeAbout.Core.Uptime, + Core: ClusterNodeCore{ + Address: nodeAbout.Core.Address, + State: nodeAbout.Core.State, + Error: nodeAbout.Core.Error, + LastContact: time.Since(nodeAbout.Core.LastContact), + Latency: nodeAbout.Core.Latency, + Version: nodeAbout.Core.Version.Number, + }, + Resources: ClusterNodeResources{ + IsThrottling: nodeAbout.Resources.IsThrottling, + NCPU: nodeAbout.Resources.NCPU, + CPU: nodeAbout.Resources.CPU, + CPULimit: nodeAbout.Resources.CPULimit, + Mem: nodeAbout.Resources.Mem, + MemLimit: nodeAbout.Resources.MemLimit, + Error: nodeAbout.Resources.Error, + }, + } + + if s, ok := serversMap[nodeAbout.ID]; ok { + node.Voter = s.Voter + node.Leader = s.Leader + } + + if storeNode, hasStoreNode := storeNodes[nodeAbout.ID]; hasStoreNode { + if storeNode.State == "maintenance" { + node.State = storeNode.State + } + } + + if about.State == "online" && node.State != "online" { + about.State = "degraded" + } + + about.Nodes = append(about.Nodes, node) + } + + return about, nil +} diff --git a/cluster/affinity.go b/cluster/affinity.go new file mode 100644 index 00000000..0ffc678b --- /dev/null +++ b/cluster/affinity.go @@ -0,0 +1,181 @@ +package cluster + +import ( + "sort" + + "github.com/datarhei/core/v16/cluster/node" +) + +type referenceAffinityNodeCount struct { + nodeid string + count uint64 +} + +type referenceAffinity struct { + m map[string][]referenceAffinityNodeCount +} + +// NewReferenceAffinity returns a referenceAffinity. This is a map of references (per domain) to an array of +// nodes this reference is found on and their count. +func NewReferenceAffinity(processes []node.Process) *referenceAffinity { + ra := &referenceAffinity{ + m: map[string][]referenceAffinityNodeCount{}, + } + + for _, p := range processes { + if len(p.Config.Reference) == 0 { + continue + } + + key := p.Config.Reference + "@" + p.Config.Domain + + // Here we count how often a reference is present on a node. When + // moving processes to a different node, the node with the highest + // count of same references will be the first candidate. + found := false + arr := ra.m[key] + for i, count := range arr { + if count.nodeid == p.NodeID { + count.count++ + arr[i] = count + found = true + break + } + } + + if !found { + arr = append(arr, referenceAffinityNodeCount{ + nodeid: p.NodeID, + count: 1, + }) + } + + ra.m[key] = arr + } + + // Sort every reference count in decreasing order for each reference. + for ref, count := range ra.m { + sort.SliceStable(count, func(a, b int) bool { + return count[a].count > count[b].count + }) + + ra.m[ref] = count + } + + return ra +} + +// Nodes returns a list of node IDs for the provided reference and domain. The list +// is ordered by how many references are on the nodes in descending order. +func (ra *referenceAffinity) Nodes(reference, domain string) []string { + if len(reference) == 0 { + return nil + } + + key := reference + "@" + domain + + counts, ok := ra.m[key] + if !ok { + return nil + } + + nodes := []string{} + + for _, count := range counts { + nodes = append(nodes, count.nodeid) + } + + return nodes +} + +// Add adds a reference on a node to an existing reference affinity. +func (ra *referenceAffinity) Add(reference, domain, nodeid string) { + if len(reference) == 0 { + return + } + + key := reference + "@" + domain + + counts, ok := ra.m[key] + if !ok { + ra.m[key] = []referenceAffinityNodeCount{ + { + nodeid: nodeid, + count: 1, + }, + } + + return + } + + found := false + for i, count := range counts { + if count.nodeid == nodeid { + count.count++ + counts[i] = count + found = true + break + } + } + + if !found { + counts = append(counts, referenceAffinityNodeCount{ + nodeid: nodeid, + count: 1, + }) + } + + ra.m[key] = counts +} + +// Move moves a reference from one node to another node in an existing reference affinity. +func (ra *referenceAffinity) Move(reference, domain, fromnodeid, tonodeid string) { + if len(reference) == 0 { + return + } + + key := reference + "@" + domain + + counts, ok := ra.m[key] + if !ok { + ra.m[key] = []referenceAffinityNodeCount{ + { + nodeid: tonodeid, + count: 1, + }, + } + + return + } + + found := false + for i, count := range counts { + if count.nodeid == tonodeid { + count.count++ + counts[i] = count + found = true + } else if count.nodeid == fromnodeid { + count.count-- + counts[i] = count + } + } + + if !found { + counts = append(counts, referenceAffinityNodeCount{ + nodeid: tonodeid, + count: 1, + }) + } + + newCounts := []referenceAffinityNodeCount{} + + for _, count := range counts { + if count.count == 0 { + continue + } + + newCounts = append(newCounts, count) + } + + ra.m[key] = newCounts +} diff --git a/cluster/api.go b/cluster/api.go index 04c47acd..4a2328b4 100644 --- a/cluster/api.go +++ b/cluster/api.go @@ -17,11 +17,12 @@ import ( "context" "errors" "fmt" - "io/fs" "net/http" "strings" + "time" "github.com/datarhei/core/v16/cluster/client" + "github.com/datarhei/core/v16/cluster/store" "github.com/datarhei/core/v16/encoding/json" "github.com/datarhei/core/v16/http/handler/util" httplog "github.com/datarhei/core/v16/http/log" @@ -38,11 +39,12 @@ import ( ) type api struct { - id string - address string - router *echo.Echo - cluster Cluster - logger log.Logger + id string + address string + router *echo.Echo + cluster *cluster + logger log.Logger + startedAt time.Time } type API interface { @@ -52,15 +54,16 @@ type API interface { type APIConfig struct { ID string - Cluster Cluster + Cluster *cluster Logger log.Logger } func NewAPI(config APIConfig) (API, error) { a := &api{ - id: config.ID, - cluster: config.Cluster, - logger: config.Logger, + id: config.ID, + cluster: config.Cluster, + logger: config.Logger, + startedAt: time.Now(), } if a.logger == nil { @@ -98,6 +101,7 @@ func NewAPI(config APIConfig) (API, error) { doc.GET("", echoSwagger.EchoWrapHandler(echoSwagger.InstanceName("ClusterAPI"))) a.router.GET("/", a.Version) + a.router.GET("/v1/about", a.About) a.router.GET("/v1/barrier/:name", a.Barrier) @@ -108,27 +112,27 @@ func NewAPI(config APIConfig) (API, error) { a.router.GET("/v1/snaphot", a.Snapshot) - a.router.POST("/v1/process", a.AddProcess) - a.router.DELETE("/v1/process/:id", a.RemoveProcess) - a.router.PUT("/v1/process/:id", a.UpdateProcess) - a.router.PUT("/v1/process/:id/command", a.SetProcessCommand) - a.router.PUT("/v1/process/:id/metadata/:key", a.SetProcessMetadata) + a.router.POST("/v1/process", a.ProcessAdd) + a.router.DELETE("/v1/process/:id", a.ProcessRemove) + a.router.PUT("/v1/process/:id", a.ProcessUpdate) + a.router.PUT("/v1/process/:id/command", a.ProcessSetCommand) + a.router.PUT("/v1/process/:id/metadata/:key", a.ProcessSetMetadata) - a.router.PUT("/v1/relocate", a.RelocateProcesses) + a.router.PUT("/v1/relocate", a.ProcessesRelocate) - a.router.POST("/v1/iam/user", a.AddIdentity) - a.router.PUT("/v1/iam/user/:name", a.UpdateIdentity) - a.router.PUT("/v1/iam/user/:name/policies", a.SetIdentityPolicies) - a.router.DELETE("/v1/iam/user/:name", a.RemoveIdentity) + a.router.POST("/v1/iam/user", a.IAMIdentityAdd) + a.router.PUT("/v1/iam/user/:name", a.IAMIdentityUpdate) + a.router.PUT("/v1/iam/user/:name/policies", a.IAMPoliciesSet) + a.router.DELETE("/v1/iam/user/:name", a.IAMIdentityRemove) - a.router.POST("/v1/lock", a.Lock) - a.router.DELETE("/v1/lock/:name", a.Unlock) + a.router.POST("/v1/lock", a.LockCreate) + a.router.DELETE("/v1/lock/:name", a.LockDelete) - a.router.POST("/v1/kv", a.SetKV) - a.router.GET("/v1/kv/:key", a.GetKV) - a.router.DELETE("/v1/kv/:key", a.UnsetKV) + a.router.POST("/v1/kv", a.KVSet) + a.router.GET("/v1/kv/:key", a.KVGet) + a.router.DELETE("/v1/kv/:key", a.KVUnset) - a.router.PUT("/v1/node/:id/state", a.SetNodeState) + a.router.PUT("/v1/node/:id/state", a.NodeSetState) a.router.GET("/v1/core", a.CoreAPIAddress) a.router.GET("/v1/core/config", a.CoreConfig) @@ -160,6 +164,40 @@ func (a *api) Version(c echo.Context) error { return c.JSON(http.StatusOK, Version.String()) } +// About returns the version of the cluster +// @Summary The cluster version +// @Description The cluster version +// @Tags v1.0.0 +// @ID cluster-1-about +// @Produce json +// @Success 200 {string} About +// @Success 500 {object} Error +// @Router /v1/about [get] +func (a *api) About(c echo.Context) error { + resources, err := a.cluster.Resources() + + about := client.AboutResponse{ + ID: a.id, + Version: Version.String(), + Address: a.cluster.Address(), + StartedAt: a.startedAt, + Resources: client.AboutResponseResources{ + IsThrottling: resources.CPU.Throttling, + NCPU: resources.CPU.NCPU, + CPU: (100 - resources.CPU.Idle) * resources.CPU.NCPU, + CPULimit: resources.CPU.Limit * resources.CPU.NCPU, + Mem: resources.Mem.Total - resources.Mem.Available, + MemLimit: resources.Mem.Total, + }, + } + + if err != nil { + about.Resources.Error = err.Error() + } + + return c.JSON(http.StatusOK, about) +} + // Barrier returns if the barrier already has been passed // @Summary Has the barrier already has been passed // @Description Has the barrier already has been passed @@ -210,7 +248,7 @@ func (a *api) AddServer(c echo.Context) error { err := a.cluster.Join(origin, r.ID, r.RaftAddress, "") if err != nil { a.logger.Debug().WithError(err).WithField("id", r.ID).Log("Unable to join cluster") - return Err(http.StatusInternalServerError, "", "unable to join cluster: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") @@ -244,7 +282,7 @@ func (a *api) RemoveServer(c echo.Context) error { err := a.cluster.Leave(origin, id) if err != nil { a.logger.Debug().WithError(err).WithField("id", id).Log("Unable to leave cluster") - return Err(http.StatusInternalServerError, "", "unable to leave cluster: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") @@ -278,7 +316,7 @@ func (a *api) TransferLeadership(c echo.Context) error { err := a.cluster.TransferLeadership(origin, id) if err != nil { a.logger.Debug().WithError(err).WithField("id", id).Log("Unable to transfer leadership") - return Err(http.StatusInternalServerError, "", "unable to transfer leadership: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") @@ -303,7 +341,7 @@ func (a *api) Snapshot(c echo.Context) error { data, err := a.cluster.Snapshot(origin) if err != nil { a.logger.Debug().WithError(err).Log("Unable to create snaphot") - return Err(http.StatusInternalServerError, "", "unable to create snapshot: %s", err.Error()) + return ErrFromClusterError(err) } defer data.Close() @@ -311,7 +349,7 @@ func (a *api) Snapshot(c echo.Context) error { return c.Stream(http.StatusOK, "application/octet-stream", data) } -// AddProcess adds a process to the cluster DB +// ProcessAdd adds a process to the cluster DB // @Summary Add a process // @Description Add a process to the cluster DB // @Tags v1.0.0 @@ -325,7 +363,7 @@ func (a *api) Snapshot(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/process [post] -func (a *api) AddProcess(c echo.Context) error { +func (a *api) ProcessAdd(c echo.Context) error { r := client.AddProcessRequest{} if err := util.ShouldBindJSON(c, &r); err != nil { @@ -340,16 +378,16 @@ func (a *api) AddProcess(c echo.Context) error { a.logger.Debug().WithField("id", r.Config.ID).Log("Add process request") - err := a.cluster.AddProcess(origin, &r.Config) + err := a.cluster.ProcessAdd(origin, &r.Config) if err != nil { a.logger.Debug().WithError(err).WithField("id", r.Config.ID).Log("Unable to add process") - return Err(http.StatusInternalServerError, "", "unable to add process: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// RemoveProcess removes a process from the cluster DB +// ProcessRemove removes a process from the cluster DB // @Summary Remove a process // @Description Remove a process from the cluster DB // @Tags v1.0.0 @@ -362,7 +400,7 @@ func (a *api) AddProcess(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/process/{id} [delete] -func (a *api) RemoveProcess(c echo.Context) error { +func (a *api) ProcessRemove(c echo.Context) error { id := util.PathParam(c, "id") domain := util.DefaultQuery(c, "domain", "") @@ -376,16 +414,16 @@ func (a *api) RemoveProcess(c echo.Context) error { a.logger.Debug().WithField("id", pid).Log("Remove process request") - err := a.cluster.RemoveProcess(origin, pid) + err := a.cluster.ProcessRemove(origin, pid) if err != nil { a.logger.Debug().WithError(err).WithField("id", pid).Log("Unable to remove process") - return Err(http.StatusInternalServerError, "", "unable to remove process: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// UpdateProcess replaces an existing process in the cluster DB +// ProcessUpdate replaces an existing process in the cluster DB // @Summary Replace an existing process // @Description Replace an existing process in the cluster DB // @Tags v1.0.0 @@ -399,7 +437,7 @@ func (a *api) RemoveProcess(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/process/{id} [put] -func (a *api) UpdateProcess(c echo.Context) error { +func (a *api) ProcessUpdate(c echo.Context) error { id := util.PathParam(c, "id") domain := util.DefaultQuery(c, "domain", "") @@ -422,16 +460,16 @@ func (a *api) UpdateProcess(c echo.Context) error { "new_id": r.Config.ProcessID(), }).Log("Update process request") - err := a.cluster.UpdateProcess(origin, pid, &r.Config) + err := a.cluster.ProcessUpdate(origin, pid, &r.Config) if err != nil { a.logger.Debug().WithError(err).WithField("id", pid).Log("Unable to update process") - return Err(http.StatusInternalServerError, "", "unable to update process: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// SetProcessCommand sets the order for a process +// ProcessSetCommand sets the order for a process // @Summary Set the order for a process // @Description Set the order for a process. // @Tags v1.0.0 @@ -444,7 +482,7 @@ func (a *api) UpdateProcess(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/process/{id}/command [put] -func (a *api) SetProcessCommand(c echo.Context) error { +func (a *api) ProcessSetCommand(c echo.Context) error { id := util.PathParam(c, "id") domain := util.DefaultQuery(c, "domain", "") @@ -462,16 +500,16 @@ func (a *api) SetProcessCommand(c echo.Context) error { pid := app.ProcessID{ID: id, Domain: domain} - err := a.cluster.SetProcessCommand(origin, pid, r.Command) + err := a.cluster.ProcessSetCommand(origin, pid, r.Command) if err != nil { a.logger.Debug().WithError(err).WithField("id", pid).Log("Unable to set order") - return Err(http.StatusInternalServerError, "", "unable to set order: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// SetProcessMetadata stores metadata with a process +// ProcessSetMetadata stores metadata with a process // @Summary Add JSON metadata with a process under the given key // @Description Add arbitrary JSON metadata under the given key. If the key exists, all already stored metadata with this key will be overwritten. If the key doesn't exist, it will be created. // @Tags v1.0.0 @@ -485,7 +523,7 @@ func (a *api) SetProcessCommand(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/process/{id}/metadata/{key} [put] -func (a *api) SetProcessMetadata(c echo.Context) error { +func (a *api) ProcessSetMetadata(c echo.Context) error { id := util.PathParam(c, "id") key := util.PathParam(c, "key") domain := util.DefaultQuery(c, "domain", "") @@ -504,16 +542,16 @@ func (a *api) SetProcessMetadata(c echo.Context) error { pid := app.ProcessID{ID: id, Domain: domain} - err := a.cluster.SetProcessMetadata(origin, pid, key, r.Metadata) + err := a.cluster.ProcessSetMetadata(origin, pid, key, r.Metadata) if err != nil { a.logger.Debug().WithError(err).WithField("id", pid).Log("Unable to update metadata") - return Err(http.StatusInternalServerError, "", "unable to update metadata: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// RelocateProcesses relocates processes to another node +// ProcessesRelocate relocates processes to another node // @Summary Relocate processes to another node // @Description Relocate processes to another node. // @Tags v1.0.0 @@ -524,7 +562,7 @@ func (a *api) SetProcessMetadata(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/relocate [put] -func (a *api) RelocateProcesses(c echo.Context) error { +func (a *api) ProcessesRelocate(c echo.Context) error { r := client.RelocateProcessesRequest{} if err := util.ShouldBindJSON(c, &r); err != nil { @@ -537,16 +575,16 @@ func (a *api) RelocateProcesses(c echo.Context) error { return Err(http.StatusLoopDetected, "", "breaking circuit") } - err := a.cluster.RelocateProcesses(origin, r.Map) + err := a.cluster.ProcessesRelocate(origin, r.Map) if err != nil { a.logger.Debug().WithError(err).Log("Unable to apply process relocation request") - return Err(http.StatusInternalServerError, "", "unable to apply process relocation request: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// AddIdentity adds an identity to the cluster DB +// IAMIdentityAdd adds an identity to the cluster DB // @Summary Add an identity // @Description Add an identity to the cluster DB // @Tags v1.0.0 @@ -560,7 +598,7 @@ func (a *api) RelocateProcesses(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/iam/user [post] -func (a *api) AddIdentity(c echo.Context) error { +func (a *api) IAMIdentityAdd(c echo.Context) error { r := client.AddIdentityRequest{} if err := util.ShouldBindJSON(c, &r); err != nil { @@ -575,16 +613,16 @@ func (a *api) AddIdentity(c echo.Context) error { a.logger.Debug().WithField("identity", r.Identity).Log("Add identity request") - err := a.cluster.AddIdentity(origin, r.Identity) + err := a.cluster.IAMIdentityAdd(origin, r.Identity) if err != nil { a.logger.Debug().WithError(err).WithField("identity", r.Identity).Log("Unable to add identity") - return Err(http.StatusInternalServerError, "", "unable to add identity: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// UpdateIdentity replaces an existing identity in the cluster DB +// IAMIdentityUpdate replaces an existing identity in the cluster DB // @Summary Replace an existing identity // @Description Replace an existing identity in the cluster DB // @Tags v1.0.0 @@ -597,7 +635,7 @@ func (a *api) AddIdentity(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/iam/user/{name} [put] -func (a *api) UpdateIdentity(c echo.Context) error { +func (a *api) IAMIdentityUpdate(c echo.Context) error { name := util.PathParam(c, "name") r := client.UpdateIdentityRequest{} @@ -617,32 +655,32 @@ func (a *api) UpdateIdentity(c echo.Context) error { "identity": r.Identity, }).Log("Update identity request") - err := a.cluster.UpdateIdentity(origin, name, r.Identity) + err := a.cluster.IAMIdentityUpdate(origin, name, r.Identity) if err != nil { a.logger.Debug().WithError(err).WithFields(log.Fields{ "name": name, "identity": r.Identity, }).Log("Unable to add identity") - return Err(http.StatusInternalServerError, "", "unable to update identity: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// SetIdentityPolicies set policies for an identity in the cluster DB +// IAMPoliciesSet set policies for an identity in the cluster DB // @Summary Set identity policies // @Description Set policies for an identity in the cluster DB. Any existing policies will be replaced. // @Tags v1.0.0 // @ID cluster-3-set-identity-policies // @Produce json -// @Param id path string true "Process ID"SetPoliciesRequest +// @Param id path string true "Process ID" SetPoliciesRequest // @Param data body client.SetPoliciesRequest true "Policies for that user" // @Success 200 {string} string // @Failure 400 {object} Error // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/iam/user/{name}/policies [put] -func (a *api) SetIdentityPolicies(c echo.Context) error { +func (a *api) IAMPoliciesSet(c echo.Context) error { name := util.PathParam(c, "name") r := client.SetPoliciesRequest{} @@ -659,16 +697,16 @@ func (a *api) SetIdentityPolicies(c echo.Context) error { a.logger.Debug().WithField("policies", r.Policies).Log("Set policiesrequest") - err := a.cluster.SetPolicies(origin, name, r.Policies) + err := a.cluster.IAMPoliciesSet(origin, name, r.Policies) if err != nil { a.logger.Debug().WithError(err).WithField("policies", r.Policies).Log("Unable to set policies") - return Err(http.StatusInternalServerError, "", "unable to add identity: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// RemoveIdentity removes an identity from the cluster DB +// IAMIdentityRemove removes an identity from the cluster DB // @Summary Remove an identity // @Description Remove an identity from the cluster DB // @Tags v1.0.0 @@ -680,7 +718,7 @@ func (a *api) SetIdentityPolicies(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/iam/user/{name} [delete] -func (a *api) RemoveIdentity(c echo.Context) error { +func (a *api) IAMIdentityRemove(c echo.Context) error { name := util.PathParam(c, "name") origin := c.Request().Header.Get("X-Cluster-Origin") @@ -691,10 +729,10 @@ func (a *api) RemoveIdentity(c echo.Context) error { a.logger.Debug().WithField("identity", name).Log("Remove identity request") - err := a.cluster.RemoveIdentity(origin, name) + err := a.cluster.IAMIdentityRemove(origin, name) if err != nil { a.logger.Debug().WithError(err).WithField("identity", name).Log("Unable to remove identity") - return Err(http.StatusInternalServerError, "", "unable to remove identity: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") @@ -739,19 +777,19 @@ func (a *api) CoreSkills(c echo.Context) error { return c.JSON(http.StatusOK, skills) } -// Lock tries to acquire a named lock +// LockCreate tries to acquire a named lock // @Summary Acquire a named lock // @Description Acquire a named lock // @Tags v1.0.0 // @ID cluster-1-lock // @Produce json -// @Param data body client.LockRequest true "Lock request" +// @Param data body client.LockRequest true "LockCreate request" // @Param X-Cluster-Origin header string false "Origin ID of request" // @Success 200 {string} string // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/lock [post] -func (a *api) Lock(c echo.Context) error { +func (a *api) LockCreate(c echo.Context) error { r := client.LockRequest{} if err := util.ShouldBindJSON(c, &r); err != nil { @@ -766,16 +804,16 @@ func (a *api) Lock(c echo.Context) error { a.logger.Debug().WithField("name", r.Name).Log("Acquire lock") - _, err := a.cluster.CreateLock(origin, r.Name, r.ValidUntil) + _, err := a.cluster.LockCreate(origin, r.Name, r.ValidUntil) if err != nil { a.logger.Debug().WithError(err).WithField("name", r.Name).Log("Unable to acquire lock") - return Err(http.StatusInternalServerError, "", "unable to acquire lock: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// Unlock removes a named lock +// LockDelete removes a named lock // @Summary Remove a lock // @Description Remove a lock // @Tags v1.0.0 @@ -788,7 +826,7 @@ func (a *api) Lock(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/lock/{name} [delete] -func (a *api) Unlock(c echo.Context) error { +func (a *api) LockDelete(c echo.Context) error { name := util.PathParam(c, "name") origin := c.Request().Header.Get("X-Cluster-Origin") @@ -799,16 +837,16 @@ func (a *api) Unlock(c echo.Context) error { a.logger.Debug().WithField("name", name).Log("Remove lock request") - err := a.cluster.DeleteLock(origin, name) + err := a.cluster.LockDelete(origin, name) if err != nil { a.logger.Debug().WithError(err).WithField("name", name).Log("Unable to remove lock") - return Err(http.StatusInternalServerError, "", "unable to remove lock: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// SetKV stores the value under key +// KVSet stores the value under key // @Summary Store value under key // @Description Store value under key // @Tags v1.0.0 @@ -820,7 +858,7 @@ func (a *api) Unlock(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/kv [post] -func (a *api) SetKV(c echo.Context) error { +func (a *api) KVSet(c echo.Context) error { r := client.SetKVRequest{} if err := util.ShouldBindJSON(c, &r); err != nil { @@ -835,16 +873,16 @@ func (a *api) SetKV(c echo.Context) error { a.logger.Debug().WithField("key", r.Key).Log("Store value") - err := a.cluster.SetKV(origin, r.Key, r.Value) + err := a.cluster.KVSet(origin, r.Key, r.Value) if err != nil { a.logger.Debug().WithError(err).WithField("key", r.Key).Log("Unable to store value") - return Err(http.StatusInternalServerError, "", "unable to store value: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// UnsetKV removes a key +// KVUnset removes a key // @Summary Remove a key // @Description Remove a key // @Tags v1.0.0 @@ -857,7 +895,7 @@ func (a *api) SetKV(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/kv/{key} [delete] -func (a *api) UnsetKV(c echo.Context) error { +func (a *api) KVUnset(c echo.Context) error { key := util.PathParam(c, "key") origin := c.Request().Header.Get("X-Cluster-Origin") @@ -868,20 +906,16 @@ func (a *api) UnsetKV(c echo.Context) error { a.logger.Debug().WithField("key", key).Log("Delete key") - err := a.cluster.UnsetKV(origin, key) + err := a.cluster.KVUnset(origin, key) if err != nil { - if err == fs.ErrNotExist { - a.logger.Debug().WithError(err).WithField("key", key).Log("Delete key: not found") - return Err(http.StatusNotFound, "", "%s", err.Error()) - } a.logger.Debug().WithError(err).WithField("key", key).Log("Unable to remove key") - return Err(http.StatusInternalServerError, "", "unable to remove key: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") } -// GetKV fetches a key +// KVGet fetches a key // @Summary Fetch a key // @Description Fetch a key // @Tags v1.0.0 @@ -894,7 +928,7 @@ func (a *api) UnsetKV(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/kv/{key} [get] -func (a *api) GetKV(c echo.Context) error { +func (a *api) KVGet(c echo.Context) error { key := util.PathParam(c, "key") origin := c.Request().Header.Get("X-Cluster-Origin") @@ -905,14 +939,10 @@ func (a *api) GetKV(c echo.Context) error { a.logger.Debug().WithField("key", key).Log("Get key") - value, updatedAt, err := a.cluster.GetKV(origin, key, false) + value, updatedAt, err := a.cluster.KVGet(origin, key, false) if err != nil { - if err == fs.ErrNotExist { - a.logger.Debug().WithError(err).WithField("key", key).Log("Get key: not found") - return Err(http.StatusNotFound, "", "%s", err.Error()) - } a.logger.Debug().WithError(err).WithField("key", key).Log("Unable to retrieve key") - return Err(http.StatusInternalServerError, "", "unable to retrieve key: %s", err.Error()) + return ErrFromClusterError(err) } res := client.GetKVResponse{ @@ -923,7 +953,7 @@ func (a *api) GetKV(c echo.Context) error { return c.JSON(http.StatusOK, res) } -// SetNodeState sets a state for a node +// NodeSetState sets a state for a node // @Summary Set a state for a node // @Description Set a state for a node // @Tags v1.0.0 @@ -936,7 +966,7 @@ func (a *api) GetKV(c echo.Context) error { // @Failure 500 {object} Error // @Failure 508 {object} Error // @Router /v1/node/{id}/state [get] -func (a *api) SetNodeState(c echo.Context) error { +func (a *api) NodeSetState(c echo.Context) error { nodeid := util.PathParam(c, "id") r := client.SetNodeStateRequest{} @@ -956,7 +986,7 @@ func (a *api) SetNodeState(c echo.Context) error { "state": r.State, }).Log("Set node state") - err := a.cluster.SetNodeState(origin, nodeid, r.State) + err := a.cluster.NodeSetState(origin, nodeid, r.State) if err != nil { a.logger.Debug().WithError(err).WithFields(log.Fields{ "node": nodeid, @@ -966,7 +996,7 @@ func (a *api) SetNodeState(c echo.Context) error { if errors.Is(err, ErrUnsupportedNodeState) { return Err(http.StatusBadRequest, "", "%s: %s", err.Error(), r.State) } - return Err(http.StatusInternalServerError, "", "unable to set state: %s", err.Error()) + return ErrFromClusterError(err) } return c.JSON(http.StatusOK, "OK") @@ -1007,6 +1037,17 @@ func Err(code int, message string, args ...interface{}) Error { return e } +func ErrFromClusterError(err error) Error { + status := http.StatusInternalServerError + if errors.Is(err, store.ErrNotFound) { + status = http.StatusNotFound + } else if errors.Is(err, store.ErrBadRequest) { + status = http.StatusBadRequest + } + + return Err(status, "", "%s", err.Error()) +} + // ErrorHandler is a genral handler for echo handler errors func ErrorHandler(err error, c echo.Context) { var code int = 0 diff --git a/cluster/client/client.go b/cluster/client/client.go index 0dd417cb..a5df0f49 100644 --- a/cluster/client/client.go +++ b/cluster/client/client.go @@ -4,15 +4,14 @@ import ( "bytes" "fmt" "io" - "io/fs" "net/http" "net/url" + "strings" "time" "github.com/datarhei/core/v16/config" "github.com/datarhei/core/v16/encoding/json" "github.com/datarhei/core/v16/ffmpeg/skills" - httpapi "github.com/datarhei/core/v16/http/api" iamaccess "github.com/datarhei/core/v16/iam/access" iamidentity "github.com/datarhei/core/v16/iam/identity" "github.com/datarhei/core/v16/restream/app" @@ -40,7 +39,7 @@ type SetProcessMetadataRequest struct { } type RelocateProcessesRequest struct { - Map map[app.ProcessID]string + Map map[app.ProcessID]string `json:"map"` } type AddIdentityRequest struct { @@ -70,6 +69,24 @@ type GetKVResponse struct { UpdatedAt time.Time `json:"updated_at"` } +type AboutResponse struct { + ID string `json:"id"` + Version string `json:"version"` + Address string `json:"address"` + StartedAt time.Time `json:"started_at"` + Resources AboutResponseResources `json:"resources"` +} + +type AboutResponseResources struct { + IsThrottling bool `json:"is_throttling"` // Whether this core is currently throttling + NCPU float64 `json:"ncpu"` // Number of CPU on this node + CPU float64 `json:"cpu"` // Current CPU load, 0-100*ncpu + CPULimit float64 `json:"cpu_limit"` // Defined CPU load limit, 0-100*ncpu + Mem uint64 `json:"memory_bytes"` // Currently used memory in bytes + MemLimit uint64 `json:"memory_limit_bytes"` // Defined memory limit in bytes + Error string `json:"error"` // Last error +} + type SetNodeStateRequest struct { State string `json:"state"` } @@ -94,8 +111,23 @@ func (c *APIClient) Version() (string, error) { return version, nil } +func (c *APIClient) About() (AboutResponse, error) { + data, err := c.call(http.MethodGet, "/v1/about", "", nil, "") + if err != nil { + return AboutResponse{}, err + } + + var about AboutResponse + err = json.Unmarshal(data, &about) + if err != nil { + return AboutResponse{}, err + } + + return about, nil +} + func (c *APIClient) Barrier(name string) (bool, error) { - data, err := c.call(http.MethodGet, "/v1/barrier/"+url.PathEscape(name), "application/json", nil, "") + data, err := c.call(http.MethodGet, "/v1/barrier/"+url.PathEscape(name), "", nil, "") if err != nil { return false, err } @@ -177,177 +209,6 @@ func (c *APIClient) TransferLeadership(origin, id string) error { return err } -func (c *APIClient) AddProcess(origin string, r AddProcessRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPost, "/v1/process", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) RemoveProcess(origin string, id app.ProcessID) error { - _, err := c.call(http.MethodDelete, "/v1/process/"+url.PathEscape(id.ID)+"?domain="+url.QueryEscape(id.Domain), "application/json", nil, origin) - - return err -} - -func (c *APIClient) UpdateProcess(origin string, id app.ProcessID, r UpdateProcessRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) SetProcessCommand(origin string, id app.ProcessID, r SetProcessCommandRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"/command?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) SetProcessMetadata(origin string, id app.ProcessID, key string, r SetProcessMetadataRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"/metadata/"+url.PathEscape(key)+"?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) RelocateProcesses(origin string, r RelocateProcessesRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/relocate", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) AddIdentity(origin string, r AddIdentityRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPost, "/v1/iam/user", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) UpdateIdentity(origin, name string, r UpdateIdentityRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/iam/user/"+url.PathEscape(name), "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) SetPolicies(origin, name string, r SetPoliciesRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/iam/user/"+url.PathEscape(name)+"/policies", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) RemoveIdentity(origin string, name string) error { - _, err := c.call(http.MethodDelete, "/v1/iam/user/"+url.PathEscape(name), "application/json", nil, origin) - - return err -} - -func (c *APIClient) Lock(origin string, r LockRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPost, "/v1/lock", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) Unlock(origin string, name string) error { - _, err := c.call(http.MethodDelete, "/v1/lock/"+url.PathEscape(name), "application/json", nil, origin) - - return err -} - -func (c *APIClient) SetKV(origin string, r SetKVRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPost, "/v1/kv", "application/json", bytes.NewReader(data), origin) - - return err -} - -func (c *APIClient) UnsetKV(origin string, key string) error { - _, err := c.call(http.MethodDelete, "/v1/kv/"+url.PathEscape(key), "application/json", nil, origin) - if err != nil { - e, ok := err.(httpapi.Error) - if ok && e.Code == 404 { - return fs.ErrNotExist - } - } - - return err -} - -func (c *APIClient) GetKV(origin string, key string) (string, time.Time, error) { - data, err := c.call(http.MethodGet, "/v1/kv/"+url.PathEscape(key), "application/json", nil, origin) - if err != nil { - e, ok := err.(httpapi.Error) - if ok && e.Code == 404 { - return "", time.Time{}, fs.ErrNotExist - } - - return "", time.Time{}, err - } - - res := GetKVResponse{} - err = json.Unmarshal(data, &res) - if err != nil { - return "", time.Time{}, err - } - - return res.Value, res.UpdatedAt, nil -} - -func (c *APIClient) SetNodeState(origin string, nodeid string, r SetNodeStateRequest) error { - data, err := json.Marshal(r) - if err != nil { - return err - } - - _, err = c.call(http.MethodPut, "/v1/node/"+url.PathEscape(nodeid)+"/state", "application/json", bytes.NewReader(data), origin) - - return err -} - func (c *APIClient) Snapshot(origin string) (io.ReadCloser, error) { return c.stream(http.MethodGet, "/v1/snapshot", "", nil, origin) } @@ -376,7 +237,7 @@ func (c *APIClient) stream(method, path, contentType string, data io.Reader, ori } if status < 200 || status >= 300 { - e := httpapi.Error{} + e := Error{} defer body.Close() @@ -422,3 +283,14 @@ func (c *APIClient) request(req *http.Request) (int, io.ReadCloser, error) { return resp.StatusCode, resp.Body, nil } + +// Error represents an error response of the API +type Error struct { + Code int `json:"code" jsonschema:"required" format:"int"` + Message string `json:"message" jsonschema:""` + Details []string `json:"details" jsonschema:""` +} + +func (e Error) Error() string { + return strings.Join(e.Details, ", ") +} diff --git a/cluster/client/iam.go b/cluster/client/iam.go new file mode 100644 index 00000000..2f20a63f --- /dev/null +++ b/cluster/client/iam.go @@ -0,0 +1,48 @@ +package client + +import ( + "bytes" + "net/http" + "net/url" + + "github.com/datarhei/core/v16/encoding/json" +) + +func (c *APIClient) IAMIdentityAdd(origin string, r AddIdentityRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPost, "/v1/iam/user", "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) IAMIdentityUpdate(origin, name string, r UpdateIdentityRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/iam/user/"+url.PathEscape(name), "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) IAMPoliciesSet(origin, name string, r SetPoliciesRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/iam/user/"+url.PathEscape(name)+"/policies", "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) IAMIdentityRemove(origin string, name string) error { + _, err := c.call(http.MethodDelete, "/v1/iam/user/"+url.PathEscape(name), "application/json", nil, origin) + + return err +} diff --git a/cluster/client/kvs.go b/cluster/client/kvs.go new file mode 100644 index 00000000..4e9b5a4a --- /dev/null +++ b/cluster/client/kvs.go @@ -0,0 +1,54 @@ +package client + +import ( + "bytes" + "io/fs" + "net/http" + "net/url" + "time" + + "github.com/datarhei/core/v16/encoding/json" +) + +func (c *APIClient) KVSet(origin string, r SetKVRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPost, "/v1/kv", "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) KVUnset(origin string, key string) error { + _, err := c.call(http.MethodDelete, "/v1/kv/"+url.PathEscape(key), "application/json", nil, origin) + if err != nil { + e, ok := err.(Error) + if ok && e.Code == 404 { + return fs.ErrNotExist + } + } + + return err +} + +func (c *APIClient) KVGet(origin string, key string) (string, time.Time, error) { + data, err := c.call(http.MethodGet, "/v1/kv/"+url.PathEscape(key), "application/json", nil, origin) + if err != nil { + e, ok := err.(Error) + if ok && e.Code == 404 { + return "", time.Time{}, fs.ErrNotExist + } + + return "", time.Time{}, err + } + + res := GetKVResponse{} + err = json.Unmarshal(data, &res) + if err != nil { + return "", time.Time{}, err + } + + return res.Value, res.UpdatedAt, nil +} diff --git a/cluster/client/lock.go b/cluster/client/lock.go new file mode 100644 index 00000000..926f9ee1 --- /dev/null +++ b/cluster/client/lock.go @@ -0,0 +1,26 @@ +package client + +import ( + "bytes" + "net/http" + "net/url" + + "github.com/datarhei/core/v16/encoding/json" +) + +func (c *APIClient) LockCreate(origin string, r LockRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPost, "/v1/lock", "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) LockDelete(origin string, name string) error { + _, err := c.call(http.MethodDelete, "/v1/lock/"+url.PathEscape(name), "application/json", nil, origin) + + return err +} diff --git a/cluster/client/node.go b/cluster/client/node.go new file mode 100644 index 00000000..6ea61be2 --- /dev/null +++ b/cluster/client/node.go @@ -0,0 +1,20 @@ +package client + +import ( + "bytes" + "net/http" + "net/url" + + "github.com/datarhei/core/v16/encoding/json" +) + +func (c *APIClient) NodeSetState(origin string, nodeid string, r SetNodeStateRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/node/"+url.PathEscape(nodeid)+"/state", "application/json", bytes.NewReader(data), origin) + + return err +} diff --git a/cluster/client/proces.go b/cluster/client/proces.go new file mode 100644 index 00000000..aee4dd14 --- /dev/null +++ b/cluster/client/proces.go @@ -0,0 +1,71 @@ +package client + +import ( + "bytes" + "net/http" + "net/url" + + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/restream/app" +) + +func (c *APIClient) ProcessAdd(origin string, r AddProcessRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPost, "/v1/process", "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) ProcessRemove(origin string, id app.ProcessID) error { + _, err := c.call(http.MethodDelete, "/v1/process/"+url.PathEscape(id.ID)+"?domain="+url.QueryEscape(id.Domain), "application/json", nil, origin) + + return err +} + +func (c *APIClient) ProcessUpdate(origin string, id app.ProcessID, r UpdateProcessRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) ProcessSetCommand(origin string, id app.ProcessID, r SetProcessCommandRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"/command?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) ProcessSetMetadata(origin string, id app.ProcessID, key string, r SetProcessMetadataRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/process/"+url.PathEscape(id.ID)+"/metadata/"+url.PathEscape(key)+"?domain="+url.QueryEscape(id.Domain), "application/json", bytes.NewReader(data), origin) + + return err +} + +func (c *APIClient) ProcessesRelocate(origin string, r RelocateProcessesRequest) error { + data, err := json.Marshal(r) + if err != nil { + return err + } + + _, err = c.call(http.MethodPut, "/v1/relocate", "application/json", bytes.NewReader(data), origin) + + return err +} diff --git a/cluster/cluster.go b/cluster/cluster.go index 4866bb86..c7add98b 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -7,7 +7,6 @@ import ( "io" gonet "net" "net/url" - "sort" "strconv" "sync" "time" @@ -18,7 +17,6 @@ import ( "github.com/datarhei/core/v16/cluster/forwarder" "github.com/datarhei/core/v16/cluster/kvs" clusternode "github.com/datarhei/core/v16/cluster/node" - "github.com/datarhei/core/v16/cluster/proxy" "github.com/datarhei/core/v16/cluster/raft" "github.com/datarhei/core/v16/cluster/store" "github.com/datarhei/core/v16/config" @@ -29,8 +27,8 @@ import ( iamidentity "github.com/datarhei/core/v16/iam/identity" "github.com/datarhei/core/v16/log" "github.com/datarhei/core/v16/net" + "github.com/datarhei/core/v16/resources" "github.com/datarhei/core/v16/restream/app" - "github.com/datarhei/core/v16/slices" ) type Cluster interface { @@ -51,50 +49,42 @@ type Cluster interface { CoreSkills() skills.Skills About() (ClusterAbout, error) - IsClusterDegraded() (bool, error) - IsDegraded() (bool, error) GetBarrier(name string) bool Join(origin, id, raftAddress, peerAddress string) error Leave(origin, id string) error // gracefully remove a node from the cluster TransferLeadership(origin, id string) error // transfer leadership to another node Snapshot(origin string) (io.ReadCloser, error) + HasRaftLeader() bool - ListProcesses() []store.Process - GetProcess(id app.ProcessID) (store.Process, error) - AddProcess(origin string, config *app.Config) error - RemoveProcess(origin string, id app.ProcessID) error - UpdateProcess(origin string, id app.ProcessID, config *app.Config) error - SetProcessCommand(origin string, id app.ProcessID, order string) error - SetProcessMetadata(origin string, id app.ProcessID, key string, data interface{}) error - GetProcessMetadata(origin string, id app.ProcessID, key string) (interface{}, error) - GetProcessNodeMap() map[string]string - RelocateProcesses(origin string, relocations map[app.ProcessID]string) error + ProcessAdd(origin string, config *app.Config) error + ProcessRemove(origin string, id app.ProcessID) error + ProcessUpdate(origin string, id app.ProcessID, config *app.Config) error + ProcessSetCommand(origin string, id app.ProcessID, order string) error + ProcessSetMetadata(origin string, id app.ProcessID, key string, data interface{}) error + ProcessGetMetadata(origin string, id app.ProcessID, key string) (interface{}, error) + ProcessesRelocate(origin string, relocations map[app.ProcessID]string) error IAM(superuser iamidentity.User, jwtRealm, jwtSecret string) (iam.IAM, error) - ListIdentities() (time.Time, []iamidentity.User) - ListIdentity(name string) (time.Time, iamidentity.User, error) - ListPolicies() (time.Time, []iamaccess.Policy) - ListUserPolicies(name string) (time.Time, []iamaccess.Policy) - AddIdentity(origin string, identity iamidentity.User) error - UpdateIdentity(origin, name string, identity iamidentity.User) error - SetPolicies(origin, name string, policies []iamaccess.Policy) error - RemoveIdentity(origin string, name string) error + IAMIdentityAdd(origin string, identity iamidentity.User) error + IAMIdentityUpdate(origin, name string, identity iamidentity.User) error + IAMIdentityRemove(origin string, name string) error + IAMPoliciesSet(origin, name string, policies []iamaccess.Policy) error - CreateLock(origin string, name string, validUntil time.Time) (*kvs.Lock, error) - DeleteLock(origin string, name string) error - ListLocks() map[string]time.Time + LockCreate(origin string, name string, validUntil time.Time) (*kvs.Lock, error) + LockDelete(origin string, name string) error - SetKV(origin, key, value string) error - UnsetKV(origin, key string) error - GetKV(origin, key string, stale bool) (string, time.Time, error) - ListKV(prefix string) map[string]store.Value + KVSet(origin, key, value string) error + KVUnset(origin, key string) error + KVGet(origin, key string, stale bool) (string, time.Time, error) - ListNodes() map[string]store.Node - SetNodeState(origin, id, state string) error + NodeSetState(origin, id, state string) error - ProxyReader() proxy.ProxyReader + Manager() *clusternode.Manager CertManager() autocert.Manager + Store() store.Store + + Resources() (resources.Info, error) } type Peer struct { @@ -122,6 +112,7 @@ type Config struct { CoreSkills skills.Skills IPLimiter net.IPLimiter + Resources resources.Resources Logger log.Logger Debug DebugConfig @@ -154,20 +145,16 @@ type cluster struct { nodeRecoverTimeout time.Duration emergencyLeaderTimeout time.Duration - forwarder forwarder.Forwarder + forwarder *forwarder.Forwarder api API - proxy proxy.Proxy + manager *clusternode.Manager config *config.Config skills skills.Skills coreAddress string - isDegraded bool - isDegradedErr error - isCoreDegraded bool - isCoreDegradedErr error - hostnames []string - stateLock sync.RWMutex + hostnames []string + stateLock sync.RWMutex isRaftLeader bool hasRaftLeader bool @@ -178,14 +165,13 @@ type cluster struct { clusterKVS ClusterKVS certManager autocert.Manager - nodes map[string]clusternode.Node - nodesLock sync.RWMutex - barrier map[string]bool barrierLock sync.RWMutex limiter net.IPLimiter + resources resources.Resources + debugDisableFFmpegCheck bool } @@ -209,20 +195,15 @@ func New(config Config) (Cluster, error) { nodeRecoverTimeout: config.NodeRecoverTimeout, emergencyLeaderTimeout: config.EmergencyLeaderTimeout, - isDegraded: true, - isDegradedErr: fmt.Errorf("cluster not yet startet"), - - isCoreDegraded: true, - isCoreDegradedErr: fmt.Errorf("cluster not yet started"), - config: config.CoreConfig, skills: config.CoreSkills, - nodes: map[string]clusternode.Node{}, barrier: map[string]bool{}, limiter: config.IPLimiter, + resources: config.Resources, + debugDisableFFmpegCheck: config.Debug.DisableFFmpegCheck, } @@ -301,7 +282,7 @@ func New(config Config) (Cluster, error) { c.api = api - nodeproxy, err := proxy.NewProxy(proxy.ProxyConfig{ + nodemanager, err := clusternode.NewManager(clusternode.ManagerConfig{ ID: c.nodeID, Logger: c.logger.WithField("logname", "proxy"), }) @@ -310,13 +291,9 @@ func New(config Config) (Cluster, error) { return nil, err } - go func(nodeproxy proxy.Proxy) { - nodeproxy.Start() - }(nodeproxy) + c.manager = nodemanager - c.proxy = nodeproxy - - if forwarder, err := forwarder.New(forwarder.ForwarderConfig{ + if forwarder, err := forwarder.New(forwarder.Config{ ID: c.nodeID, Logger: c.logger.WithField("logname", "forwarder"), }); err != nil { @@ -475,12 +452,12 @@ func (c *cluster) setup(ctx context.Context) error { c.logger.Info().Log("Waiting for cluster to become operational ...") for { - ok, err := c.IsClusterDegraded() + ok, err := c.isClusterOperational() if !ok { break } - c.logger.Warn().WithError(err).Log("Cluster is in degraded state") + c.logger.Warn().WithError(err).Log("Waiting for all nodes to be registered") select { case <-ctx.Done(): @@ -644,15 +621,9 @@ func (c *cluster) Shutdown() error { c.shutdown = true close(c.shutdownCh) - for id, node := range c.nodes { - node.Stop() - if c.proxy != nil { - c.proxy.RemoveNode(id) - } - } - - if c.proxy != nil { - c.proxy.Stop() + if c.manager != nil { + c.manager.NodesClear() + c.manager = nil } if c.api != nil { @@ -677,56 +648,35 @@ func (c *cluster) IsRaftLeader() bool { return c.isRaftLeader } -func (c *cluster) IsDegraded() (bool, error) { - c.stateLock.RLock() - defer c.stateLock.RUnlock() +func (c *cluster) HasRaftLeader() bool { + c.leaderLock.Lock() + defer c.leaderLock.Unlock() - if c.isDegraded { - return c.isDegraded, c.isDegradedErr - } - - return c.isCoreDegraded, c.isCoreDegradedErr + return c.hasRaftLeader } -func (c *cluster) IsClusterDegraded() (bool, error) { - c.stateLock.Lock() - isDegraded, isDegradedErr := c.isDegraded, c.isDegradedErr - c.stateLock.Unlock() - - if isDegraded { - return isDegraded, isDegradedErr - } - +func (c *cluster) isClusterOperational() (bool, error) { servers, err := c.raft.Servers() if err != nil { return true, err } - c.nodesLock.RLock() - nodes := len(c.nodes) - c.nodesLock.RUnlock() + serverCount := len(servers) + nodeCount := c.manager.NodeCount() - if len(servers) != nodes { - return true, fmt.Errorf("not all nodes are connected") + if serverCount != nodeCount { + return true, fmt.Errorf("%d of %d nodes registered", nodeCount, serverCount) } return false, nil } func (c *cluster) Leave(origin, id string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - if len(id) == 0 { id = c.nodeID } - c.nodesLock.RLock() - _, hasNode := c.nodes[id] - c.nodesLock.RUnlock() - - if !hasNode { + if !c.manager.NodeHasNode(id) { return ErrUnknownNode } @@ -860,10 +810,6 @@ func (c *cluster) Leave(origin, id string) error { } func (c *cluster) Join(origin, id, raftAddress, peerAddress string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - if !c.IsRaftLeader() { c.logger.Debug().Log("Not leader, forwarding to leader") return c.forwarder.Join(origin, id, raftAddress, peerAddress) @@ -923,10 +869,6 @@ func (c *cluster) Join(origin, id, raftAddress, peerAddress string) error { } func (c *cluster) TransferLeadership(origin, id string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - if !c.IsRaftLeader() { c.logger.Debug().Log("Not leader, forwarding to leader") return c.forwarder.TransferLeadership(origin, id) @@ -936,10 +878,6 @@ func (c *cluster) TransferLeadership(origin, id string) error { } func (c *cluster) Snapshot(origin string) (io.ReadCloser, error) { - if ok, _ := c.IsDegraded(); ok { - return nil, ErrDegraded - } - if !c.IsRaftLeader() { c.logger.Debug().Log("Not leader, forwarding to leader") return c.forwarder.Snapshot(origin) @@ -962,18 +900,15 @@ func (c *cluster) trackNodeChanges() { continue } - c.nodesLock.Lock() - removeNodes := map[string]struct{}{} - for id := range c.nodes { + for _, id := range c.manager.NodeIDs() { removeNodes[id] = struct{}{} } for _, server := range servers { id := server.ID - _, ok := c.nodes[id] - if !ok { + if !c.manager.NodeHasNode(id) { logger := c.logger.WithFields(log.Fields{ "id": server.ID, "address": server.Address, @@ -993,18 +928,12 @@ func (c *cluster) trackNodeChanges() { }), }) - if err := verifyClusterVersion(node.Version()); err != nil { - logger.Warn().Log("Version mismatch. Cluster will end up in degraded mode") - } - - if _, err := c.proxy.AddNode(id, node.Proxy()); err != nil { + if _, err := c.manager.NodeAdd(id, node); err != nil { logger.Warn().WithError(err).Log("Adding node") node.Stop() continue } - c.nodes[id] = node - ips := node.IPs() for _, ip := range ips { c.limiter.AddBlock(ip) @@ -1015,57 +944,21 @@ func (c *cluster) trackNodeChanges() { } for id := range removeNodes { - node, ok := c.nodes[id] - if !ok { - continue - } - - c.proxy.RemoveNode(id) - node.Stop() - - ips := node.IPs() - for _, ip := range ips { - c.limiter.RemoveBlock(ip) - } - - delete(c.nodes, id) - /* - if id == c.nodeID { - c.logger.Warn().WithField("id", id).Log("This node left the cluster. Shutting down.") - // We got removed from the cluster, shutdown - c.Shutdown() + if node, err := c.manager.NodeRemove(id); err != nil { + ips := node.IPs() + for _, ip := range ips { + c.limiter.RemoveBlock(ip) } - */ + } } - c.nodesLock.Unlock() - // Put the cluster in "degraded" mode in case there's a mismatch in expected values - hostnames, err := c.checkClusterNodes() + c.manager.NodeCheckCompatibility(c.debugDisableFFmpegCheck) + + hostnames, _ := c.manager.GetHostnames(true) c.stateLock.Lock() - if err != nil { - c.isDegraded = true - c.isDegradedErr = err - c.hostnames = []string{} - } else { - c.isDegraded = false - c.isDegradedErr = nil - c.hostnames = hostnames - } - c.stateLock.Unlock() - - // Put the cluster in "coreDegraded" mode in case there's a mismatch in expected values - err = c.checkClusterCoreNodes() - - c.stateLock.Lock() - if err != nil { - c.isCoreDegraded = true - c.isCoreDegradedErr = err - } else { - c.isCoreDegraded = false - c.isCoreDegradedErr = nil - } + c.hostnames = hostnames c.stateLock.Unlock() case <-c.shutdownCh: return @@ -1073,240 +966,15 @@ func (c *cluster) trackNodeChanges() { } } -// checkClusterNodes returns a list of hostnames that are configured on all nodes. The -// returned list will not contain any duplicates. An error is returned in case the -// node is not compatible. -func (c *cluster) checkClusterNodes() ([]string, error) { - hostnames := map[string]int{} - - c.nodesLock.RLock() - defer c.nodesLock.RUnlock() - - for id, node := range c.nodes { - if status, err := node.Status(); status == "offline" { - return nil, fmt.Errorf("node %s is offline: %w", id, err) - } - - version := node.Version() - if err := verifyClusterVersion(version); err != nil { - return nil, fmt.Errorf("node %s has a different cluster version: %s: %w", id, version, err) - } - - config, err := node.CoreConfig() - if err != nil { - return nil, fmt.Errorf("node %s has no configuration available: %w", id, err) - } - if err := verifyClusterConfig(c.config, config); err != nil { - return nil, fmt.Errorf("node %s has a different configuration: %w", id, err) - } - - if !c.debugDisableFFmpegCheck { - skills, err := node.CoreSkills() - if err != nil { - return nil, fmt.Errorf("node %s has no FFmpeg skills available: %w", id, err) - } - if !c.skills.Equal(skills) { - return nil, fmt.Errorf("node %s has mismatching FFmpeg skills", id) - } - } - - for _, name := range config.Host.Name { - hostnames[name]++ - } - } - - names := []string{} - - for key, value := range hostnames { - if value != len(c.nodes) { - continue - } - - names = append(names, key) - } - - sort.Strings(names) - - return names, nil -} - -func (c *cluster) checkClusterCoreNodes() error { - c.nodesLock.RLock() - defer c.nodesLock.RUnlock() - - for id, node := range c.nodes { - if status, err := node.CoreStatus(); status == "offline" { - return fmt.Errorf("node %s core is offline: %w", id, err) - } - } - - return nil -} - // getClusterHostnames return a list of all hostnames configured on all nodes. The // returned list will not contain any duplicates. func (c *cluster) getClusterHostnames() ([]string, error) { - hostnames := map[string]struct{}{} - - c.nodesLock.RLock() - defer c.nodesLock.RUnlock() - - for id, node := range c.nodes { - config, err := node.CoreConfig() - if err != nil { - return nil, fmt.Errorf("node %s has no configuration available: %w", id, err) - } - - for _, name := range config.Host.Name { - hostnames[name] = struct{}{} - } - } - - names := []string{} - - for key := range hostnames { - names = append(names, key) - } - - sort.Strings(names) - - return names, nil + return c.manager.GetHostnames(false) } // getClusterBarrier returns whether all nodes are currently on the same barrier. func (c *cluster) getClusterBarrier(name string) (bool, error) { - c.nodesLock.RLock() - defer c.nodesLock.RUnlock() - - for _, node := range c.nodes { - ok, err := node.Barrier(name) - if !ok { - return false, err - } - } - - return true, nil -} - -func verifyClusterVersion(v string) error { - version, err := ParseClusterVersion(v) - if err != nil { - return fmt.Errorf("parsing version %s: %w", v, err) - } - - if !Version.Equal(version) { - return fmt.Errorf("version %s not equal to my version %s", version.String(), Version.String()) - } - - return nil -} - -func verifyClusterConfig(local, remote *config.Config) error { - if local == nil || remote == nil { - return fmt.Errorf("config is not available") - } - - if local.Cluster.Enable != remote.Cluster.Enable { - return fmt.Errorf("cluster.enable is different") - } - - if local.Cluster.ID != remote.Cluster.ID { - return fmt.Errorf("cluster.id is different") - } - - if local.Cluster.SyncInterval != remote.Cluster.SyncInterval { - return fmt.Errorf("cluster.sync_interval_sec is different") - } - - if local.Cluster.NodeRecoverTimeout != remote.Cluster.NodeRecoverTimeout { - return fmt.Errorf("cluster.node_recover_timeout_sec is different") - } - - if local.Cluster.EmergencyLeaderTimeout != remote.Cluster.EmergencyLeaderTimeout { - return fmt.Errorf("cluster.emergency_leader_timeout_sec is different") - } - - if local.Cluster.Debug.DisableFFmpegCheck != remote.Cluster.Debug.DisableFFmpegCheck { - return fmt.Errorf("cluster.debug.disable_ffmpeg_check is different") - } - - if !local.API.Auth.Enable { - return fmt.Errorf("api.auth.enable must be true") - } - - if local.API.Auth.Enable != remote.API.Auth.Enable { - return fmt.Errorf("api.auth.enable is different") - } - - if local.API.Auth.Username != remote.API.Auth.Username { - return fmt.Errorf("api.auth.username is different") - } - - if local.API.Auth.Password != remote.API.Auth.Password { - return fmt.Errorf("api.auth.password is different") - } - - if local.API.Auth.JWT.Secret != remote.API.Auth.JWT.Secret { - return fmt.Errorf("api.auth.jwt.secret is different") - } - - if local.RTMP.Enable != remote.RTMP.Enable { - return fmt.Errorf("rtmp.enable is different") - } - - if local.RTMP.Enable { - if local.RTMP.App != remote.RTMP.App { - return fmt.Errorf("rtmp.app is different") - } - } - - if local.SRT.Enable != remote.SRT.Enable { - return fmt.Errorf("srt.enable is different") - } - - if local.SRT.Enable { - if local.SRT.Passphrase != remote.SRT.Passphrase { - return fmt.Errorf("srt.passphrase is different") - } - } - - if local.Resources.MaxCPUUsage == 0 || remote.Resources.MaxCPUUsage == 0 { - return fmt.Errorf("resources.max_cpu_usage must be defined") - } - - if local.Resources.MaxMemoryUsage == 0 || remote.Resources.MaxMemoryUsage == 0 { - return fmt.Errorf("resources.max_memory_usage must be defined") - } - - if local.TLS.Enable != remote.TLS.Enable { - return fmt.Errorf("tls.enable is different") - } - - if local.TLS.Enable { - if local.TLS.Auto != remote.TLS.Auto { - return fmt.Errorf("tls.auto is different") - } - - if len(local.Host.Name) == 0 || len(remote.Host.Name) == 0 { - return fmt.Errorf("host.name must be set") - } - - if local.TLS.Auto { - if local.TLS.Email != remote.TLS.Email { - return fmt.Errorf("tls.email is different") - } - - if local.TLS.Staging != remote.TLS.Staging { - return fmt.Errorf("tls.staging is different") - } - - if local.TLS.Secret != remote.TLS.Secret { - return fmt.Errorf("tls.secret is different") - } - } - } - - return nil + return c.manager.Barrier(name) } // trackLeaderChanges registers an Observer with raft in order to receive updates @@ -1370,169 +1038,6 @@ func (c *cluster) applyCommand(cmd *store.Command) error { return nil } -type ClusterRaft struct { - Address string - State string - LastContact time.Duration - NumPeers uint64 - LogTerm uint64 - LogIndex uint64 -} - -type ClusterNodeResources struct { - IsThrottling bool // Whether this core is currently throttling - NCPU float64 // Number of CPU on this node - CPU float64 // Current CPU load, 0-100*ncpu - CPULimit float64 // Defined CPU load limit, 0-100*ncpu - Mem uint64 // Currently used memory in bytes - MemLimit uint64 // Defined memory limit in bytes - Error error -} - -type ClusterNode struct { - ID string - Name string - Version string - Status string - Error error - Voter bool - Leader bool - Address string - CreatedAt time.Time - Uptime time.Duration - LastContact time.Duration - Latency time.Duration - Core ClusterNodeCore - Resources ClusterNodeResources -} - -type ClusterNodeCore struct { - Address string - Status string - Error error - LastContact time.Duration - Latency time.Duration - Version string -} - -type ClusterAboutLeader struct { - ID string - Address string - ElectedSince time.Duration -} - -type ClusterAbout struct { - ID string - Domains []string - Leader ClusterAboutLeader - Status string - Raft ClusterRaft - Nodes []ClusterNode - Version ClusterVersion - Degraded bool - DegradedErr error -} - -func (c *cluster) About() (ClusterAbout, error) { - degraded, degradedErr := c.IsDegraded() - - about := ClusterAbout{ - ID: c.id, - Leader: ClusterAboutLeader{}, - Status: "online", - Version: Version, - Degraded: degraded, - DegradedErr: degradedErr, - } - - if about.Degraded { - about.Status = "offline" - } - - c.stateLock.RLock() - about.Domains = slices.Copy(c.hostnames) - c.stateLock.RUnlock() - - stats := c.raft.Stats() - - about.Raft.Address = stats.Address - about.Raft.State = stats.State - about.Raft.LastContact = stats.LastContact - about.Raft.NumPeers = stats.NumPeers - about.Raft.LogIndex = stats.LogIndex - about.Raft.LogTerm = stats.LogTerm - - servers, err := c.raft.Servers() - if err != nil { - c.logger.Warn().WithError(err).Log("Raft configuration") - } - - serversMap := map[string]raft.Server{} - - for _, s := range servers { - serversMap[s.ID] = s - - if s.Leader { - about.Leader.ID = s.ID - about.Leader.Address = s.Address - about.Leader.ElectedSince = s.LastChange - } - } - - storeNodes := c.ListNodes() - - c.nodesLock.RLock() - for id, node := range c.nodes { - nodeAbout := node.About() - - node := ClusterNode{ - ID: id, - Name: nodeAbout.Name, - Version: nodeAbout.Version, - Status: nodeAbout.Status, - Error: nodeAbout.Error, - Address: nodeAbout.Address, - LastContact: nodeAbout.LastContact, - Latency: nodeAbout.Latency, - CreatedAt: nodeAbout.Core.CreatedAt, - Uptime: nodeAbout.Core.Uptime, - Core: ClusterNodeCore{ - Address: nodeAbout.Core.Address, - Status: nodeAbout.Core.Status, - Error: nodeAbout.Core.Error, - LastContact: nodeAbout.Core.LastContact, - Latency: nodeAbout.Core.Latency, - Version: nodeAbout.Core.Version, - }, - Resources: ClusterNodeResources{ - IsThrottling: nodeAbout.Resources.IsThrottling, - NCPU: nodeAbout.Resources.NCPU, - CPU: nodeAbout.Resources.CPU, - CPULimit: nodeAbout.Resources.CPULimit, - Mem: nodeAbout.Resources.Mem, - MemLimit: nodeAbout.Resources.MemLimit, - Error: nodeAbout.Resources.Error, - }, - } - - if s, ok := serversMap[id]; ok { - node.Voter = s.Voter - node.Leader = s.Leader - } - - if storeNode, hasStoreNode := storeNodes[id]; hasStoreNode { - if storeNode.State == "maintenance" { - node.Status = storeNode.State - } - } - - about.Nodes = append(about.Nodes, node) - } - c.nodesLock.RUnlock() - - return about, nil -} - func (c *cluster) sentinel() { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -1570,6 +1075,26 @@ func (c *cluster) sentinel() { } } -func (c *cluster) ProxyReader() proxy.ProxyReader { - return c.proxy.Reader() +func (c *cluster) Resources() (resources.Info, error) { + if c.resources == nil { + return resources.Info{}, fmt.Errorf("resource information is not available") + } + + return c.resources.Info(), nil +} + +func (c *cluster) Manager() *clusternode.Manager { + if c.manager == nil { + return nil + } + + return c.manager +} + +func (c *cluster) Store() store.Store { + if c.store == nil { + return nil + } + + return c.store } diff --git a/cluster/forwarder/errors.go b/cluster/forwarder/errors.go new file mode 100644 index 00000000..0d1fda38 --- /dev/null +++ b/cluster/forwarder/errors.go @@ -0,0 +1,21 @@ +package forwarder + +import ( + "fmt" + + apiclient "github.com/datarhei/core/v16/cluster/client" + "github.com/datarhei/core/v16/cluster/store" +) + +func reconstructError(err error) error { + if cerr, ok := err.(apiclient.Error); ok { + switch cerr.Code { + case 400: + err = fmt.Errorf("%s%w", err.Error(), store.ErrBadRequest) + case 404: + err = fmt.Errorf("%s%w", err.Error(), store.ErrNotFound) + } + } + + return err +} diff --git a/cluster/forwarder/forwarder.go b/cluster/forwarder/forwarder.go index cac84815..90d600db 100644 --- a/cluster/forwarder/forwarder.go +++ b/cluster/forwarder/forwarder.go @@ -7,66 +7,31 @@ import ( "time" apiclient "github.com/datarhei/core/v16/cluster/client" - iamaccess "github.com/datarhei/core/v16/iam/access" - iamidentity "github.com/datarhei/core/v16/iam/identity" "github.com/datarhei/core/v16/log" - "github.com/datarhei/core/v16/restream/app" ) // Forwarder forwards any HTTP request from a follower to the leader -type Forwarder interface { - SetLeader(address string) - HasLeader() bool - - Join(origin, id, raftAddress, peerAddress string) error - Leave(origin, id string) error - TransferLeadership(origin, id string) error - Snapshot(origin string) (io.ReadCloser, error) - - AddProcess(origin string, config *app.Config) error - UpdateProcess(origin string, id app.ProcessID, config *app.Config) error - RemoveProcess(origin string, id app.ProcessID) error - SetProcessCommand(origin string, id app.ProcessID, command string) error - SetProcessMetadata(origin string, id app.ProcessID, key string, data interface{}) error - RelocateProcesses(origin string, relocations map[app.ProcessID]string) error - - AddIdentity(origin string, identity iamidentity.User) error - UpdateIdentity(origin, name string, identity iamidentity.User) error - SetPolicies(origin, name string, policies []iamaccess.Policy) error - RemoveIdentity(origin string, name string) error - - CreateLock(origin string, name string, validUntil time.Time) error - DeleteLock(origin string, name string) error - - SetKV(origin, key, value string) error - UnsetKV(origin, key string) error - GetKV(origin, key string) (string, time.Time, error) - - SetNodeState(origin, nodeid, state string) error -} - -type forwarder struct { - id string - lock sync.RWMutex +type Forwarder struct { + ID string + Logger log.Logger + lock sync.RWMutex client apiclient.APIClient - - logger log.Logger } -type ForwarderConfig struct { +type Config struct { ID string Logger log.Logger } -func New(config ForwarderConfig) (Forwarder, error) { - f := &forwarder{ - id: config.ID, - logger: config.Logger, +func New(config Config) (*Forwarder, error) { + f := &Forwarder{ + ID: config.ID, + Logger: config.Logger, } - if f.logger == nil { - f.logger = log.New("") + if f.Logger == nil { + f.Logger = log.New("") } tr := http.DefaultTransport.(*http.Transport).Clone() @@ -85,7 +50,7 @@ func New(config ForwarderConfig) (Forwarder, error) { return f, nil } -func (f *forwarder) SetLeader(address string) { +func (f *Forwarder) SetLeader(address string) { f.lock.Lock() defer f.lock.Unlock() @@ -93,18 +58,18 @@ func (f *forwarder) SetLeader(address string) { return } - f.logger.Debug().Log("Setting leader address to %s", address) + f.Logger.Debug().Log("Setting leader address to %s", address) f.client.Address = address } -func (f *forwarder) HasLeader() bool { +func (f *Forwarder) HasLeader() bool { return len(f.client.Address) != 0 } -func (f *forwarder) Join(origin, id, raftAddress, peerAddress string) error { +func (f *Forwarder) Join(origin, id, raftAddress, peerAddress string) error { if origin == "" { - origin = f.id + origin = f.ID } r := apiclient.JoinRequest{ @@ -112,7 +77,7 @@ func (f *forwarder) Join(origin, id, raftAddress, peerAddress string) error { RaftAddress: raftAddress, } - f.logger.Debug().WithField("request", r).Log("Forwarding to leader") + f.Logger.Debug().WithField("request", r).Log("Forwarding to leader") f.lock.RLock() client := f.client @@ -128,12 +93,12 @@ func (f *forwarder) Join(origin, id, raftAddress, peerAddress string) error { return client.Join(origin, r) } -func (f *forwarder) Leave(origin, id string) error { +func (f *Forwarder) Leave(origin, id string) error { if origin == "" { - origin = f.id + origin = f.ID } - f.logger.Debug().WithField("id", id).Log("Forwarding to leader") + f.Logger.Debug().WithField("id", id).Log("Forwarding to leader") f.lock.RLock() client := f.client @@ -142,12 +107,12 @@ func (f *forwarder) Leave(origin, id string) error { return client.Leave(origin, id) } -func (f *forwarder) TransferLeadership(origin, id string) error { +func (f *Forwarder) TransferLeadership(origin, id string) error { if origin == "" { - origin = f.id + origin = f.ID } - f.logger.Debug().WithField("id", id).Log("Transferring leadership") + f.Logger.Debug().WithField("id", id).Log("Transferring leadership") f.lock.RLock() client := f.client @@ -156,248 +121,10 @@ func (f *forwarder) TransferLeadership(origin, id string) error { return client.TransferLeadership(origin, id) } -func (f *forwarder) Snapshot(origin string) (io.ReadCloser, error) { +func (f *Forwarder) Snapshot(origin string) (io.ReadCloser, error) { f.lock.RLock() client := f.client f.lock.RUnlock() return client.Snapshot(origin) } - -func (f *forwarder) AddProcess(origin string, config *app.Config) error { - if origin == "" { - origin = f.id - } - - r := apiclient.AddProcessRequest{ - Config: *config, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.AddProcess(origin, r) -} - -func (f *forwarder) UpdateProcess(origin string, id app.ProcessID, config *app.Config) error { - if origin == "" { - origin = f.id - } - - r := apiclient.UpdateProcessRequest{ - Config: *config, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.UpdateProcess(origin, id, r) -} - -func (f *forwarder) SetProcessCommand(origin string, id app.ProcessID, command string) error { - if origin == "" { - origin = f.id - } - - r := apiclient.SetProcessCommandRequest{ - Command: command, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.SetProcessCommand(origin, id, r) -} - -func (f *forwarder) SetProcessMetadata(origin string, id app.ProcessID, key string, data interface{}) error { - if origin == "" { - origin = f.id - } - - r := apiclient.SetProcessMetadataRequest{ - Metadata: data, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.SetProcessMetadata(origin, id, key, r) -} - -func (f *forwarder) RemoveProcess(origin string, id app.ProcessID) error { - if origin == "" { - origin = f.id - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.RemoveProcess(origin, id) -} - -func (f *forwarder) RelocateProcesses(origin string, relocations map[app.ProcessID]string) error { - if origin == "" { - origin = f.id - } - - r := apiclient.RelocateProcessesRequest{ - Map: relocations, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.RelocateProcesses(origin, r) -} - -func (f *forwarder) AddIdentity(origin string, identity iamidentity.User) error { - if origin == "" { - origin = f.id - } - - r := apiclient.AddIdentityRequest{ - Identity: identity, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.AddIdentity(origin, r) -} - -func (f *forwarder) UpdateIdentity(origin, name string, identity iamidentity.User) error { - if origin == "" { - origin = f.id - } - - r := apiclient.UpdateIdentityRequest{ - Identity: identity, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.UpdateIdentity(origin, name, r) -} - -func (f *forwarder) SetPolicies(origin, name string, policies []iamaccess.Policy) error { - if origin == "" { - origin = f.id - } - - r := apiclient.SetPoliciesRequest{ - Policies: policies, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.SetPolicies(origin, name, r) -} - -func (f *forwarder) RemoveIdentity(origin string, name string) error { - if origin == "" { - origin = f.id - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.RemoveIdentity(origin, name) -} - -func (f *forwarder) CreateLock(origin string, name string, validUntil time.Time) error { - if origin == "" { - origin = f.id - } - - r := apiclient.LockRequest{ - Name: name, - ValidUntil: validUntil, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.Lock(origin, r) -} - -func (f *forwarder) DeleteLock(origin string, name string) error { - if origin == "" { - origin = f.id - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.Unlock(origin, name) -} - -func (f *forwarder) SetKV(origin, key, value string) error { - if origin == "" { - origin = f.id - } - - r := apiclient.SetKVRequest{ - Key: key, - Value: value, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.SetKV(origin, r) -} - -func (f *forwarder) UnsetKV(origin, key string) error { - if origin == "" { - origin = f.id - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.UnsetKV(origin, key) -} - -func (f *forwarder) GetKV(origin, key string) (string, time.Time, error) { - if origin == "" { - origin = f.id - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.GetKV(origin, key) -} - -func (f *forwarder) SetNodeState(origin, nodeid, state string) error { - if origin == "" { - origin = f.id - } - - r := apiclient.SetNodeStateRequest{ - State: state, - } - - f.lock.RLock() - client := f.client - f.lock.RUnlock() - - return client.SetNodeState(origin, nodeid, r) -} diff --git a/cluster/forwarder/iam.go b/cluster/forwarder/iam.go new file mode 100644 index 00000000..bd695380 --- /dev/null +++ b/cluster/forwarder/iam.go @@ -0,0 +1,67 @@ +package forwarder + +import ( + apiclient "github.com/datarhei/core/v16/cluster/client" + iamaccess "github.com/datarhei/core/v16/iam/access" + iamidentity "github.com/datarhei/core/v16/iam/identity" +) + +func (f *Forwarder) IAMIdentityAdd(origin string, identity iamidentity.User) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.AddIdentityRequest{ + Identity: identity, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.IAMIdentityAdd(origin, r)) +} + +func (f *Forwarder) IAMIdentityUpdate(origin, name string, identity iamidentity.User) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.UpdateIdentityRequest{ + Identity: identity, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.IAMIdentityUpdate(origin, name, r)) +} + +func (f *Forwarder) IAMPoliciesSet(origin, name string, policies []iamaccess.Policy) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.SetPoliciesRequest{ + Policies: policies, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.IAMPoliciesSet(origin, name, r)) +} + +func (f *Forwarder) IAMIdentityRemove(origin string, name string) error { + if origin == "" { + origin = f.ID + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.IAMIdentityRemove(origin, name)) +} diff --git a/cluster/forwarder/kvs.go b/cluster/forwarder/kvs.go new file mode 100644 index 00000000..5aeaa953 --- /dev/null +++ b/cluster/forwarder/kvs.go @@ -0,0 +1,50 @@ +package forwarder + +import ( + "time" + + apiclient "github.com/datarhei/core/v16/cluster/client" +) + +func (f *Forwarder) KVSet(origin, key, value string) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.SetKVRequest{ + Key: key, + Value: value, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.KVSet(origin, r)) +} + +func (f *Forwarder) KVUnset(origin, key string) error { + if origin == "" { + origin = f.ID + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.KVUnset(origin, key)) +} + +func (f *Forwarder) KVGet(origin, key string) (string, time.Time, error) { + if origin == "" { + origin = f.ID + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + value, at, err := client.KVGet(origin, key) + + return value, at, reconstructError(err) +} diff --git a/cluster/forwarder/lock.go b/cluster/forwarder/lock.go new file mode 100644 index 00000000..1b3df004 --- /dev/null +++ b/cluster/forwarder/lock.go @@ -0,0 +1,36 @@ +package forwarder + +import ( + "time" + + apiclient "github.com/datarhei/core/v16/cluster/client" +) + +func (f *Forwarder) LockCreate(origin string, name string, validUntil time.Time) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.LockRequest{ + Name: name, + ValidUntil: validUntil, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.LockCreate(origin, r)) +} + +func (f *Forwarder) LockDelete(origin string, name string) error { + if origin == "" { + origin = f.ID + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.LockDelete(origin, name)) +} diff --git a/cluster/forwarder/node.go b/cluster/forwarder/node.go new file mode 100644 index 00000000..8991ec44 --- /dev/null +++ b/cluster/forwarder/node.go @@ -0,0 +1,21 @@ +package forwarder + +import ( + apiclient "github.com/datarhei/core/v16/cluster/client" +) + +func (f *Forwarder) NodeSetState(origin, nodeid, state string) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.SetNodeStateRequest{ + State: state, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.NodeSetState(origin, nodeid, r)) +} diff --git a/cluster/forwarder/process.go b/cluster/forwarder/process.go new file mode 100644 index 00000000..12a36392 --- /dev/null +++ b/cluster/forwarder/process.go @@ -0,0 +1,98 @@ +package forwarder + +import ( + apiclient "github.com/datarhei/core/v16/cluster/client" + "github.com/datarhei/core/v16/restream/app" +) + +func (f *Forwarder) ProcessAdd(origin string, config *app.Config) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.AddProcessRequest{ + Config: *config, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessAdd(origin, r)) +} + +func (f *Forwarder) ProcessUpdate(origin string, id app.ProcessID, config *app.Config) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.UpdateProcessRequest{ + Config: *config, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessUpdate(origin, id, r)) +} + +func (f *Forwarder) ProcessSetCommand(origin string, id app.ProcessID, command string) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.SetProcessCommandRequest{ + Command: command, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessSetCommand(origin, id, r)) +} + +func (f *Forwarder) ProcessSetMetadata(origin string, id app.ProcessID, key string, data interface{}) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.SetProcessMetadataRequest{ + Metadata: data, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessSetMetadata(origin, id, key, r)) +} + +func (f *Forwarder) ProcessRemove(origin string, id app.ProcessID) error { + if origin == "" { + origin = f.ID + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessRemove(origin, id)) +} + +func (f *Forwarder) ProcessesRelocate(origin string, relocations map[app.ProcessID]string) error { + if origin == "" { + origin = f.ID + } + + r := apiclient.RelocateProcessesRequest{ + Map: relocations, + } + + f.lock.RLock() + client := f.client + f.lock.RUnlock() + + return reconstructError(client.ProcessesRelocate(origin, r)) +} diff --git a/cluster/iam.go b/cluster/iam.go index 79da50ac..2dc30616 100644 --- a/cluster/iam.go +++ b/cluster/iam.go @@ -39,13 +39,13 @@ func (c *cluster) IAM(superuser iamidentity.User, jwtRealm, jwtSecret string) (i } func (c *cluster) ListIdentities() (time.Time, []iamidentity.User) { - users := c.store.ListUsers() + users := c.store.IAMIdentityList() return users.UpdatedAt, users.Users } func (c *cluster) ListIdentity(name string) (time.Time, iamidentity.User, error) { - user := c.store.GetUser(name) + user := c.store.IAMIdentityGet(name) if len(user.Users) == 0 { return time.Time{}, iamidentity.User{}, fmt.Errorf("not found") @@ -55,28 +55,24 @@ func (c *cluster) ListIdentity(name string) (time.Time, iamidentity.User, error) } func (c *cluster) ListPolicies() (time.Time, []iamaccess.Policy) { - policies := c.store.ListPolicies() + policies := c.store.IAMPolicyList() return policies.UpdatedAt, policies.Policies } func (c *cluster) ListUserPolicies(name string) (time.Time, []iamaccess.Policy) { - policies := c.store.ListUserPolicies(name) + policies := c.store.IAMIdentityPolicyList(name) return policies.UpdatedAt, policies.Policies } -func (c *cluster) AddIdentity(origin string, identity iamidentity.User) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) IAMIdentityAdd(origin string, identity iamidentity.User) error { if err := identity.Validate(); err != nil { return fmt.Errorf("invalid identity: %w", err) } if !c.IsRaftLeader() { - return c.forwarder.AddIdentity(origin, identity) + return c.forwarder.IAMIdentityAdd(origin, identity) } cmd := &store.Command{ @@ -89,13 +85,9 @@ func (c *cluster) AddIdentity(origin string, identity iamidentity.User) error { return c.applyCommand(cmd) } -func (c *cluster) UpdateIdentity(origin, name string, identity iamidentity.User) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) IAMIdentityUpdate(origin, name string, identity iamidentity.User) error { if !c.IsRaftLeader() { - return c.forwarder.UpdateIdentity(origin, name, identity) + return c.forwarder.IAMIdentityUpdate(origin, name, identity) } cmd := &store.Command{ @@ -109,13 +101,9 @@ func (c *cluster) UpdateIdentity(origin, name string, identity iamidentity.User) return c.applyCommand(cmd) } -func (c *cluster) SetPolicies(origin, name string, policies []iamaccess.Policy) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) IAMPoliciesSet(origin, name string, policies []iamaccess.Policy) error { if !c.IsRaftLeader() { - return c.forwarder.SetPolicies(origin, name, policies) + return c.forwarder.IAMPoliciesSet(origin, name, policies) } cmd := &store.Command{ @@ -129,13 +117,9 @@ func (c *cluster) SetPolicies(origin, name string, policies []iamaccess.Policy) return c.applyCommand(cmd) } -func (c *cluster) RemoveIdentity(origin string, name string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) IAMIdentityRemove(origin string, name string) error { if !c.IsRaftLeader() { - return c.forwarder.RemoveIdentity(origin, name) + return c.forwarder.IAMIdentityRemove(origin, name) } cmd := &store.Command{ diff --git a/cluster/iam/adapter/identity.go b/cluster/iam/adapter/identity.go index 27bd2114..63dc114d 100644 --- a/cluster/iam/adapter/identity.go +++ b/cluster/iam/adapter/identity.go @@ -18,7 +18,7 @@ func NewIdentityAdapter(store store.Store) (iamidentity.Adapter, error) { } func (a *identityAdapter) LoadIdentities() ([]iamidentity.User, error) { - users := a.store.ListUsers() + users := a.store.IAMIdentityList() return users.Users, nil } diff --git a/cluster/iam/adapter/policy.go b/cluster/iam/adapter/policy.go index 2fb46d40..b6250ece 100644 --- a/cluster/iam/adapter/policy.go +++ b/cluster/iam/adapter/policy.go @@ -25,7 +25,7 @@ func NewPolicyAdapter(store store.Store) (iamaccess.Adapter, error) { } func (a *policyAdapter) LoadPolicy(model model.Model) error { - policies := a.store.ListPolicies() + policies := a.store.IAMPolicyList() rules := [][]string{} domains := map[string]struct{}{} diff --git a/cluster/kvs.go b/cluster/kvs.go index dc03e0c7..18c2b02d 100644 --- a/cluster/kvs.go +++ b/cluster/kvs.go @@ -10,13 +10,9 @@ import ( "github.com/datarhei/core/v16/log" ) -func (c *cluster) CreateLock(origin string, name string, validUntil time.Time) (*kvs.Lock, error) { - if ok, _ := c.IsClusterDegraded(); ok { - return nil, ErrDegraded - } - +func (c *cluster) LockCreate(origin string, name string, validUntil time.Time) (*kvs.Lock, error) { if !c.IsRaftLeader() { - err := c.forwarder.CreateLock(origin, name, validUntil) + err := c.forwarder.LockCreate(origin, name, validUntil) if err != nil { return nil, err } @@ -28,7 +24,7 @@ func (c *cluster) CreateLock(origin string, name string, validUntil time.Time) ( return l, nil } - if c.store.HasLock(name) { + if c.store.LockHasLock(name) { return nil, fmt.Errorf("the lock '%s' already exists", name) } @@ -52,13 +48,9 @@ func (c *cluster) CreateLock(origin string, name string, validUntil time.Time) ( return l, nil } -func (c *cluster) DeleteLock(origin string, name string) error { - if ok, _ := c.IsClusterDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) LockDelete(origin string, name string) error { if !c.IsRaftLeader() { - return c.forwarder.DeleteLock(origin, name) + return c.forwarder.LockDelete(origin, name) } cmd := &store.Command{ @@ -71,17 +63,9 @@ func (c *cluster) DeleteLock(origin string, name string) error { return c.applyCommand(cmd) } -func (c *cluster) ListLocks() map[string]time.Time { - return c.store.ListLocks() -} - -func (c *cluster) SetKV(origin, key, value string) error { - if ok, _ := c.IsClusterDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) KVSet(origin, key, value string) error { if !c.IsRaftLeader() { - return c.forwarder.SetKV(origin, key, value) + return c.forwarder.KVSet(origin, key, value) } cmd := &store.Command{ @@ -95,13 +79,9 @@ func (c *cluster) SetKV(origin, key, value string) error { return c.applyCommand(cmd) } -func (c *cluster) UnsetKV(origin, key string) error { - if ok, _ := c.IsClusterDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) KVUnset(origin, key string) error { if !c.IsRaftLeader() { - return c.forwarder.UnsetKV(origin, key) + return c.forwarder.KVUnset(origin, key) } cmd := &store.Command{ @@ -114,18 +94,14 @@ func (c *cluster) UnsetKV(origin, key string) error { return c.applyCommand(cmd) } -func (c *cluster) GetKV(origin, key string, stale bool) (string, time.Time, error) { +func (c *cluster) KVGet(origin, key string, stale bool) (string, time.Time, error) { if !stale { - if ok, _ := c.IsClusterDegraded(); ok { - return "", time.Time{}, ErrDegraded - } - if !c.IsRaftLeader() { - return c.forwarder.GetKV(origin, key) + return c.forwarder.KVGet(origin, key) } } - value, err := c.store.GetFromKVS(key) + value, err := c.store.KVSGetValue(key) if err != nil { return "", time.Time{}, err } @@ -133,12 +109,6 @@ func (c *cluster) GetKV(origin, key string, stale bool) (string, time.Time, erro return value.Value, value.UpdatedAt, nil } -func (c *cluster) ListKV(prefix string) map[string]store.Value { - storeValues := c.store.ListKVS(prefix) - - return storeValues -} - type ClusterKVS interface { kvs.KVS @@ -178,17 +148,17 @@ func (s *clusterKVS) CreateLock(name string, validUntil time.Time) (*kvs.Lock, e "name": name, "valid_until": validUntil, }).Log("Create lock") - return s.cluster.CreateLock("", name, validUntil) + return s.cluster.LockCreate("", name, validUntil) } func (s *clusterKVS) DeleteLock(name string) error { s.logger.Debug().WithField("name", name).Log("Delete lock") - return s.cluster.DeleteLock("", name) + return s.cluster.LockDelete("", name) } func (s *clusterKVS) ListLocks() map[string]time.Time { s.logger.Debug().Log("List locks") - return s.cluster.ListLocks() + return s.cluster.Store().LockList() } func (s *clusterKVS) SetKV(key, value string) error { @@ -196,12 +166,12 @@ func (s *clusterKVS) SetKV(key, value string) error { "key": key, "value": value, }).Log("Set KV") - return s.cluster.SetKV("", key, value) + return s.cluster.KVSet("", key, value) } func (s *clusterKVS) UnsetKV(key string) error { s.logger.Debug().WithField("key", key).Log("Unset KV") - return s.cluster.UnsetKV("", key) + return s.cluster.KVUnset("", key) } func (s *clusterKVS) GetKV(key string) (string, time.Time, error) { @@ -213,10 +183,10 @@ func (s *clusterKVS) GetKV(key string) (string, time.Time, error) { "key": key, "stale": stale, }).Log("Get KV") - return s.cluster.GetKV("", key, stale) + return s.cluster.KVGet("", key, stale) } func (s *clusterKVS) ListKV(prefix string) map[string]store.Value { s.logger.Debug().Log("List KV") - return s.cluster.ListKV(prefix) + return s.cluster.Store().KVSList(prefix) } diff --git a/cluster/leader.go b/cluster/leader.go index 95cbd3cc..2f3c8e96 100644 --- a/cluster/leader.go +++ b/cluster/leader.go @@ -1,7 +1,6 @@ package cluster import ( - "bytes" "context" "errors" "fmt" @@ -9,11 +8,9 @@ import ( "sync" "time" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/cluster/store" - "github.com/datarhei/core/v16/encoding/json" "github.com/datarhei/core/v16/log" - "github.com/datarhei/core/v16/maps" "github.com/datarhei/core/v16/restream/app" ) @@ -350,10 +347,6 @@ func (c *cluster) synchronizeAndRebalance(ctx context.Context, interval time.Dur return case <-ticker.C: if !emergency { - if ok, _ := c.IsDegraded(); ok { - break - } - c.doSynchronize(emergency, term) c.doRebalance(emergency, term) c.doRelocate(emergency, term) @@ -375,7 +368,7 @@ func (c *cluster) clearLocks(ctx context.Context, interval time.Duration) { case <-ctx.Done(): return case <-ticker.C: - locks := c.ListLocks() + locks := c.store.LockList() hasExpiredLocks := false for _, validUntil := range locks { @@ -503,7 +496,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { switch v := op.(type) { case processOpAdd: - err := c.proxy.AddProcess(v.nodeid, v.config, v.metadata) + err := c.manager.ProcessAdd(v.nodeid, v.config, v.metadata) if err != nil { opErr = processOpError{ processid: v.config.ProcessID(), @@ -517,7 +510,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { } if v.order == "start" { - err = c.proxy.CommandProcess(v.nodeid, v.config.ProcessID(), "start") + err = c.manager.ProcessCommand(v.nodeid, v.config.ProcessID(), "start") if err != nil { opErr = processOpError{ processid: v.config.ProcessID(), @@ -541,7 +534,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { "nodeid": v.nodeid, }).Log("Adding process") case processOpUpdate: - err := c.proxy.UpdateProcess(v.nodeid, v.processid, v.config, v.metadata) + err := c.manager.ProcessUpdate(v.nodeid, v.processid, v.config, v.metadata) if err != nil { opErr = processOpError{ processid: v.processid, @@ -564,7 +557,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { "nodeid": v.nodeid, }).Log("Updating process") case processOpDelete: - err := c.proxy.DeleteProcess(v.nodeid, v.processid) + err := c.manager.ProcessDelete(v.nodeid, v.processid) if err != nil { opErr = processOpError{ processid: v.processid, @@ -587,7 +580,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { "nodeid": v.nodeid, }).Log("Removing process") case processOpMove: - err := c.proxy.AddProcess(v.toNodeid, v.config, v.metadata) + err := c.manager.ProcessAdd(v.toNodeid, v.config, v.metadata) if err != nil { opErr = processOpError{ processid: v.config.ProcessID(), @@ -601,7 +594,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { break } - err = c.proxy.DeleteProcess(v.fromNodeid, v.config.ProcessID()) + err = c.manager.ProcessDelete(v.fromNodeid, v.config.ProcessID()) if err != nil { opErr = processOpError{ processid: v.config.ProcessID(), @@ -616,7 +609,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { } if v.order == "start" { - err = c.proxy.CommandProcess(v.toNodeid, v.config.ProcessID(), "start") + err = c.manager.ProcessCommand(v.toNodeid, v.config.ProcessID(), "start") if err != nil { opErr = processOpError{ processid: v.config.ProcessID(), @@ -642,7 +635,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { "tonodeid": v.toNodeid, }).Log("Moving process") case processOpStart: - err := c.proxy.CommandProcess(v.nodeid, v.processid, "start") + err := c.manager.ProcessCommand(v.nodeid, v.processid, "start") if err != nil { opErr = processOpError{ processid: v.processid, @@ -665,7 +658,7 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { "nodeid": v.nodeid, }).Log("Starting process") case processOpStop: - err := c.proxy.CommandProcess(v.nodeid, v.processid, "stop") + err := c.manager.ProcessCommand(v.nodeid, v.processid, "stop") if err != nil { opErr = processOpError{ processid: v.processid, @@ -708,1043 +701,12 @@ func (c *cluster) applyOp(op interface{}, logger log.Logger) processOpError { return opErr } -func (c *cluster) doSynchronize(emergency bool, term uint64) { - wish := c.store.GetProcessNodeMap() - want := c.store.ListProcesses() - storeNodes := c.store.ListNodes() - have := c.proxy.ListProxyProcesses() - nodes := c.proxy.ListNodes() - - logger := c.logger.WithField("term", term) - - logger.Debug().WithField("emergency", emergency).Log("Synchronizing") - - nodesMap := map[string]proxy.NodeAbout{} - - for _, node := range nodes { - about := node.About() - - if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { - about.State = storeNode.State - } - - nodesMap[about.ID] = about - } - - logger.Debug().WithFields(log.Fields{ - "want": want, - "have": have, - "nodes": nodesMap, - }).Log("Synchronize") - - opStack, _, reality := synchronize(wish, want, have, nodesMap, c.nodeRecoverTimeout) - - if !emergency && !maps.Equal(wish, reality) { - cmd := &store.Command{ - Operation: store.OpSetProcessNodeMap, - Data: store.CommandSetProcessNodeMap{ - Map: reality, - }, - } - - c.applyCommand(cmd) - } - - errors := c.applyOpStack(opStack, term) - - if !emergency { - for _, e := range errors { - // Only apply the command if the error is different. - process, err := c.store.GetProcess(e.processid) - if err != nil { - continue - } - - var errmessage string = "" - - if e.err != nil { - if process.Error == e.err.Error() { - continue - } - - errmessage = e.err.Error() - } else { - if len(process.Error) == 0 { - continue - } - } - - cmd := &store.Command{ - Operation: store.OpSetProcessError, - Data: store.CommandSetProcessError{ - ID: e.processid, - Error: errmessage, - }, - } - - c.applyCommand(cmd) - } - } -} - -func (c *cluster) doRebalance(emergency bool, term uint64) { - if emergency { - // Don't rebalance in emergency mode. - return - } - - logger := c.logger.WithField("term", term) - - logger.Debug().WithField("emergency", emergency).Log("Rebalancing") - - storeNodes := c.store.ListNodes() - have := c.proxy.ListProxyProcesses() - nodes := c.proxy.ListNodes() - - nodesMap := map[string]proxy.NodeAbout{} - - for _, node := range nodes { - about := node.About() - - if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { - about.State = storeNode.State - } - - nodesMap[about.ID] = about - } - - logger.Debug().WithFields(log.Fields{ - "have": have, - "nodes": nodesMap, - }).Log("Rebalance") - - opStack, _ := rebalance(have, nodesMap) - - errors := c.applyOpStack(opStack, term) - - for _, e := range errors { - // Only apply the command if the error is different. - process, err := c.store.GetProcess(e.processid) - if err != nil { - continue - } - - var errmessage string = "" - - if e.err != nil { - if process.Error == e.err.Error() { - continue - } - - errmessage = e.err.Error() - } else { - if len(process.Error) == 0 { - continue - } - } - - cmd := &store.Command{ - Operation: store.OpSetProcessError, - Data: store.CommandSetProcessError{ - ID: e.processid, - Error: errmessage, - }, - } - - c.applyCommand(cmd) - } -} - -func (c *cluster) doRelocate(emergency bool, term uint64) { - if emergency { - // Don't relocate in emergency mode. - return - } - - logger := c.logger.WithField("term", term) - - logger.Debug().WithField("emergency", emergency).Log("Relocating") - - relocateMap := c.store.GetProcessRelocateMap() - storeNodes := c.store.ListNodes() - have := c.proxy.ListProxyProcesses() - nodes := c.proxy.ListNodes() - - nodesMap := map[string]proxy.NodeAbout{} - - for _, node := range nodes { - about := node.About() - - if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { - about.State = storeNode.State - } - - nodesMap[about.ID] = about - } - - logger.Debug().WithFields(log.Fields{ - "relocate": relocate, - "have": have, - "nodes": nodesMap, - }).Log("Rebalance") - - opStack, _, relocatedProcessIDs := relocate(have, nodesMap, relocateMap) - - errors := c.applyOpStack(opStack, term) - - for _, e := range errors { - // Only apply the command if the error is different. - process, err := c.store.GetProcess(e.processid) - if err != nil { - continue - } - - var errmessage string = "" - - if e.err != nil { - if process.Error == e.err.Error() { - continue - } - - errmessage = e.err.Error() - } else { - if len(process.Error) == 0 { - continue - } - } - - cmd := &store.Command{ - Operation: store.OpSetProcessError, - Data: store.CommandSetProcessError{ - ID: e.processid, - Error: errmessage, - }, - } - - c.applyCommand(cmd) - } - - cmd := store.CommandUnsetRelocateProcess{ - ID: []app.ProcessID{}, - } - - for _, processid := range relocatedProcessIDs { - cmd.ID = append(cmd.ID, app.ParseProcessID(processid)) - } - - if len(cmd.ID) != 0 { - c.applyCommand(&store.Command{ - Operation: store.OpUnsetRelocateProcess, - Data: cmd, - }) - } -} - -// isMetadataUpdateRequired compares two metadata. It relies on the documented property that json.Marshal -// sorts the map keys prior encoding. -func isMetadataUpdateRequired(wantMap map[string]interface{}, haveMap map[string]interface{}) (bool, map[string]interface{}) { - hasChanges := false - changeMap := map[string]interface{}{} - - haveMapKeys := map[string]struct{}{} - - for key := range haveMap { - haveMapKeys[key] = struct{}{} - } - - for key, wantMapValue := range wantMap { - haveMapValue, ok := haveMap[key] - if !ok { - // A key in map1 exists, that doesn't exist in map2, we need to update. - hasChanges = true - } - - // Compare the values - changesData, err := json.Marshal(wantMapValue) - if err != nil { - continue - } - - completeData, err := json.Marshal(haveMapValue) - if err != nil { - continue - } - - if !bytes.Equal(changesData, completeData) { - // The values are not equal, we need to update. - hasChanges = true - } - - delete(haveMapKeys, key) - - changeMap[key] = wantMapValue - } - - for key := range haveMapKeys { - // If there keys in map2 that are not in map1, we have to update. - hasChanges = true - changeMap[key] = nil - } - - return hasChanges, changeMap -} - -// synchronize returns a list of operations in order to adjust the "have" list to the "want" list -// with taking the available resources on each node into account. -func synchronize(wish map[string]string, want []store.Process, have []proxy.Process, nodes map[string]proxy.NodeAbout, nodeRecoverTimeout time.Duration) ([]interface{}, map[string]proxy.NodeResources, map[string]string) { - resources := NewResources(nodes) - - // Mark nodes as throttling where at least one process is still throttling - for _, haveP := range have { - if haveP.Throttling { - resources.Throttling(haveP.NodeID, true) - } - } - - // A map same as wish, but reflecting the actual situation. - reality := map[string]string{} - - // A map from the process ID to the process config of the processes - // we want to be running on the nodes. - wantMap := map[string]store.Process{} - for _, wantP := range want { - pid := wantP.Config.ProcessID().String() - wantMap[pid] = wantP - } - - opStack := []interface{}{} - - // Now we iterate through the processes we actually have running on the nodes - // and remove them from the wantMap. We also make sure that they have the correct order. - // If a process cannot be found on the wantMap, it will be deleted from the nodes. - haveAfterRemove := []proxy.Process{} - wantOrderStart := []proxy.Process{} - - for _, haveP := range have { - pid := haveP.Config.ProcessID().String() - wantP, ok := wantMap[pid] - if !ok { - // The process is not on the wantMap. Delete it and adjust the resources. - opStack = append(opStack, processOpDelete{ - nodeid: haveP.NodeID, - processid: haveP.Config.ProcessID(), - }) - - resources.Remove(haveP.NodeID, haveP.CPU, haveP.Mem) - - continue - } - - // The process is on the wantMap. Update the process if the configuration and/or metadata differ. - hasConfigChanges := !wantP.Config.Equal(haveP.Config) - hasMetadataChanges, metadata := isMetadataUpdateRequired(wantP.Metadata, haveP.Metadata) - if hasConfigChanges || hasMetadataChanges { - // TODO: When the required resources increase, should we move this process to a node - // that has them available? Otherwise, this node might start throttling. However, this - // will result in rebalancing. - opStack = append(opStack, processOpUpdate{ - nodeid: haveP.NodeID, - processid: haveP.Config.ProcessID(), - config: wantP.Config, - metadata: metadata, - }) - } - - delete(wantMap, pid) - reality[pid] = haveP.NodeID - - if haveP.Order != wantP.Order { - if wantP.Order == "start" { - // Delay pushing them to the stack in order to have - // all resources released first. - wantOrderStart = append(wantOrderStart, haveP) - } else { - opStack = append(opStack, processOpStop{ - nodeid: haveP.NodeID, - processid: haveP.Config.ProcessID(), - }) - - // Release the resources. - resources.Remove(haveP.NodeID, haveP.CPU, haveP.Mem) - } - } - - haveAfterRemove = append(haveAfterRemove, haveP) - } - - for _, haveP := range wantOrderStart { - nodeid := haveP.NodeID - - resources.Add(nodeid, haveP.Config.LimitCPU, haveP.Config.LimitMemory) - - // TODO: check if the current node has actually enough resources available, - // otherwise it needs to be moved somewhere else. If the node doesn't - // have enough resources available, the process will be prevented - // from starting. - - /* - if hasNodeEnoughResources(r, haveP.Config.LimitCPU, haveP.Config.LimitMemory) { - // Consume the resources - r.CPU += haveP.Config.LimitCPU - r.Mem += haveP.Config.LimitMemory - resources[nodeid] = r - } else { - nodeid = findBestNodeForProcess(resources, haveP.Config.LimitCPU, haveP.Config.LimitMemory) - if len(nodeid) == 0 { - // Start it anyways and let it run into an error - opStack = append(opStack, processOpStart{ - nodeid: nodeid, - processid: haveP.Config.ProcessID(), - }) - - continue - } - - if nodeid != haveP.NodeID { - opStack = append(opStack, processOpMove{ - fromNodeid: haveP.NodeID, - toNodeid: nodeid, - config: haveP.Config, - metadata: haveP.Metadata, - order: haveP.Order, - }) - } - - // Consume the resources - r, ok := resources[nodeid] - if ok { - r.CPU += haveP.Config.LimitCPU - r.Mem += haveP.Config.LimitMemory - resources[nodeid] = r - } - } - */ - - opStack = append(opStack, processOpStart{ - nodeid: nodeid, - processid: haveP.Config.ProcessID(), - }) - } - - have = haveAfterRemove - - // In case a node didn't respond, some PID are still on the wantMap, that would run on - // the currently not responding nodes. We use the wish map to assign them to the node. - // If the node is unavailable for too long, keep these processes on the wantMap, otherwise - // remove them and hope that they will reappear during the nodeRecoverTimeout. - for pid := range wantMap { - // Check if this PID is be assigned to a node. - if nodeid, ok := wish[pid]; ok { - // Check for how long the node hasn't been contacted, or if it still exists. - if node, ok := nodes[nodeid]; ok { - if node.State != "disconnected" { - continue - } - - if time.Since(node.LastContact) <= nodeRecoverTimeout { - reality[pid] = nodeid - delete(wantMap, pid) - } - } - } - } - - // The wantMap now contains only those processes that need to be installed on a node. - // We will rebuild the "want" array from the wantMap in the same order as the original - // "want" array to make the resulting opStack deterministic. - wantReduced := []store.Process{} - for _, wantP := range want { - pid := wantP.Config.ProcessID().String() - if _, ok := wantMap[pid]; !ok { - continue - } - - wantReduced = append(wantReduced, wantP) - } - - // Create a map from the process reference to the node it is running on. - haveReferenceAffinity := NewReferenceAffinity(have) - - // Now, all remaining processes in the wantMap must be added to one of the nodes. - for _, wantP := range wantReduced { - pid := wantP.Config.ProcessID().String() - - // If a process doesn't have any limits defined, reject that process. - if wantP.Config.LimitCPU <= 0 || wantP.Config.LimitMemory <= 0 { - opStack = append(opStack, processOpReject{ - processid: wantP.Config.ProcessID(), - err: errNoLimitsDefined, - }) - - continue - } - - // Check if there are already processes with the same reference, and if so - // choose this node. Then check the node if it has enough resources left. If - // not, then select a node with the most available resources. - nodeid := "" - - // Try to add the process to a node where other processes with the same reference currently reside. - raNodes := haveReferenceAffinity.Nodes(wantP.Config.Reference, wantP.Config.Domain) - for _, raNodeid := range raNodes { - if resources.HasNodeEnough(raNodeid, wantP.Config.LimitCPU, wantP.Config.LimitMemory) { - nodeid = raNodeid - break - } - } - - // Find the node with the most resources available. - if len(nodeid) == 0 { - nodes := resources.FindBestNodes(wantP.Config.LimitCPU, wantP.Config.LimitMemory) - if len(nodes) > 0 { - nodeid = nodes[0] - } - } - - if len(nodeid) != 0 { - opStack = append(opStack, processOpAdd{ - nodeid: nodeid, - config: wantP.Config, - metadata: wantP.Metadata, - order: wantP.Order, - }) - - // Consume the resources - resources.Add(nodeid, wantP.Config.LimitCPU, wantP.Config.LimitMemory) - - reality[pid] = nodeid - - haveReferenceAffinity.Add(wantP.Config.Reference, wantP.Config.Domain, nodeid) - } else { - opStack = append(opStack, processOpReject{ - processid: wantP.Config.ProcessID(), - err: errNotEnoughResourcesForDeployment, - }) - } - } - - return opStack, resources.Map(), reality -} - -type resources struct { - nodes map[string]proxy.NodeResources - blocked map[string]struct{} -} - -func NewResources(nodes map[string]proxy.NodeAbout) *resources { - r := &resources{ - nodes: map[string]proxy.NodeResources{}, - blocked: map[string]struct{}{}, - } - - for nodeid, about := range nodes { - r.nodes[nodeid] = about.Resources - if about.State != "connected" { - r.blocked[nodeid] = struct{}{} - } - } - - return r -} - -func (r *resources) Throttling(nodeid string, throttling bool) { - res, hasNode := r.nodes[nodeid] - if !hasNode { - return - } - - res.IsThrottling = throttling - - r.nodes[nodeid] = res -} - -// HasNodeEnough returns whether a node has enough resources available for the -// requested cpu and memory consumption. -func (r *resources) HasNodeEnough(nodeid string, cpu float64, mem uint64) bool { - res, hasNode := r.nodes[nodeid] - if !hasNode { - return false - } - - if _, hasNode := r.blocked[nodeid]; hasNode { - return false - } - - if res.CPU+cpu < res.CPULimit && res.Mem+mem < res.MemLimit && !res.IsThrottling { - return true - } - - return false -} - -// FindBestNodes returns an array of nodeids that can fit the requested cpu and memory requirements. If no -// such node is available, an empty array is returned. The array is sorted by the most suitable node first. -func (r *resources) FindBestNodes(cpu float64, mem uint64) []string { - nodes := []string{} - - for id := range r.nodes { - if r.HasNodeEnough(id, cpu, mem) { - nodes = append(nodes, id) - } - } - - sort.SliceStable(nodes, func(i, j int) bool { - nodeA, nodeB := nodes[i], nodes[j] - - if r.nodes[nodeA].CPU != r.nodes[nodeB].CPU { - return r.nodes[nodeA].CPU < r.nodes[nodeB].CPU - } - - return r.nodes[nodeA].Mem <= r.nodes[nodeB].Mem - }) - - return nodes -} - -// Add adds the resources of the node according to the cpu and memory utilization. -func (r *resources) Add(nodeid string, cpu float64, mem uint64) { - res, hasRes := r.nodes[nodeid] - if !hasRes { - return - } - - res.CPU += cpu - res.Mem += mem - r.nodes[nodeid] = res -} - -// Remove subtracts the resources from the node according to the cpu and memory utilization. -func (r *resources) Remove(nodeid string, cpu float64, mem uint64) { - res, hasRes := r.nodes[nodeid] - if !hasRes { - return - } - - res.CPU -= cpu - if res.CPU < 0 { - res.CPU = 0 - } - if mem >= res.Mem { - res.Mem = 0 - } else { - res.Mem -= mem - } - r.nodes[nodeid] = res -} - -// Move adjusts the resources from the target and source node according to the cpu and memory utilization. -func (r *resources) Move(target, source string, cpu float64, mem uint64) { - r.Add(target, cpu, mem) - r.Remove(source, cpu, mem) -} - -func (r *resources) Map() map[string]proxy.NodeResources { - return r.nodes -} - -type referenceAffinityNodeCount struct { - nodeid string - count uint64 -} - -type referenceAffinity struct { - m map[string][]referenceAffinityNodeCount -} - -// NewReferenceAffinity returns a referenceAffinity. This is a map of references (per domain) to an array of -// nodes this reference is found on and their count. -func NewReferenceAffinity(processes []proxy.Process) *referenceAffinity { - ra := &referenceAffinity{ - m: map[string][]referenceAffinityNodeCount{}, - } - - for _, p := range processes { - if len(p.Config.Reference) == 0 { - continue - } - - key := p.Config.Reference + "@" + p.Config.Domain - - // Here we count how often a reference is present on a node. When - // moving processes to a different node, the node with the highest - // count of same references will be the first candidate. - found := false - arr := ra.m[key] - for i, count := range arr { - if count.nodeid == p.NodeID { - count.count++ - arr[i] = count - found = true - break - } - } - - if !found { - arr = append(arr, referenceAffinityNodeCount{ - nodeid: p.NodeID, - count: 1, - }) - } - - ra.m[key] = arr - } - - // Sort every reference count in decreasing order for each reference. - for ref, count := range ra.m { - sort.SliceStable(count, func(a, b int) bool { - return count[a].count > count[b].count - }) - - ra.m[ref] = count - } - - return ra -} - -// Nodes returns a list of node IDs for the provided reference and domain. The list -// is ordered by how many references are on the nodes in descending order. -func (ra *referenceAffinity) Nodes(reference, domain string) []string { - if len(reference) == 0 { - return nil - } - - key := reference + "@" + domain - - counts, ok := ra.m[key] - if !ok { - return nil - } - - nodes := []string{} - - for _, count := range counts { - nodes = append(nodes, count.nodeid) - } - - return nodes -} - -// Add adds a reference on a node to an existing reference affinity. -func (ra *referenceAffinity) Add(reference, domain, nodeid string) { - if len(reference) == 0 { - return - } - - key := reference + "@" + domain - - counts, ok := ra.m[key] - if !ok { - ra.m[key] = []referenceAffinityNodeCount{ - { - nodeid: nodeid, - count: 1, - }, - } - - return - } - - found := false - for i, count := range counts { - if count.nodeid == nodeid { - count.count++ - counts[i] = count - found = true - break - } - } - - if !found { - counts = append(counts, referenceAffinityNodeCount{ - nodeid: nodeid, - count: 1, - }) - } - - ra.m[key] = counts -} - -// Move moves a reference from one node to another node in an existing reference affinity. -func (ra *referenceAffinity) Move(reference, domain, fromnodeid, tonodeid string) { - if len(reference) == 0 { - return - } - - key := reference + "@" + domain - - counts, ok := ra.m[key] - if !ok { - ra.m[key] = []referenceAffinityNodeCount{ - { - nodeid: tonodeid, - count: 1, - }, - } - - return - } - - found := false - for i, count := range counts { - if count.nodeid == tonodeid { - count.count++ - counts[i] = count - found = true - } else if count.nodeid == fromnodeid { - count.count-- - counts[i] = count - } - } - - if !found { - counts = append(counts, referenceAffinityNodeCount{ - nodeid: tonodeid, - count: 1, - }) - } - - newCounts := []referenceAffinityNodeCount{} - - for _, count := range counts { - if count.count == 0 { - continue - } - - newCounts = append(newCounts, count) - } - - ra.m[key] = newCounts -} - -// rebalance returns a list of operations that will move running processes away from nodes that are overloaded. -func rebalance(have []proxy.Process, nodes map[string]proxy.NodeAbout) ([]interface{}, map[string]proxy.NodeResources) { - resources := NewResources(nodes) - - // Mark nodes as throttling where at least one process is still throttling - for _, haveP := range have { - if haveP.Throttling { - resources.Throttling(haveP.NodeID, true) - } - } - - // Group all running processes by node and sort them by their runtime in ascending order. - nodeProcessMap := createNodeProcessMap(have) - - // A map from the process reference to the nodes it is running on. - haveReferenceAffinity := NewReferenceAffinity(have) - - opStack := []interface{}{} - - // Check if any of the nodes is overloaded. - for id, r := range resources.Map() { - // Ignore this node if the resource values are not reliable. - if r.Error != nil { - continue - } - - // Check if node is overloaded. - if r.CPU < r.CPULimit && r.Mem < r.MemLimit && !r.IsThrottling { - continue - } - - // Move processes from this node to another node with enough free resources. - // The processes are ordered ascending by their runtime. - processes := nodeProcessMap[id] - if len(processes) == 0 { - // If there are no processes on that node, we can't do anything. - continue - } - - overloadedNodeid := id - - for i, p := range processes { - availableNodeid := "" - - // Try to move the process to a node where other processes with the same - // reference currently reside. - if len(p.Config.Reference) != 0 { - raNodes := haveReferenceAffinity.Nodes(p.Config.Reference, p.Config.Domain) - for _, raNodeid := range raNodes { - // Do not move the process to the node it is currently on. - if raNodeid == overloadedNodeid { - continue - } - - if resources.HasNodeEnough(raNodeid, p.Config.LimitCPU, p.Config.LimitMemory) { - availableNodeid = raNodeid - break - } - } - } - - // Find the best node with enough resources available. - if len(availableNodeid) == 0 { - nodes := resources.FindBestNodes(p.Config.LimitCPU, p.Config.LimitMemory) - for _, nodeid := range nodes { - if nodeid == overloadedNodeid { - continue - } - - availableNodeid = nodeid - break - } - } - - if len(availableNodeid) == 0 { - // There's no other node with enough resources to take over this process. - opStack = append(opStack, processOpSkip{ - nodeid: overloadedNodeid, - processid: p.Config.ProcessID(), - err: errNotEnoughResourcesForRebalancing, - }) - continue - } - - opStack = append(opStack, processOpMove{ - fromNodeid: overloadedNodeid, - toNodeid: availableNodeid, - config: p.Config, - metadata: p.Metadata, - order: p.Order, - }) - - // Adjust the process. - p.NodeID = availableNodeid - processes[i] = p - - // Adjust the resources. - resources.Move(availableNodeid, overloadedNodeid, p.CPU, p.Mem) - - // Adjust the reference affinity. - haveReferenceAffinity.Move(p.Config.Reference, p.Config.Domain, overloadedNodeid, availableNodeid) - - // Move only one process at a time. - break - } - } - - return opStack, resources.Map() -} - -// relocate returns a list of operations that will move deployed processes to different nodes. -func relocate(have []proxy.Process, nodes map[string]proxy.NodeAbout, relocateMap map[string]string) ([]interface{}, map[string]proxy.NodeResources, []string) { - resources := NewResources(nodes) - - // Mark nodes as throttling where at least one process is still throttling - for _, haveP := range have { - if haveP.Throttling { - resources.Throttling(haveP.NodeID, true) - } - } - - relocatedProcessIDs := []string{} - - // A map from the process reference to the nodes it is running on. - haveReferenceAffinity := NewReferenceAffinity(have) - - opStack := []interface{}{} - - // Check for any requested relocations. - for processid, targetNodeid := range relocateMap { - process := proxy.Process{} - - found := false - for _, p := range have { - if processid == p.Config.ProcessID().String() { - process = p - found = true - break - } - } - - if !found { - relocatedProcessIDs = append(relocatedProcessIDs, processid) - continue - } - - sourceNodeid := process.NodeID - - if sourceNodeid == targetNodeid { - relocatedProcessIDs = append(relocatedProcessIDs, processid) - continue - } - - if len(targetNodeid) != 0 { - _, hasNode := nodes[targetNodeid] - - if !hasNode || !resources.HasNodeEnough(targetNodeid, process.Config.LimitCPU, process.Config.LimitMemory) { - targetNodeid = "" - } - } - - if len(targetNodeid) == 0 { - // Try to move the process to a node where other processes with the same - // reference currently reside. - if len(process.Config.Reference) != 0 { - raNodes := haveReferenceAffinity.Nodes(process.Config.Reference, process.Config.Domain) - for _, raNodeid := range raNodes { - // Do not move the process to the node it is currently on. - if raNodeid == sourceNodeid { - continue - } - - if resources.HasNodeEnough(raNodeid, process.Config.LimitCPU, process.Config.LimitMemory) { - targetNodeid = raNodeid - break - } - } - } - - // Find the best node with enough resources available. - if len(targetNodeid) == 0 { - nodes := resources.FindBestNodes(process.Config.LimitCPU, process.Config.LimitMemory) - for _, nodeid := range nodes { - if nodeid == sourceNodeid { - continue - } - - targetNodeid = nodeid - break - } - } - - if len(targetNodeid) == 0 { - // There's no other node with enough resources to take over this process. - opStack = append(opStack, processOpSkip{ - nodeid: sourceNodeid, - processid: process.Config.ProcessID(), - err: errNotEnoughResourcesForRelocating, - }) - continue - } - } - - opStack = append(opStack, processOpMove{ - fromNodeid: sourceNodeid, - toNodeid: targetNodeid, - config: process.Config, - metadata: process.Metadata, - order: process.Order, - }) - - // Adjust the resources. - resources.Move(targetNodeid, sourceNodeid, process.CPU, process.Mem) - - // Adjust the reference affinity. - haveReferenceAffinity.Move(process.Config.Reference, process.Config.Domain, sourceNodeid, targetNodeid) - - relocatedProcessIDs = append(relocatedProcessIDs, processid) - } - - return opStack, resources.Map(), relocatedProcessIDs -} - // createNodeProcessMap takes a list of processes and groups them by the nodeid they // are running on. Each group contains only running processes and gets sorted by their // preference to be moved somewhere else, increasing. From the running processes, the // ones with the shortest runtime have the highest preference. -func createNodeProcessMap(processes []proxy.Process) map[string][]proxy.Process { - nodeProcessMap := map[string][]proxy.Process{} +func createNodeProcessMap(processes []node.Process) map[string][]node.Process { + nodeProcessMap := map[string][]node.Process{} for _, p := range processes { if p.State != "running" { diff --git a/cluster/leader_rebalance.go b/cluster/leader_rebalance.go new file mode 100644 index 00000000..ebfc3120 --- /dev/null +++ b/cluster/leader_rebalance.go @@ -0,0 +1,185 @@ +package cluster + +import ( + "github.com/datarhei/core/v16/cluster/node" + "github.com/datarhei/core/v16/cluster/store" + "github.com/datarhei/core/v16/log" +) + +func (c *cluster) doRebalance(emergency bool, term uint64) { + if emergency { + // Don't rebalance in emergency mode. + return + } + + logger := c.logger.WithField("term", term) + + logger.Debug().WithField("emergency", emergency).Log("Rebalancing") + + storeNodes := c.store.NodeList() + have := c.manager.ClusterProcessList() + nodes := c.manager.NodeList() + + nodesMap := map[string]node.About{} + + for _, node := range nodes { + about := node.About() + + if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { + about.State = storeNode.State + } + + nodesMap[about.ID] = about + } + + logger.Debug().WithFields(log.Fields{ + "have": have, + "nodes": nodesMap, + }).Log("Rebalance") + + opStack, _ := rebalance(have, nodesMap) + + errors := c.applyOpStack(opStack, term) + + for _, e := range errors { + // Only apply the command if the error is different. + process, err := c.store.ProcessGet(e.processid) + if err != nil { + continue + } + + var errmessage string = "" + + if e.err != nil { + if process.Error == e.err.Error() { + continue + } + + errmessage = e.err.Error() + } else { + if len(process.Error) == 0 { + continue + } + } + + cmd := &store.Command{ + Operation: store.OpSetProcessError, + Data: store.CommandSetProcessError{ + ID: e.processid, + Error: errmessage, + }, + } + + c.applyCommand(cmd) + } +} + +// rebalance returns a list of operations that will move running processes away from nodes that are overloaded. +func rebalance(have []node.Process, nodes map[string]node.About) ([]interface{}, map[string]node.Resources) { + resources := NewResourcePlanner(nodes) + + // Mark nodes as throttling where at least one process is still throttling + for _, haveP := range have { + if haveP.Throttling { + resources.Throttling(haveP.NodeID, true) + } + } + + // Group all running processes by node and sort them by their runtime in ascending order. + nodeProcessMap := createNodeProcessMap(have) + + // A map from the process reference to the nodes it is running on. + haveReferenceAffinity := NewReferenceAffinity(have) + + opStack := []interface{}{} + + // Check if any of the nodes is overloaded. + for id, r := range resources.Map() { + // Ignore this node if the resource values are not reliable. + if r.Error != nil { + continue + } + + // Check if node is overloaded. + if r.CPU < r.CPULimit && r.Mem < r.MemLimit && !r.IsThrottling { + continue + } + + // Move processes from this node to another node with enough free resources. + // The processes are ordered ascending by their runtime. + processes := nodeProcessMap[id] + if len(processes) == 0 { + // If there are no processes on that node, we can't do anything. + continue + } + + overloadedNodeid := id + + for i, p := range processes { + availableNodeid := "" + + // Try to move the process to a node where other processes with the same + // reference currently reside. + if len(p.Config.Reference) != 0 { + raNodes := haveReferenceAffinity.Nodes(p.Config.Reference, p.Config.Domain) + for _, raNodeid := range raNodes { + // Do not move the process to the node it is currently on. + if raNodeid == overloadedNodeid { + continue + } + + if resources.HasNodeEnough(raNodeid, p.Config.LimitCPU, p.Config.LimitMemory) { + availableNodeid = raNodeid + break + } + } + } + + // Find the best node with enough resources available. + if len(availableNodeid) == 0 { + nodes := resources.FindBestNodes(p.Config.LimitCPU, p.Config.LimitMemory) + for _, nodeid := range nodes { + if nodeid == overloadedNodeid { + continue + } + + availableNodeid = nodeid + break + } + } + + if len(availableNodeid) == 0 { + // There's no other node with enough resources to take over this process. + opStack = append(opStack, processOpSkip{ + nodeid: overloadedNodeid, + processid: p.Config.ProcessID(), + err: errNotEnoughResourcesForRebalancing, + }) + continue + } + + opStack = append(opStack, processOpMove{ + fromNodeid: overloadedNodeid, + toNodeid: availableNodeid, + config: p.Config, + metadata: p.Metadata, + order: p.Order, + }) + + // Adjust the process. + p.NodeID = availableNodeid + processes[i] = p + + // Adjust the resources. + resources.Move(availableNodeid, overloadedNodeid, p.CPU, p.Mem) + + // Adjust the reference affinity. + haveReferenceAffinity.Move(p.Config.Reference, p.Config.Domain, overloadedNodeid, availableNodeid) + + // Move only one process at a time. + break + } + } + + return opStack, resources.Map() +} diff --git a/cluster/leader_relocate.go b/cluster/leader_relocate.go new file mode 100644 index 00000000..5879a1a9 --- /dev/null +++ b/cluster/leader_relocate.go @@ -0,0 +1,206 @@ +package cluster + +import ( + "github.com/datarhei/core/v16/cluster/node" + "github.com/datarhei/core/v16/cluster/store" + "github.com/datarhei/core/v16/log" + "github.com/datarhei/core/v16/restream/app" +) + +func (c *cluster) doRelocate(emergency bool, term uint64) { + if emergency { + // Don't relocate in emergency mode. + return + } + + logger := c.logger.WithField("term", term) + + logger.Debug().WithField("emergency", emergency).Log("Relocating") + + relocateMap := c.store.ProcessGetRelocateMap() + storeNodes := c.store.NodeList() + have := c.manager.ClusterProcessList() + nodes := c.manager.NodeList() + + nodesMap := map[string]node.About{} + + for _, node := range nodes { + about := node.About() + + if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { + about.State = storeNode.State + } + + nodesMap[about.ID] = about + } + + logger.Debug().WithFields(log.Fields{ + "relocate": relocate, + "have": have, + "nodes": nodesMap, + }).Log("Rebalance") + + opStack, _, relocatedProcessIDs := relocate(have, nodesMap, relocateMap) + + errors := c.applyOpStack(opStack, term) + + for _, e := range errors { + // Only apply the command if the error is different. + process, err := c.store.ProcessGet(e.processid) + if err != nil { + continue + } + + var errmessage string = "" + + if e.err != nil { + if process.Error == e.err.Error() { + continue + } + + errmessage = e.err.Error() + } else { + if len(process.Error) == 0 { + continue + } + } + + cmd := &store.Command{ + Operation: store.OpSetProcessError, + Data: store.CommandSetProcessError{ + ID: e.processid, + Error: errmessage, + }, + } + + c.applyCommand(cmd) + } + + cmd := store.CommandUnsetRelocateProcess{ + ID: []app.ProcessID{}, + } + + for _, processid := range relocatedProcessIDs { + cmd.ID = append(cmd.ID, app.ParseProcessID(processid)) + } + + if len(cmd.ID) != 0 { + c.applyCommand(&store.Command{ + Operation: store.OpUnsetRelocateProcess, + Data: cmd, + }) + } +} + +// relocate returns a list of operations that will move deployed processes to different nodes. +func relocate(have []node.Process, nodes map[string]node.About, relocateMap map[string]string) ([]interface{}, map[string]node.Resources, []string) { + resources := NewResourcePlanner(nodes) + + // Mark nodes as throttling where at least one process is still throttling + for _, haveP := range have { + if haveP.Throttling { + resources.Throttling(haveP.NodeID, true) + } + } + + relocatedProcessIDs := []string{} + + // A map from the process reference to the nodes it is running on. + haveReferenceAffinity := NewReferenceAffinity(have) + + opStack := []interface{}{} + + // Check for any requested relocations. + for processid, targetNodeid := range relocateMap { + process := node.Process{} + + found := false + for _, p := range have { + if processid == p.Config.ProcessID().String() { + process = p + found = true + break + } + } + + if !found { + relocatedProcessIDs = append(relocatedProcessIDs, processid) + continue + } + + sourceNodeid := process.NodeID + + if sourceNodeid == targetNodeid { + relocatedProcessIDs = append(relocatedProcessIDs, processid) + continue + } + + if len(targetNodeid) != 0 { + _, hasNode := nodes[targetNodeid] + + if !hasNode || !resources.HasNodeEnough(targetNodeid, process.Config.LimitCPU, process.Config.LimitMemory) { + targetNodeid = "" + } + } + + if len(targetNodeid) == 0 { + // Try to move the process to a node where other processes with the same + // reference currently reside. + if len(process.Config.Reference) != 0 { + raNodes := haveReferenceAffinity.Nodes(process.Config.Reference, process.Config.Domain) + for _, raNodeid := range raNodes { + // Do not move the process to the node it is currently on. + if raNodeid == sourceNodeid { + continue + } + + if resources.HasNodeEnough(raNodeid, process.Config.LimitCPU, process.Config.LimitMemory) { + targetNodeid = raNodeid + break + } + } + } + + // Find the best node with enough resources available. + if len(targetNodeid) == 0 { + nodes := resources.FindBestNodes(process.Config.LimitCPU, process.Config.LimitMemory) + for _, nodeid := range nodes { + if nodeid == sourceNodeid { + continue + } + + targetNodeid = nodeid + break + } + } + + if len(targetNodeid) == 0 { + // There's no other node with enough resources to take over this process. + opStack = append(opStack, processOpSkip{ + nodeid: sourceNodeid, + processid: process.Config.ProcessID(), + err: errNotEnoughResourcesForRelocating, + }) + continue + } + } + + opStack = append(opStack, processOpMove{ + fromNodeid: sourceNodeid, + toNodeid: targetNodeid, + config: process.Config, + metadata: process.Metadata, + order: process.Order, + }) + + // Adjust the resources. + resources.Move(targetNodeid, sourceNodeid, process.CPU, process.Mem) + + // Adjust the reference affinity. + haveReferenceAffinity.Move(process.Config.Reference, process.Config.Domain, sourceNodeid, targetNodeid) + + relocatedProcessIDs = append(relocatedProcessIDs, processid) + } + + return opStack, resources.Map(), relocatedProcessIDs +} diff --git a/cluster/leader_synchronize.go b/cluster/leader_synchronize.go new file mode 100644 index 00000000..60508433 --- /dev/null +++ b/cluster/leader_synchronize.go @@ -0,0 +1,377 @@ +package cluster + +import ( + "bytes" + "maps" + "time" + + "github.com/datarhei/core/v16/cluster/node" + "github.com/datarhei/core/v16/cluster/store" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/log" +) + +func (c *cluster) doSynchronize(emergency bool, term uint64) { + wish := c.store.ProcessGetNodeMap() + want := c.store.ProcessList() + storeNodes := c.store.NodeList() + have := c.manager.ClusterProcessList() + nodes := c.manager.NodeList() + + logger := c.logger.WithField("term", term) + + logger.Debug().WithField("emergency", emergency).Log("Synchronizing") + + nodesMap := map[string]node.About{} + + for _, node := range nodes { + about := node.About() + + if storeNode, hasStoreNode := storeNodes[about.ID]; hasStoreNode { + about.State = storeNode.State + } + + nodesMap[about.ID] = about + } + + logger.Debug().WithFields(log.Fields{ + "want": want, + "have": have, + "nodes": nodesMap, + }).Log("Synchronize") + + opStack, _, reality := synchronize(wish, want, have, nodesMap, c.nodeRecoverTimeout) + + if !emergency && !maps.Equal(wish, reality) { + cmd := &store.Command{ + Operation: store.OpSetProcessNodeMap, + Data: store.CommandSetProcessNodeMap{ + Map: reality, + }, + } + + c.applyCommand(cmd) + } + + errors := c.applyOpStack(opStack, term) + + if !emergency { + for _, e := range errors { + // Only apply the command if the error is different. + process, err := c.store.ProcessGet(e.processid) + if err != nil { + continue + } + + var errmessage string = "" + + if e.err != nil { + if process.Error == e.err.Error() { + continue + } + + errmessage = e.err.Error() + } else { + if len(process.Error) == 0 { + continue + } + } + + cmd := &store.Command{ + Operation: store.OpSetProcessError, + Data: store.CommandSetProcessError{ + ID: e.processid, + Error: errmessage, + }, + } + + c.applyCommand(cmd) + } + } +} + +// isMetadataUpdateRequired compares two metadata. It relies on the documented property that json.Marshal +// sorts the map keys prior encoding. +func isMetadataUpdateRequired(wantMap map[string]interface{}, haveMap map[string]interface{}) (bool, map[string]interface{}) { + hasChanges := false + changeMap := map[string]interface{}{} + + haveMapKeys := map[string]struct{}{} + + for key := range haveMap { + haveMapKeys[key] = struct{}{} + } + + for key, wantMapValue := range wantMap { + haveMapValue, ok := haveMap[key] + if !ok { + // A key in map1 exists, that doesn't exist in map2, we need to update. + hasChanges = true + } + + // Compare the values + changesData, err := json.Marshal(wantMapValue) + if err != nil { + continue + } + + completeData, err := json.Marshal(haveMapValue) + if err != nil { + continue + } + + if !bytes.Equal(changesData, completeData) { + // The values are not equal, we need to update. + hasChanges = true + } + + delete(haveMapKeys, key) + + changeMap[key] = wantMapValue + } + + for key := range haveMapKeys { + // If there keys in map2 that are not in map1, we have to update. + hasChanges = true + changeMap[key] = nil + } + + return hasChanges, changeMap +} + +// synchronize returns a list of operations in order to adjust the "have" list to the "want" list +// with taking the available resources on each node into account. +func synchronize(wish map[string]string, want []store.Process, have []node.Process, nodes map[string]node.About, nodeRecoverTimeout time.Duration) ([]interface{}, map[string]node.Resources, map[string]string) { + resources := NewResourcePlanner(nodes) + + // Mark nodes as throttling where at least one process is still throttling + for _, haveP := range have { + if haveP.Throttling { + resources.Throttling(haveP.NodeID, true) + } + } + + // A map same as wish, but reflecting the actual situation. + reality := map[string]string{} + + // A map from the process ID to the process config of the processes + // we want to be running on the nodes. + wantMap := map[string]store.Process{} + for _, wantP := range want { + pid := wantP.Config.ProcessID().String() + wantMap[pid] = wantP + } + + opStack := []interface{}{} + + // Now we iterate through the processes we actually have running on the nodes + // and remove them from the wantMap. We also make sure that they have the correct order. + // If a process cannot be found on the wantMap, it will be deleted from the nodes. + haveAfterRemove := []node.Process{} + wantOrderStart := []node.Process{} + + for _, haveP := range have { + pid := haveP.Config.ProcessID().String() + wantP, ok := wantMap[pid] + if !ok { + // The process is not on the wantMap. Delete it and adjust the resources. + opStack = append(opStack, processOpDelete{ + nodeid: haveP.NodeID, + processid: haveP.Config.ProcessID(), + }) + + resources.Remove(haveP.NodeID, haveP.CPU, haveP.Mem) + + continue + } + + // The process is on the wantMap. Update the process if the configuration and/or metadata differ. + hasConfigChanges := !wantP.Config.Equal(haveP.Config) + hasMetadataChanges, metadata := isMetadataUpdateRequired(wantP.Metadata, haveP.Metadata) + if hasConfigChanges || hasMetadataChanges { + // TODO: When the required resources increase, should we move this process to a node + // that has them available? Otherwise, this node might start throttling. However, this + // will result in rebalancing. + opStack = append(opStack, processOpUpdate{ + nodeid: haveP.NodeID, + processid: haveP.Config.ProcessID(), + config: wantP.Config, + metadata: metadata, + }) + } + + delete(wantMap, pid) + reality[pid] = haveP.NodeID + + if haveP.Order != wantP.Order { + if wantP.Order == "start" { + // Delay pushing them to the stack in order to have + // all resources released first. + wantOrderStart = append(wantOrderStart, haveP) + } else { + opStack = append(opStack, processOpStop{ + nodeid: haveP.NodeID, + processid: haveP.Config.ProcessID(), + }) + + // Release the resources. + resources.Remove(haveP.NodeID, haveP.CPU, haveP.Mem) + } + } + + haveAfterRemove = append(haveAfterRemove, haveP) + } + + for _, haveP := range wantOrderStart { + nodeid := haveP.NodeID + + resources.Add(nodeid, haveP.Config.LimitCPU, haveP.Config.LimitMemory) + + // TODO: check if the current node has actually enough resources available, + // otherwise it needs to be moved somewhere else. If the node doesn't + // have enough resources available, the process will be prevented + // from starting. + + /* + if hasNodeEnoughResources(r, haveP.Config.LimitCPU, haveP.Config.LimitMemory) { + // Consume the resources + r.CPU += haveP.Config.LimitCPU + r.Mem += haveP.Config.LimitMemory + resources[nodeid] = r + } else { + nodeid = findBestNodeForProcess(resources, haveP.Config.LimitCPU, haveP.Config.LimitMemory) + if len(nodeid) == 0 { + // Start it anyways and let it run into an error + opStack = append(opStack, processOpStart{ + nodeid: nodeid, + processid: haveP.Config.ProcessID(), + }) + + continue + } + + if nodeid != haveP.NodeID { + opStack = append(opStack, processOpMove{ + fromNodeid: haveP.NodeID, + toNodeid: nodeid, + config: haveP.Config, + metadata: haveP.Metadata, + order: haveP.Order, + }) + } + + // Consume the resources + r, ok := resources[nodeid] + if ok { + r.CPU += haveP.Config.LimitCPU + r.Mem += haveP.Config.LimitMemory + resources[nodeid] = r + } + } + */ + + opStack = append(opStack, processOpStart{ + nodeid: nodeid, + processid: haveP.Config.ProcessID(), + }) + } + + have = haveAfterRemove + + // In case a node didn't respond, some PID are still on the wantMap, that would run on + // the currently not responding nodes. We use the wish map to assign them to the node. + // If the node is unavailable for too long, keep these processes on the wantMap, otherwise + // remove them and hope that they will reappear during the nodeRecoverTimeout. + for pid := range wantMap { + // Check if this PID is be assigned to a node. + if nodeid, ok := wish[pid]; ok { + // Check for how long the node hasn't been contacted, or if it still exists. + if node, ok := nodes[nodeid]; ok { + if node.State == "online" { + continue + } + + if time.Since(node.LastContact) <= nodeRecoverTimeout { + reality[pid] = nodeid + delete(wantMap, pid) + } + } + } + } + + // The wantMap now contains only those processes that need to be installed on a node. + // We will rebuild the "want" array from the wantMap in the same order as the original + // "want" array to make the resulting opStack deterministic. + wantReduced := []store.Process{} + for _, wantP := range want { + pid := wantP.Config.ProcessID().String() + if _, ok := wantMap[pid]; !ok { + continue + } + + wantReduced = append(wantReduced, wantP) + } + + // Create a map from the process reference to the node it is running on. + haveReferenceAffinity := NewReferenceAffinity(have) + + // Now, all remaining processes in the wantMap must be added to one of the nodes. + for _, wantP := range wantReduced { + pid := wantP.Config.ProcessID().String() + + // If a process doesn't have any limits defined, reject that process. + if wantP.Config.LimitCPU <= 0 || wantP.Config.LimitMemory <= 0 { + opStack = append(opStack, processOpReject{ + processid: wantP.Config.ProcessID(), + err: errNoLimitsDefined, + }) + + continue + } + + // Check if there are already processes with the same reference, and if so + // choose this node. Then check the node if it has enough resources left. If + // not, then select a node with the most available resources. + nodeid := "" + + // Try to add the process to a node where other processes with the same reference currently reside. + raNodes := haveReferenceAffinity.Nodes(wantP.Config.Reference, wantP.Config.Domain) + for _, raNodeid := range raNodes { + if resources.HasNodeEnough(raNodeid, wantP.Config.LimitCPU, wantP.Config.LimitMemory) { + nodeid = raNodeid + break + } + } + + // Find the node with the most resources available. + if len(nodeid) == 0 { + nodes := resources.FindBestNodes(wantP.Config.LimitCPU, wantP.Config.LimitMemory) + if len(nodes) > 0 { + nodeid = nodes[0] + } + } + + if len(nodeid) != 0 { + opStack = append(opStack, processOpAdd{ + nodeid: nodeid, + config: wantP.Config, + metadata: wantP.Metadata, + order: wantP.Order, + }) + + // Consume the resources + resources.Add(nodeid, wantP.Config.LimitCPU, wantP.Config.LimitMemory) + + reality[pid] = nodeid + + haveReferenceAffinity.Add(wantP.Config.Reference, wantP.Config.Domain, nodeid) + } else { + opStack = append(opStack, processOpReject{ + processid: wantP.Config.ProcessID(), + err: errNotEnoughResourcesForDeployment, + }) + } + } + + return opStack, resources.Map(), reality +} diff --git a/cluster/leader_test.go b/cluster/leader_test.go index a1da5518..4f1d6bba 100644 --- a/cluster/leader_test.go +++ b/cluster/leader_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/cluster/store" "github.com/datarhei/core/v16/restream/app" @@ -35,13 +35,13 @@ func TestSynchronizeAdd(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 35, @@ -50,9 +50,9 @@ func TestSynchronizeAdd(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 11, @@ -90,7 +90,7 @@ func TestSynchronizeAdd(t *testing.T) { "foobaz@": "node1", }, reality) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 27, @@ -127,13 +127,13 @@ func TestSynchronizeAddDeleted(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -161,7 +161,7 @@ func TestSynchronizeAddDeleted(t *testing.T) { "foobar@": "node1", }, reality) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 11, @@ -191,7 +191,7 @@ func TestSynchronizeOrderStop(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -208,11 +208,11 @@ func TestSynchronizeOrderStop(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 20, Mem: 35, @@ -221,9 +221,9 @@ func TestSynchronizeOrderStop(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -246,7 +246,7 @@ func TestSynchronizeOrderStop(t *testing.T) { "foobar@": "node1", }, reality) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 8, @@ -283,7 +283,7 @@ func TestSynchronizeOrderStart(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "stop", @@ -300,11 +300,11 @@ func TestSynchronizeOrderStart(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 20, Mem: 35, @@ -313,9 +313,9 @@ func TestSynchronizeOrderStart(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -338,7 +338,7 @@ func TestSynchronizeOrderStart(t *testing.T) { "foobar@": "node1", }, reality) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 30, @@ -386,7 +386,7 @@ func TestSynchronizeAddReferenceAffinity(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node2", Order: "start", @@ -404,11 +404,11 @@ func TestSynchronizeAddReferenceAffinity(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -417,9 +417,9 @@ func TestSynchronizeAddReferenceAffinity(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -488,7 +488,7 @@ func TestSynchronizeAddReferenceAffinityMultiple(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node2", Order: "start", @@ -506,11 +506,11 @@ func TestSynchronizeAddReferenceAffinityMultiple(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -519,9 +519,9 @@ func TestSynchronizeAddReferenceAffinityMultiple(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -601,13 +601,13 @@ func TestSynchronizeAddReferenceAffinityMultipleEmptyNodes(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -616,9 +616,9 @@ func TestSynchronizeAddReferenceAffinityMultipleEmptyNodes(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -656,13 +656,13 @@ func TestSynchronizeAddLimit(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 81, Mem: 72, @@ -671,9 +671,9 @@ func TestSynchronizeAddLimit(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 79, Mem: 72, @@ -701,7 +701,7 @@ func TestSynchronizeAddLimit(t *testing.T) { "foobar@": "node2", }, reality) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 81, @@ -734,13 +734,13 @@ func TestSynchronizeAddNoResourcesCPU(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 81, Mem: 72, @@ -749,9 +749,9 @@ func TestSynchronizeAddNoResourcesCPU(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 79, Mem: 72, @@ -786,13 +786,13 @@ func TestSynchronizeAddNoResourcesMemory(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 81, Mem: 72, @@ -801,9 +801,9 @@ func TestSynchronizeAddNoResourcesMemory(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 79, Mem: 72, @@ -836,13 +836,13 @@ func TestSynchronizeAddNoLimits(t *testing.T) { }, } - have := []proxy.Process{} + have := []node.Process{} - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 81, Mem: 72, @@ -851,9 +851,9 @@ func TestSynchronizeAddNoLimits(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 79, Mem: 72, @@ -880,7 +880,7 @@ func TestSynchronizeRemove(t *testing.T) { want := []store.Process{} - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node2", Order: "start", @@ -894,11 +894,11 @@ func TestSynchronizeRemove(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 65, @@ -907,9 +907,9 @@ func TestSynchronizeRemove(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 11, @@ -928,7 +928,7 @@ func TestSynchronizeRemove(t *testing.T) { }, }, stack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 7, @@ -965,7 +965,7 @@ func TestSynchronizeAddRemove(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node2", Order: "start", @@ -979,11 +979,11 @@ func TestSynchronizeAddRemove(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 35, @@ -992,9 +992,9 @@ func TestSynchronizeAddRemove(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 65, @@ -1022,7 +1022,7 @@ func TestSynchronizeAddRemove(t *testing.T) { }, }, stack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 17, @@ -1062,7 +1062,7 @@ func TestSynchronizeNoUpdate(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1079,11 +1079,11 @@ func TestSynchronizeNoUpdate(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 35, @@ -1092,9 +1092,9 @@ func TestSynchronizeNoUpdate(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 65, @@ -1131,7 +1131,7 @@ func TestSynchronizeUpdate(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1148,11 +1148,11 @@ func TestSynchronizeUpdate(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 35, @@ -1161,9 +1161,9 @@ func TestSynchronizeUpdate(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 65, @@ -1215,7 +1215,7 @@ func TestSynchronizeUpdateMetadata(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1232,11 +1232,11 @@ func TestSynchronizeUpdateMetadata(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 7, Mem: 35, @@ -1245,9 +1245,9 @@ func TestSynchronizeUpdateMetadata(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 85, Mem: 65, @@ -1311,7 +1311,7 @@ func TestSynchronizeWaitDisconnectedNode(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1329,11 +1329,11 @@ func TestSynchronizeWaitDisconnectedNode(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -1342,9 +1342,9 @@ func TestSynchronizeWaitDisconnectedNode(t *testing.T) { }, }, "node2": { - State: "disconnected", + State: "offline", LastContact: time.Now().Add(-time.Minute), - Resources: proxy.NodeResources{ + Resources: node.Resources{ IsThrottling: true, NCPU: 1, CPU: 1, @@ -1395,7 +1395,7 @@ func TestSynchronizeWaitDisconnectedNodeNoWish(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1413,11 +1413,11 @@ func TestSynchronizeWaitDisconnectedNodeNoWish(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -1426,9 +1426,9 @@ func TestSynchronizeWaitDisconnectedNodeNoWish(t *testing.T) { }, }, "node2": { - State: "disconnected", + State: "offline", LastContact: time.Now().Add(-time.Minute), - Resources: proxy.NodeResources{ + Resources: node.Resources{ IsThrottling: true, NCPU: 1, CPU: 1, @@ -1491,7 +1491,7 @@ func TestSynchronizeWaitDisconnectedNodeUnrealisticWish(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1509,11 +1509,11 @@ func TestSynchronizeWaitDisconnectedNodeUnrealisticWish(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -1522,9 +1522,9 @@ func TestSynchronizeWaitDisconnectedNodeUnrealisticWish(t *testing.T) { }, }, "node2": { - State: "disconnected", + State: "offline", LastContact: time.Now().Add(-time.Minute), - Resources: proxy.NodeResources{ + Resources: node.Resources{ IsThrottling: true, NCPU: 1, CPU: 1, @@ -1587,7 +1587,7 @@ func TestSynchronizeTimeoutDisconnectedNode(t *testing.T) { }, } - have := []proxy.Process{ + have := []node.Process{ { NodeID: "node1", Order: "start", @@ -1605,11 +1605,11 @@ func TestSynchronizeTimeoutDisconnectedNode(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -1619,8 +1619,8 @@ func TestSynchronizeTimeoutDisconnectedNode(t *testing.T) { }, "node2": { LastContact: time.Now().Add(-3 * time.Minute), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ IsThrottling: true, NCPU: 1, CPU: 1, @@ -1653,7 +1653,7 @@ func TestSynchronizeTimeoutDisconnectedNode(t *testing.T) { } func TestRebalanceNothingToDo(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -1678,11 +1678,11 @@ func TestRebalanceNothingToDo(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 42, Mem: 35, @@ -1692,8 +1692,8 @@ func TestRebalanceNothingToDo(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 37, Mem: 11, @@ -1709,7 +1709,7 @@ func TestRebalanceNothingToDo(t *testing.T) { } func TestRebalanceOverload(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -1745,11 +1745,11 @@ func TestRebalanceOverload(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 91, Mem: 35, @@ -1759,8 +1759,8 @@ func TestRebalanceOverload(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 15, Mem: 11, @@ -1785,7 +1785,7 @@ func TestRebalanceOverload(t *testing.T) { }, }, opStack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 74, @@ -1804,7 +1804,7 @@ func TestRebalanceOverload(t *testing.T) { } func TestRebalanceSkip(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -1840,11 +1840,11 @@ func TestRebalanceSkip(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 91, Mem: 35, @@ -1854,8 +1854,8 @@ func TestRebalanceSkip(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 15, Mem: 92, @@ -1887,7 +1887,7 @@ func TestRebalanceSkip(t *testing.T) { }, }, opStack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 91, @@ -1906,7 +1906,7 @@ func TestRebalanceSkip(t *testing.T) { } func TestRebalanceReferenceAffinity(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -1968,11 +1968,11 @@ func TestRebalanceReferenceAffinity(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 90, Mem: 90, @@ -1982,8 +1982,8 @@ func TestRebalanceReferenceAffinity(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -1993,8 +1993,8 @@ func TestRebalanceReferenceAffinity(t *testing.T) { }, "node3": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 1, Mem: 1, @@ -2020,7 +2020,7 @@ func TestRebalanceReferenceAffinity(t *testing.T) { }, }, opStack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 89, @@ -2046,7 +2046,7 @@ func TestRebalanceReferenceAffinity(t *testing.T) { } func TestRebalanceRelocateTarget(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -2082,11 +2082,11 @@ func TestRebalanceRelocateTarget(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 27, Mem: 35, @@ -2096,8 +2096,8 @@ func TestRebalanceRelocateTarget(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 15, Mem: 11, @@ -2107,8 +2107,8 @@ func TestRebalanceRelocateTarget(t *testing.T) { }, "node3": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 0, Mem: 0, @@ -2137,7 +2137,7 @@ func TestRebalanceRelocateTarget(t *testing.T) { }, }, opStack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 0, @@ -2163,7 +2163,7 @@ func TestRebalanceRelocateTarget(t *testing.T) { } func TestRebalanceRelocateAny(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -2199,11 +2199,11 @@ func TestRebalanceRelocateAny(t *testing.T) { }, } - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 27, Mem: 35, @@ -2213,8 +2213,8 @@ func TestRebalanceRelocateAny(t *testing.T) { }, "node2": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 15, Mem: 11, @@ -2224,8 +2224,8 @@ func TestRebalanceRelocateAny(t *testing.T) { }, "node3": { LastContact: time.Now(), - State: "connected", - Resources: proxy.NodeResources{ + State: "online", + Resources: node.Resources{ NCPU: 1, CPU: 0, Mem: 0, @@ -2255,7 +2255,7 @@ func TestRebalanceRelocateAny(t *testing.T) { }, }, opStack) - require.Equal(t, map[string]proxy.NodeResources{ + require.Equal(t, map[string]node.Resources{ "node1": { NCPU: 1, CPU: 0, @@ -2281,11 +2281,11 @@ func TestRebalanceRelocateAny(t *testing.T) { } func TestFindBestNodesForProcess(t *testing.T) { - nodes := map[string]proxy.NodeAbout{ + nodes := map[string]node.About{ "node1": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 27, Mem: 35, @@ -2294,9 +2294,9 @@ func TestFindBestNodesForProcess(t *testing.T) { }, }, "node2": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 15, Mem: 11, @@ -2305,9 +2305,9 @@ func TestFindBestNodesForProcess(t *testing.T) { }, }, "node3": { - State: "connected", + State: "online", LastContact: time.Now(), - Resources: proxy.NodeResources{ + Resources: node.Resources{ NCPU: 1, CPU: 0, Mem: 0, @@ -2317,7 +2317,7 @@ func TestFindBestNodesForProcess(t *testing.T) { }, } - resources := NewResources(nodes) + resources := NewResourcePlanner(nodes) list := resources.FindBestNodes(35, 20) @@ -2325,8 +2325,8 @@ func TestFindBestNodesForProcess(t *testing.T) { } func TestFindBestNodesForProcess2(t *testing.T) { - resources := NewResources(nil) - resources.nodes = map[string]proxy.NodeResources{ + resources := NewResourcePlanner(nil) + resources.nodes = map[string]node.Resources{ "node1": { CPULimit: 104.50000000000001, CPU: 29.725299999999997, @@ -2439,7 +2439,7 @@ func TestFindBestNodesForProcess2(t *testing.T) { } func TestCreateNodeProcessMap(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", @@ -2539,7 +2539,7 @@ func TestCreateNodeProcessMap(t *testing.T) { nodeProcessMap := createNodeProcessMap(processes) - require.Equal(t, map[string][]proxy.Process{ + require.Equal(t, map[string][]node.Process{ "node1": { { NodeID: "node1", @@ -2621,7 +2621,7 @@ func TestCreateNodeProcessMap(t *testing.T) { } func TestCreateReferenceAffinityNodeMap(t *testing.T) { - processes := []proxy.Process{ + processes := []node.Process{ { NodeID: "node1", Order: "start", diff --git a/cluster/node.go b/cluster/node.go index d54ce25a..852fb100 100644 --- a/cluster/node.go +++ b/cluster/node.go @@ -7,12 +7,12 @@ import ( ) func (c *cluster) ListNodes() map[string]store.Node { - return c.store.ListNodes() + return c.store.NodeList() } var ErrUnsupportedNodeState = errors.New("unsupported node state") -func (c *cluster) SetNodeState(origin, id, state string) error { +func (c *cluster) NodeSetState(origin, id, state string) error { switch state { case "online": case "maintenance": @@ -22,7 +22,7 @@ func (c *cluster) SetNodeState(origin, id, state string) error { } if !c.IsRaftLeader() { - return c.forwarder.SetNodeState(origin, id, state) + return c.forwarder.NodeSetState(origin, id, state) } cmd := &store.Command{ diff --git a/cluster/proxy/cache.go b/cluster/node/cache.go similarity index 99% rename from cluster/proxy/cache.go rename to cluster/node/cache.go index 0e7edbb4..bce3db5c 100644 --- a/cluster/proxy/cache.go +++ b/cluster/node/cache.go @@ -1,4 +1,4 @@ -package proxy +package node import ( "errors" diff --git a/cluster/proxy/cache_test.go b/cluster/node/cache_test.go similarity index 98% rename from cluster/proxy/cache_test.go rename to cluster/node/cache_test.go index 5827cd48..2c8d7a8f 100644 --- a/cluster/proxy/cache_test.go +++ b/cluster/node/cache_test.go @@ -1,4 +1,4 @@ -package proxy +package node import ( "testing" diff --git a/cluster/node/core.go b/cluster/node/core.go new file mode 100644 index 00000000..94a919fa --- /dev/null +++ b/cluster/node/core.go @@ -0,0 +1,806 @@ +package node + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/datarhei/core/v16/config" + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/http/client" + "github.com/datarhei/core/v16/log" + "github.com/datarhei/core/v16/restream/app" +) + +type Core struct { + id string + + client client.RestClient + clientErr error + + lock sync.RWMutex + + cancel context.CancelFunc + + address string + config *config.Config + + secure bool + httpAddress *url.URL + hasRTMP bool + rtmpAddress *url.URL + hasSRT bool + srtAddress *url.URL + + logger log.Logger +} + +var ErrNoPeer = errors.New("not connected to the core API: client not available") + +func NewCore(id string, logger log.Logger) *Core { + core := &Core{ + id: id, + logger: logger, + } + + if core.logger == nil { + core.logger = log.New("") + } + + ctx, cancel := context.WithCancel(context.Background()) + core.cancel = cancel + + go core.reconnect(ctx, time.Second) + + return core +} + +func (n *Core) SetEssentials(address string, config *config.Config) { + n.lock.Lock() + defer n.lock.Unlock() + + if address != n.address { + n.address = address + n.client = nil // force reconnet + } + + if n.config == nil && config != nil { + n.config = config + n.client = nil // force reconnect + } + + if n.config.UpdatedAt != config.UpdatedAt { + n.config = config + n.client = nil // force reconnect + } +} + +func (n *Core) reconnect(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + err := n.connect() + + n.lock.Lock() + n.clientErr = err + n.lock.Unlock() + } + } +} + +func (n *Core) Stop() { + n.lock.Lock() + defer n.lock.Unlock() + + if n.cancel == nil { + return + } + + n.cancel() + n.cancel = nil +} + +func (n *Core) Reconnect() { + n.lock.Lock() + defer n.lock.Unlock() + + n.client = nil +} + +func (n *Core) connect() error { + n.lock.Lock() + + if n.client != nil { + n.lock.Unlock() + return nil + } + + if len(n.address) == 0 { + n.lock.Unlock() + return fmt.Errorf("no address provided") + } + + if n.config == nil { + n.lock.Unlock() + return fmt.Errorf("config not available") + } + + address := n.address + config := n.config.Clone() + + n.lock.Unlock() + + u, err := url.Parse(address) + if err != nil { + return fmt.Errorf("invalid address (%s): %w", address, err) + } + + secure := strings.HasPrefix(config.Address, "https://") + + nodehost, _, err := net.SplitHostPort(u.Host) + if err != nil { + return fmt.Errorf("invalid address (%s): %w", u.Host, err) + } + + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.MaxIdleConns = 10 + tr.IdleConnTimeout = 30 * time.Second + + client, err := client.New(client.Config{ + Address: u.String(), + Client: &http.Client{ + Transport: tr, + Timeout: 5 * time.Second, + }, + }) + if err != nil { + return fmt.Errorf("creating client failed (%s): %w", address, err) + } + + httpAddress := u + hasRTMP := false + rtmpAddress := &url.URL{} + + if config.RTMP.Enable { + hasRTMP = true + rtmpAddress.Scheme = "rtmp" + + isHostIP := net.ParseIP(nodehost) != nil + + address := config.RTMP.Address + if secure && config.RTMP.EnableTLS && !isHostIP { + address = config.RTMP.AddressTLS + rtmpAddress.Scheme = "rtmps" + } + + host, port, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("invalid rtmp address '%s': %w", address, err) + } + + if len(host) == 0 { + rtmpAddress.Host = net.JoinHostPort(nodehost, port) + } else { + rtmpAddress.Host = net.JoinHostPort(host, port) + } + + rtmpAddress = rtmpAddress.JoinPath(n.config.RTMP.App) + } + + hasSRT := false + srtAddress := &url.URL{} + + if config.SRT.Enable { + hasSRT = true + srtAddress.Scheme = "srt" + + host, port, err := net.SplitHostPort(config.SRT.Address) + if err != nil { + return fmt.Errorf("invalid srt address '%s': %w", config.SRT.Address, err) + } + + if len(host) == 0 { + srtAddress.Host = net.JoinHostPort(nodehost, port) + } else { + srtAddress.Host = net.JoinHostPort(host, port) + } + + v := url.Values{} + + v.Set("mode", "caller") + if len(config.SRT.Passphrase) != 0 { + v.Set("passphrase", config.SRT.Passphrase) + } + + srtAddress.RawQuery = v.Encode() + } + + n.lock.Lock() + + n.secure = secure + n.httpAddress = httpAddress + n.hasRTMP = hasRTMP + n.rtmpAddress = rtmpAddress + n.hasSRT = hasSRT + n.srtAddress = srtAddress + n.client = client + + n.lock.Unlock() + + return nil +} + +type CoreAbout struct { + ID string + Name string + Address string + State string + Error error + CreatedAt time.Time + Uptime time.Duration + LastContact time.Time + Latency time.Duration + Version CoreVersion +} + +type CoreVersion struct { + Number string + Commit string + Branch string + Build time.Time + Arch string + Compiler string +} + +func (n *Core) About() (CoreAbout, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return CoreAbout{}, ErrNoPeer + } + + about, err := client.About(false) + if err != nil { + return CoreAbout{}, err + } + + cabout := CoreAbout{ + ID: about.ID, + Name: about.Name, + Address: n.address, + Error: n.clientErr, + Version: CoreVersion{ + Number: about.Version.Number, + Commit: about.Version.Commit, + Branch: about.Version.Branch, + Arch: about.Version.Arch, + Compiler: about.Version.Compiler, + }, + } + + createdAt, err := time.Parse(time.RFC3339, about.CreatedAt) + if err != nil { + createdAt = time.Now() + } + + cabout.CreatedAt = createdAt + cabout.Uptime = time.Since(createdAt) + + build, err := time.Parse(time.RFC3339, about.Version.Build) + if err != nil { + build = time.Time{} + } + + cabout.Version.Build = build + + return cabout, nil +} + +func (n *Core) ProcessAdd(config *app.Config, metadata map[string]interface{}) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.ProcessAdd(config, metadata) +} + +func (n *Core) ProcessCommand(id app.ProcessID, command string) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.ProcessCommand(id, command) +} + +func (n *Core) ProcessDelete(id app.ProcessID) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.ProcessDelete(id) +} + +func (n *Core) ProcessUpdate(id app.ProcessID, config *app.Config, metadata map[string]interface{}) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.ProcessUpdate(id, config, metadata) +} + +func (n *Core) ProcessProbe(id app.ProcessID) (api.Probe, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + probe := api.Probe{ + Log: []string{fmt.Sprintf("the node %s where the process %s resides, is not connected", n.id, id.String())}, + } + return probe, ErrNoPeer + } + + probe, err := client.ProcessProbe(id) + + probe.Log = append([]string{fmt.Sprintf("probed on node: %s", n.id)}, probe.Log...) + + return probe, err +} + +func (n *Core) ProcessProbeConfig(config *app.Config) (api.Probe, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + probe := api.Probe{ + Log: []string{fmt.Sprintf("the node %s where the process config should be probed, is not connected", n.id)}, + } + return probe, ErrNoPeer + } + + probe, err := client.ProcessProbeConfig(config) + + probe.Log = append([]string{fmt.Sprintf("probed on node: %s", n.id)}, probe.Log...) + + return probe, err +} + +func (n *Core) ProcessList(options client.ProcessListOptions) ([]api.Process, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return nil, ErrNoPeer + } + + return client.ProcessList(options) +} + +func (n *Core) FilesystemList(storage, pattern string) ([]api.FileInfo, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return nil, ErrNoPeer + } + + files, err := client.FilesystemList(storage, pattern, "", "") + if err != nil { + return nil, err + } + + for i := range files { + files[i].CoreID = n.id + } + + return files, nil +} + +func (n *Core) FilesystemDeleteFile(storage, path string) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.FilesystemDeleteFile(storage, path) +} + +func (n *Core) FilesystemPutFile(storage, path string, data io.Reader) error { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return ErrNoPeer + } + + return client.FilesystemAddFile(storage, path, data) +} + +func (n *Core) FilesystemGetFileInfo(storage, path string) (int64, time.Time, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return 0, time.Time{}, ErrNoPeer + } + + info, err := client.FilesystemList(storage, path, "", "") + if err != nil { + return 0, time.Time{}, fmt.Errorf("file not found: %w", err) + } + + if len(info) != 1 { + return 0, time.Time{}, fmt.Errorf("ambigous result") + } + + return info[0].Size, time.Unix(info[0].LastMod, 0), nil +} + +func (n *Core) FilesystemGetFile(storage, path string, offset int64) (io.ReadCloser, error) { + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return nil, ErrNoPeer + } + + return client.FilesystemGetFileOffset(storage, path, offset) +} + +type NodeFiles struct { + ID string + Files []string + LastUpdate time.Time +} + +func (n *Core) MediaList() NodeFiles { + files := NodeFiles{ + ID: n.id, + Files: []string{}, + LastUpdate: time.Now(), + } + + errorsChan := make(chan error, 8) + filesChan := make(chan string, 1024) + errorList := []error{} + + wgList := sync.WaitGroup{} + wgList.Add(1) + + go func() { + defer wgList.Done() + + for file := range filesChan { + files.Files = append(files.Files, file) + } + + for err := range errorsChan { + errorList = append(errorList, err) + } + }() + + wg := sync.WaitGroup{} + wg.Add(2) + + go func(f chan<- string, e chan<- error) { + defer wg.Done() + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + e <- ErrNoPeer + return + } + + files, err := client.FilesystemList("mem", "/*", "name", "asc") + if err != nil { + e <- err + return + } + + for _, file := range files { + f <- "mem:" + file.Name + } + }(filesChan, errorsChan) + + go func(f chan<- string, e chan<- error) { + defer wg.Done() + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + e <- ErrNoPeer + return + } + + files, err := client.FilesystemList("disk", "/*", "name", "asc") + if err != nil { + e <- err + return + } + + for _, file := range files { + f <- "disk:" + file.Name + } + }(filesChan, errorsChan) + + if n.hasRTMP { + wg.Add(1) + + go func(f chan<- string, e chan<- error) { + defer wg.Done() + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + e <- ErrNoPeer + return + } + + files, err := client.RTMPChannels() + if err != nil { + e <- err + return + } + + for _, file := range files { + f <- "rtmp:" + file.Name + } + }(filesChan, errorsChan) + } + + if n.hasSRT { + wg.Add(1) + + go func(f chan<- string, e chan<- error) { + defer wg.Done() + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + e <- ErrNoPeer + return + } + + files, err := client.SRTChannels() + if err != nil { + e <- err + return + } + + for _, file := range files { + f <- "srt:" + file.Name + } + }(filesChan, errorsChan) + } + + wg.Wait() + + close(filesChan) + close(errorsChan) + + wgList.Wait() + + return files +} + +func cloneURL(src *url.URL) *url.URL { + dst := &url.URL{ + Scheme: src.Scheme, + Opaque: src.Opaque, + User: nil, + Host: src.Host, + Path: src.Path, + RawPath: src.RawPath, + OmitHost: src.OmitHost, + ForceQuery: src.ForceQuery, + RawQuery: src.RawQuery, + Fragment: src.Fragment, + RawFragment: src.RawFragment, + } + + if src.User != nil { + username := src.User.Username() + password, ok := src.User.Password() + + if ok { + dst.User = url.UserPassword(username, password) + } else { + dst.User = url.User(username) + } + } + + return dst +} + +func (n *Core) MediaGetURL(prefix, path string) (*url.URL, error) { + var u *url.URL + + if prefix == "mem" { + u = cloneURL(n.httpAddress) + u = u.JoinPath("memfs", path) + } else if prefix == "disk" { + u = cloneURL(n.httpAddress) + u = u.JoinPath(path) + } else if prefix == "rtmp" { + u = cloneURL(n.rtmpAddress) + u = u.JoinPath(path) + } else if prefix == "srt" { + u = cloneURL(n.srtAddress) + } else { + return nil, fmt.Errorf("unknown prefix") + } + + return u, nil +} + +func (n *Core) MediaGetInfo(prefix, path string) (int64, time.Time, error) { + if prefix == "disk" || prefix == "mem" { + return n.FilesystemGetFileInfo(prefix, path) + } + + if prefix != "rtmp" && prefix != "srt" { + return 0, time.Time{}, fmt.Errorf("unknown prefix: %s", prefix) + } + + n.lock.RLock() + client := n.client + n.lock.RUnlock() + + if client == nil { + return 0, time.Time{}, ErrNoPeer + } + + if prefix == "rtmp" { + files, err := n.client.RTMPChannels() + if err != nil { + return 0, time.Time{}, err + } + + for _, file := range files { + if path == file.Name { + return 0, time.Now(), nil + } + } + + return 0, time.Time{}, fmt.Errorf("media not found") + } + + if prefix == "srt" { + files, err := n.client.SRTChannels() + if err != nil { + return 0, time.Time{}, err + } + + for _, file := range files { + if path == file.Name { + return 0, time.Now(), nil + } + } + + return 0, time.Time{}, fmt.Errorf("media not found") + } + + return 0, time.Time{}, fmt.Errorf("unknown prefix: %s", prefix) +} + +type Process struct { + NodeID string + Order string + State string + CPU float64 // Current CPU load of this process, 0-100*ncpu + Mem uint64 // Currently consumed memory of this process in bytes + Throttling bool + Runtime time.Duration + UpdatedAt time.Time + Config *app.Config + Metadata map[string]interface{} +} + +func (n *Core) ClusterProcessList() ([]Process, error) { + list, err := n.ProcessList(client.ProcessListOptions{ + Filter: []string{"config", "state", "metadata"}, + }) + if err != nil { + return nil, err + } + + nodeid := n.id + + processes := []Process{} + + for _, p := range list { + if p.State == nil { + p.State = &api.ProcessState{} + } + + if p.Config == nil { + p.Config = &api.ProcessConfig{} + } + + cpu, err := p.State.Resources.CPU.Current.Float64() + if err != nil { + cpu = 0 + } + + process := Process{ + NodeID: nodeid, + Order: p.State.Order, + State: p.State.State, + Mem: p.State.Resources.Memory.Current, + CPU: cpu, + Throttling: p.State.Resources.CPU.IsThrottling, + Runtime: time.Duration(p.State.Runtime) * time.Second, + UpdatedAt: time.Unix(p.UpdatedAt, 0), + } + + config, metadata := p.Config.Marshal() + + process.Config = config + process.Metadata = metadata + + processes = append(processes, process) + } + + return processes, nil +} diff --git a/cluster/node/manager.go b/cluster/node/manager.go new file mode 100644 index 00000000..d0c345b7 --- /dev/null +++ b/cluster/node/manager.go @@ -0,0 +1,601 @@ +package node + +import ( + "errors" + "fmt" + "io" + "net/url" + "sort" + "sync" + "time" + + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/http/client" + "github.com/datarhei/core/v16/log" + "github.com/datarhei/core/v16/restream/app" +) + +type ProcessListOptions = client.ProcessListOptions + +type ManagerConfig struct { + ID string // ID of the node + + Logger log.Logger +} + +type Manager struct { + id string + + nodes map[string]*Node // List of known nodes + lock sync.RWMutex + + cache *Cache[string] + + logger log.Logger +} + +var ErrNodeNotFound = errors.New("node not found") + +func NewManager(config ManagerConfig) (*Manager, error) { + p := &Manager{ + id: config.ID, + nodes: map[string]*Node{}, + cache: NewCache[string](nil), + logger: config.Logger, + } + + if p.logger == nil { + p.logger = log.New("") + } + + return p, nil +} + +func (p *Manager) NodeAdd(id string, node *Node) (string, error) { + about := node.About() + + p.lock.Lock() + defer p.lock.Unlock() + + if n, ok := p.nodes[id]; ok { + n.Stop() + delete(p.nodes, id) + } + + p.nodes[id] = node + + p.logger.Info().WithFields(log.Fields{ + "address": about.Address, + "name": about.Name, + "id": id, + }).Log("Added node") + + return id, nil +} + +func (p *Manager) NodeRemove(id string) (*Node, error) { + p.lock.Lock() + defer p.lock.Unlock() + + node, ok := p.nodes[id] + if !ok { + return nil, ErrNodeNotFound + } + + node.Stop() + + delete(p.nodes, id) + + p.logger.Info().WithFields(log.Fields{ + "id": id, + }).Log("Removed node") + + return node, nil +} + +func (p *Manager) NodesClear() { + p.lock.Lock() + defer p.lock.Unlock() + + for _, node := range p.nodes { + node.Stop() + } + + p.nodes = map[string]*Node{} + + p.logger.Info().Log("Removed all nodes") +} + +func (p *Manager) NodeHasNode(id string) bool { + p.lock.RLock() + defer p.lock.RUnlock() + + _, hasNode := p.nodes[id] + + return hasNode +} + +func (p *Manager) NodeIDs() []string { + list := []string{} + + p.lock.RLock() + defer p.lock.RUnlock() + + for id := range p.nodes { + list = append(list, id) + } + + return list +} + +func (p *Manager) NodeCount() int { + p.lock.RLock() + defer p.lock.RUnlock() + + return len(p.nodes) +} + +func (p *Manager) NodeList() []*Node { + list := []*Node{} + + p.lock.RLock() + defer p.lock.RUnlock() + + for _, node := range p.nodes { + list = append(list, node) + } + + return list +} + +func (p *Manager) NodeGet(id string) (*Node, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + node, ok := p.nodes[id] + if !ok { + return nil, fmt.Errorf("node not found") + } + + return node, nil +} + +func (p *Manager) NodeCheckCompatibility(skipSkillsCheck bool) { + p.lock.RLock() + defer p.lock.RUnlock() + + local, hasLocal := p.nodes[p.id] + if !hasLocal { + local = nil + } + + for id, node := range p.nodes { + if id == p.id { + continue + } + + node.CheckCompatibility(local, skipSkillsCheck) + } +} + +func (p *Manager) Barrier(name string) (bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + for _, node := range p.nodes { + ok, err := node.Barrier(name) + if !ok { + return false, err + } + } + + return true, nil +} + +// getClusterHostnames return a list of all hostnames configured on all nodes. The +// returned list will not contain any duplicates. +func (p *Manager) GetHostnames(common bool) ([]string, error) { + hostnames := map[string]int{} + + p.lock.RLock() + defer p.lock.RUnlock() + + for id, node := range p.nodes { + config, err := node.CoreConfig(true) + if err != nil { + return nil, fmt.Errorf("node %s has no configuration available: %w", id, err) + } + + for _, name := range config.Host.Name { + hostnames[name]++ + } + } + + names := []string{} + + for key, value := range hostnames { + if common && value != len(p.nodes) { + continue + } + + names = append(names, key) + } + + sort.Strings(names) + + return names, nil +} + +func (p *Manager) MediaGetURL(prefix, path string) (*url.URL, error) { + logger := p.logger.WithFields(log.Fields{ + "path": path, + "prefix": prefix, + }) + + node, err := p.getNodeForMedia(prefix, path) + if err != nil { + logger.Debug().WithError(err).Log("Unknown node") + return nil, fmt.Errorf("file not found: %w", err) + } + + url, err := node.Core().MediaGetURL(prefix, path) + if err != nil { + logger.Debug().Log("Invalid path") + return nil, fmt.Errorf("file not found") + } + + logger.Debug().WithField("url", url).Log("File cluster url") + + return url, nil +} + +func (p *Manager) FilesystemGetFile(prefix, path string, offset int64) (io.ReadCloser, error) { + logger := p.logger.WithFields(log.Fields{ + "path": path, + "prefix": prefix, + }) + + node, err := p.getNodeForMedia(prefix, path) + if err != nil { + logger.Debug().WithError(err).Log("File not available") + return nil, fmt.Errorf("file not found") + } + + data, err := node.Core().FilesystemGetFile(prefix, path, offset) + if err != nil { + logger.Debug().Log("Invalid path") + return nil, fmt.Errorf("file not found") + } + + logger.Debug().Log("File cluster path") + + return data, nil +} + +func (p *Manager) FilesystemGetFileInfo(prefix, path string) (int64, time.Time, error) { + logger := p.logger.WithFields(log.Fields{ + "path": path, + "prefix": prefix, + }) + + node, err := p.getNodeForMedia(prefix, path) + if err != nil { + logger.Debug().WithError(err).Log("File not available") + return 0, time.Time{}, fmt.Errorf("file not found") + } + + size, lastModified, err := node.Core().FilesystemGetFileInfo(prefix, path) + if err != nil { + logger.Debug().Log("Invalid path") + return 0, time.Time{}, fmt.Errorf("file not found") + } + + logger.Debug().Log("File cluster path") + + return size, lastModified, nil +} + +func (p *Manager) getNodeIDForMedia(prefix, path string) (string, error) { + // this is only for mem and disk prefixes + nodesChan := make(chan string, 16) + nodeids := []string{} + + wgList := sync.WaitGroup{} + wgList.Add(1) + + go func() { + defer wgList.Done() + + for nodeid := range nodesChan { + if len(nodeid) == 0 { + continue + } + + nodeids = append(nodeids, nodeid) + } + }() + + wg := sync.WaitGroup{} + + p.lock.RLock() + for id, n := range p.nodes { + wg.Add(1) + + go func(nodeid string, node *Node, p chan<- string) { + defer wg.Done() + + _, _, err := node.Core().MediaGetInfo(prefix, path) + if err != nil { + nodeid = "" + } + + p <- nodeid + }(id, n, nodesChan) + } + p.lock.RUnlock() + + wg.Wait() + + close(nodesChan) + + wgList.Wait() + + if len(nodeids) == 0 { + return "", fmt.Errorf("file not found") + } + + return nodeids[0], nil +} + +func (p *Manager) getNodeForMedia(prefix, path string) (*Node, error) { + id, err := p.cache.Get(prefix + ":" + path) + if err == nil { + node, err := p.NodeGet(id) + if err == nil { + return node, nil + } + } + + id, err = p.getNodeIDForMedia(prefix, path) + if err != nil { + return nil, err + } + + p.cache.Put(prefix+":"+path, id, 5*time.Second) + + return p.NodeGet(id) +} + +func (p *Manager) FilesystemList(storage, pattern string) []api.FileInfo { + filesChan := make(chan []api.FileInfo, 64) + filesList := []api.FileInfo{} + + wgList := sync.WaitGroup{} + wgList.Add(1) + + go func() { + defer wgList.Done() + + for list := range filesChan { + filesList = append(filesList, list...) + } + }() + + wg := sync.WaitGroup{} + + p.lock.RLock() + for _, n := range p.nodes { + wg.Add(1) + + go func(node *Node, p chan<- []api.FileInfo) { + defer wg.Done() + + files, err := node.Core().FilesystemList(storage, pattern) + if err != nil { + return + } + + p <- files + }(n, filesChan) + } + p.lock.RUnlock() + + wg.Wait() + + close(filesChan) + + wgList.Wait() + + return filesList +} + +func (p *Manager) ClusterProcessList() []Process { + processChan := make(chan []Process, 64) + processList := []Process{} + + wgList := sync.WaitGroup{} + wgList.Add(1) + + go func() { + defer wgList.Done() + + for list := range processChan { + processList = append(processList, list...) + } + }() + + wg := sync.WaitGroup{} + + p.lock.RLock() + for _, n := range p.nodes { + wg.Add(1) + + go func(node *Node, p chan<- []Process) { + defer wg.Done() + + processes, err := node.Core().ClusterProcessList() + if err != nil { + return + } + + p <- processes + }(n, processChan) + } + p.lock.RUnlock() + + wg.Wait() + + close(processChan) + + wgList.Wait() + + return processList +} + +func (p *Manager) ProcessFindNodeID(id app.ProcessID) (string, error) { + procs := p.ClusterProcessList() + nodeid := "" + + for _, p := range procs { + if p.Config.ProcessID() != id { + continue + } + + nodeid = p.NodeID + + break + } + + if len(nodeid) == 0 { + return "", fmt.Errorf("the process '%s' is not registered with any node", id.String()) + } + + return nodeid, nil +} + +func (p *Manager) FindNodeForResources(nodeid string, cpu float64, memory uint64) string { + p.lock.RLock() + defer p.lock.RUnlock() + + if len(nodeid) != 0 { + node, ok := p.nodes[nodeid] + if ok { + r := node.About().Resources + if r.Error == nil && r.CPU+cpu <= r.CPULimit && r.Mem+memory <= r.MemLimit && !r.IsThrottling { + return nodeid + } + } + } + + for nodeid, node := range p.nodes { + r := node.About().Resources + if r.Error == nil && r.CPU+cpu <= r.CPULimit && r.Mem+memory <= r.MemLimit && !r.IsThrottling { + return nodeid + } + } + + return "" +} + +func (p *Manager) ProcessList(options client.ProcessListOptions) []api.Process { + processChan := make(chan []api.Process, 64) + processList := []api.Process{} + + wgList := sync.WaitGroup{} + wgList.Add(1) + + go func() { + defer wgList.Done() + + for list := range processChan { + processList = append(processList, list...) + } + }() + + wg := sync.WaitGroup{} + + p.lock.RLock() + for _, n := range p.nodes { + wg.Add(1) + + go func(node *Node, p chan<- []api.Process) { + defer wg.Done() + + processes, err := node.Core().ProcessList(options) + if err != nil { + return + } + + p <- processes + }(n, processChan) + } + p.lock.RUnlock() + + wg.Wait() + + close(processChan) + + wgList.Wait() + + return processList +} + +func (p *Manager) ProcessAdd(nodeid string, config *app.Config, metadata map[string]interface{}) error { + node, err := p.NodeGet(nodeid) + if err != nil { + return fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessAdd(config, metadata) +} + +func (p *Manager) ProcessDelete(nodeid string, id app.ProcessID) error { + node, err := p.NodeGet(nodeid) + if err != nil { + return fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessDelete(id) +} + +func (p *Manager) ProcessUpdate(nodeid string, id app.ProcessID, config *app.Config, metadata map[string]interface{}) error { + node, err := p.NodeGet(nodeid) + if err != nil { + return fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessUpdate(id, config, metadata) +} + +func (p *Manager) ProcessCommand(nodeid string, id app.ProcessID, command string) error { + node, err := p.NodeGet(nodeid) + if err != nil { + return fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessCommand(id, command) +} + +func (p *Manager) ProcessProbe(nodeid string, id app.ProcessID) (api.Probe, error) { + node, err := p.NodeGet(nodeid) + if err != nil { + probe := api.Probe{ + Log: []string{fmt.Sprintf("the node %s where the process %s should reside on, doesn't exist", nodeid, id.String())}, + } + return probe, fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessProbe(id) +} + +func (p *Manager) ProcessProbeConfig(nodeid string, config *app.Config) (api.Probe, error) { + node, err := p.NodeGet(nodeid) + if err != nil { + probe := api.Probe{ + Log: []string{fmt.Sprintf("the node %s where the process config should be probed on, doesn't exist", nodeid)}, + } + return probe, fmt.Errorf("node not found: %w", err) + } + + return node.Core().ProcessProbeConfig(config) +} diff --git a/cluster/node/node.go b/cluster/node/node.go index a361edbe..536b9991 100644 --- a/cluster/node/node.go +++ b/cluster/node/node.go @@ -2,6 +2,7 @@ package node import ( "context" + "errors" "fmt" "net" "net/http" @@ -9,48 +10,36 @@ import ( "time" "github.com/datarhei/core/v16/cluster/client" - "github.com/datarhei/core/v16/cluster/proxy" "github.com/datarhei/core/v16/config" "github.com/datarhei/core/v16/ffmpeg/skills" "github.com/datarhei/core/v16/log" ) -type Node interface { - Stop() error - About() About - Version() string - IPs() []string - Status() (string, error) - LastContact() time.Time - Barrier(name string) (bool, error) +type Node struct { + id string + address string + ips []string + version string - CoreStatus() (string, error) - CoreEssentials() (string, *config.Config, error) - CoreConfig() (*config.Config, error) - CoreSkills() (skills.Skills, error) - CoreAPIAddress() (string, error) + node client.APIClient + nodeAbout About + nodeLastContact time.Time + nodeLastErr error + nodeLatency float64 - Proxy() proxy.Node -} + core *Core + coreAbout CoreAbout + coreLastContact time.Time + coreLastErr error + coreLatency float64 -type node struct { - client client.APIClient + compatibilityErr error - id string - address string - ips []string - version string - lastContact time.Time - lastContactErr error - lastCoreContact time.Time - lastCoreContactErr error - latency float64 - pingLock sync.RWMutex + config *config.Config + skills *skills.Skills - runLock sync.Mutex - cancelPing context.CancelFunc - - proxyNode proxy.Node + lock sync.RWMutex + cancel context.CancelFunc logger log.Logger } @@ -62,15 +51,20 @@ type Config struct { Logger log.Logger } -func New(config Config) Node { - n := &node{ +func New(config Config) *Node { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.MaxIdleConns = 10 + tr.IdleConnTimeout = 30 * time.Second + + n := &Node{ id: config.ID, address: config.Address, version: "0.0.0", - client: client.APIClient{ + node: client.APIClient{ Address: config.Address, Client: &http.Client{ - Timeout: 5 * time.Second, + Transport: tr, + Timeout: 5 * time.Second, }, }, logger: config.Logger, @@ -86,286 +80,499 @@ func New(config Config) Node { } } - if version, err := n.client.Version(); err == nil { + if version, err := n.node.Version(); err == nil { n.version = version } - n.start(n.id) + ctx, cancel := context.WithCancel(context.Background()) + n.cancel = cancel + + n.nodeLastErr = fmt.Errorf("not started yet") + n.coreLastErr = fmt.Errorf("not started yet") + + address, coreConfig, coreSkills, err := n.CoreEssentials() + n.config = coreConfig + n.skills = coreSkills + + n.core = NewCore(n.id, n.logger.WithComponent("ClusterCore").WithField("address", address)) + n.core.SetEssentials(address, coreConfig) + + n.coreLastErr = err + + go n.updateCore(ctx, 5*time.Second) + go n.ping(ctx, time.Second) + go n.pingCore(ctx, time.Second) return n } -func (n *node) start(id string) error { - n.runLock.Lock() - defer n.runLock.Unlock() +func (n *Node) Stop() error { - if n.cancelPing != nil { + n.lock.Lock() + + defer n.lock.Unlock() + + if n.cancel == nil { return nil } - ctx, cancel := context.WithCancel(context.Background()) - n.cancelPing = cancel + n.cancel() + n.cancel = nil - n.lastCoreContactErr = fmt.Errorf("not started yet") - n.lastContactErr = fmt.Errorf("not started yet") - - address, config, err := n.CoreEssentials() - n.proxyNode = proxy.NewNode(proxy.NodeConfig{ - ID: id, - Address: address, - Config: config, - Logger: n.logger.WithComponent("ClusterProxyNode").WithField("address", address), - }) - - n.lastCoreContactErr = err - - if err != nil { - go func(ctx context.Context) { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - address, config, err := n.CoreEssentials() - n.pingLock.Lock() - if err == nil { - n.proxyNode.SetEssentials(address, config) - n.lastCoreContactErr = nil - } else { - n.lastCoreContactErr = err - n.logger.Error().WithError(err).Log("Failed to retrieve core essentials") - } - n.pingLock.Unlock() - case <-ctx.Done(): - return - } - } - }(ctx) - } - - go n.ping(ctx) - go n.pingCore(ctx) - - return nil -} - -func (n *node) Stop() error { - n.runLock.Lock() - defer n.runLock.Unlock() - - if n.cancelPing == nil { - return nil - } - - n.proxyNode.Disconnect() - - n.cancelPing() - n.cancelPing = nil + n.core.Stop() return nil } var maxLastContact time.Duration = 5 * time.Second -type AboutCore struct { - Address string - State string - StateError error - Status string - Error error - CreatedAt time.Time - Uptime time.Duration - LastContact time.Duration - Latency time.Duration - Version string -} - type About struct { ID string Name string Version string Address string - Status string - LastContact time.Duration + State string + Uptime time.Duration + LastContact time.Time Latency time.Duration Error error - Core AboutCore - Resources proxy.NodeResources + Core CoreAbout + Resources Resources } -func (n *node) About() About { +type Resources struct { + IsThrottling bool // Whether this core is currently throttling + NCPU float64 // Number of CPU on this node + CPU float64 // Current CPU load, 0-100*ncpu + CPULimit float64 // Defined CPU load limit, 0-100*ncpu + Mem uint64 // Currently used memory in bytes + MemLimit uint64 // Defined memory limit in bytes + Error error // Last error +} + +func (n *Node) About() About { + n.lock.RLock() + defer n.lock.RUnlock() + a := About{ ID: n.id, - Version: n.Version(), + Version: n.version, Address: n.address, } - n.pingLock.RLock() - a.LastContact = time.Since(n.lastContact) - if a.LastContact > maxLastContact { - a.Status = "offline" + a.Name = n.coreAbout.Name + a.Error = n.nodeLastErr + a.LastContact = n.nodeLastContact + if time.Since(a.LastContact) > maxLastContact { + a.State = "offline" + } else if n.nodeLastErr != nil { + a.State = "degraded" + } else if n.compatibilityErr != nil { + a.State = "degraded" + a.Error = n.compatibilityErr } else { - a.Status = "online" + a.State = "online" } - a.Latency = time.Duration(n.latency * float64(time.Second)) - a.Error = n.lastContactErr + a.Latency = time.Duration(n.nodeLatency * float64(time.Second)) - coreError := n.lastCoreContactErr - n.pingLock.RUnlock() - - about := n.CoreAbout() - - a.Name = about.Name - a.Core.Address = about.Address - a.Core.State = about.State - a.Core.StateError = about.Error - a.Core.CreatedAt = about.CreatedAt - a.Core.Uptime = about.Uptime - a.Core.LastContact = time.Since(about.LastContact) - if a.Core.LastContact > maxLastContact { - a.Core.Status = "offline" - } else { - a.Core.Status = "online" + a.Resources = n.nodeAbout.Resources + if a.Resources.Error != nil { + a.Resources.CPU = a.Resources.CPULimit + a.Resources.Mem = a.Resources.MemLimit + a.Resources.IsThrottling = true + } + + a.Core = n.coreAbout + a.Core.Error = n.coreLastErr + a.Core.LastContact = n.coreLastContact + a.Core.Latency = time.Duration(n.coreLatency * float64(time.Second)) + + if a.State == "online" { + if a.Resources.Error != nil { + a.State = "degraded" + a.Error = a.Resources.Error + } + } + + if a.State == "online" { + if time.Since(a.Core.LastContact) > maxLastContact { + a.Core.State = "offline" + } else if n.coreLastErr != nil { + a.Core.State = "degraded" + a.Error = n.coreLastErr + } else { + a.Core.State = "online" + } + + a.State = a.Core.State } - a.Core.Error = coreError - a.Core.Latency = about.Latency - a.Core.Version = about.Version - a.Resources = about.Resources return a } -func (n *node) Version() string { - n.pingLock.RLock() - defer n.pingLock.RUnlock() +func (n *Node) Version() string { + n.lock.RLock() + defer n.lock.RUnlock() return n.version } -func (n *node) IPs() []string { +func (n *Node) IPs() []string { return n.ips } -func (n *node) Status() (string, error) { - n.pingLock.RLock() - defer n.pingLock.RUnlock() +func (n *Node) Status() (string, error) { + n.lock.RLock() + defer n.lock.RUnlock() - since := time.Since(n.lastContact) + since := time.Since(n.nodeLastContact) if since > maxLastContact { - return "offline", fmt.Errorf("the cluster API didn't respond for %s because: %w", since, n.lastContactErr) + return "offline", fmt.Errorf("the cluster API didn't respond for %s because: %w", since, n.nodeLastErr) } return "online", nil } -func (n *node) CoreStatus() (string, error) { - n.pingLock.RLock() - defer n.pingLock.RUnlock() +func (n *Node) CoreStatus() (string, error) { + n.lock.RLock() + defer n.lock.RUnlock() - since := time.Since(n.lastCoreContact) + since := time.Since(n.coreLastContact) if since > maxLastContact { - return "offline", fmt.Errorf("the core API didn't respond for %s because: %w", since, n.lastCoreContactErr) + return "offline", fmt.Errorf("the core API didn't respond for %s because: %w", since, n.coreLastErr) } return "online", nil } -func (n *node) LastContact() time.Time { - n.pingLock.RLock() - defer n.pingLock.RUnlock() - - return n.lastContact -} - -func (n *node) CoreEssentials() (string, *config.Config, error) { +func (n *Node) CoreEssentials() (string, *config.Config, *skills.Skills, error) { address, err := n.CoreAPIAddress() if err != nil { - return "", nil, err + return "", nil, nil, err } - config, err := n.CoreConfig() + config, err := n.CoreConfig(false) if err != nil { - return "", nil, err + return "", nil, nil, err } - return address, config, nil + skills, err := n.CoreSkills(false) + if err != nil { + return "", nil, nil, err + } + + return address, config, skills, nil } -func (n *node) CoreConfig() (*config.Config, error) { - return n.client.CoreConfig() +func (n *Node) CoreConfig(cached bool) (*config.Config, error) { + if cached { + n.lock.RLock() + config := n.config + n.lock.RUnlock() + + if config != nil { + return config, nil + } + } + + return n.node.CoreConfig() } -func (n *node) CoreSkills() (skills.Skills, error) { - return n.client.CoreSkills() +func (n *Node) CoreSkills(cached bool) (*skills.Skills, error) { + if cached { + n.lock.RLock() + skills := n.skills + n.lock.RUnlock() + + if skills != nil { + return skills, nil + } + } + + skills, err := n.node.CoreSkills() + + return &skills, err } -func (n *node) CoreAPIAddress() (string, error) { - return n.client.CoreAPIAddress() +func (n *Node) CoreAPIAddress() (string, error) { + return n.node.CoreAPIAddress() } -func (n *node) CoreAbout() proxy.NodeAbout { - return n.proxyNode.About() +func (n *Node) Barrier(name string) (bool, error) { + return n.node.Barrier(name) } -func (n *node) Barrier(name string) (bool, error) { - return n.client.Barrier(name) +func (n *Node) CoreAbout() CoreAbout { + return n.About().Core } -func (n *node) Proxy() proxy.Node { - return n.proxyNode +func (n *Node) Core() *Core { + return n.core } -func (n *node) ping(ctx context.Context) { - ticker := time.NewTicker(time.Second) +func (n *Node) CheckCompatibility(other *Node, skipSkillsCheck bool) { + err := n.checkCompatibility(other, skipSkillsCheck) + + n.lock.Lock() + n.compatibilityErr = err + n.lock.Unlock() +} + +func (n *Node) checkCompatibility(other *Node, skipSkillsCheck bool) error { + if other == nil { + return fmt.Errorf("no other node available to compare to") + } + + n.lock.RLock() + version := n.version + config := n.config + skills := n.skills + n.lock.RUnlock() + + otherVersion := other.Version() + otherConfig, _ := other.CoreConfig(true) + otherSkills, _ := other.CoreSkills(true) + + err := verifyVersion(version, otherVersion) + if err != nil { + return fmt.Errorf("version: %w", err) + } + + err = verifyConfig(config, otherConfig) + if err != nil { + return fmt.Errorf("config: %w", err) + } + + if !skipSkillsCheck { + err := verifySkills(skills, otherSkills) + if err != nil { + return fmt.Errorf("skills: %w", err) + } + } + + return nil +} + +func verifyVersion(local, other string) error { + if local != other { + return fmt.Errorf("actual: %s, expected %s", local, other) + } + + return nil +} + +func verifyConfig(local, other *config.Config) error { + if local == nil || other == nil { + return fmt.Errorf("config is not available") + } + + if local.Cluster.Enable != other.Cluster.Enable { + return fmt.Errorf("cluster.enable: actual: %v, expected: %v", local.Cluster.Enable, other.Cluster.Enable) + } + + if local.Cluster.ID != other.Cluster.ID { + return fmt.Errorf("cluster.id: actual: %v, expected: %v", local.Cluster.ID, other.Cluster.ID) + } + + if local.Cluster.SyncInterval != other.Cluster.SyncInterval { + return fmt.Errorf("cluster.sync_interval_sec: actual: %v, expected: %v", local.Cluster.SyncInterval, other.Cluster.SyncInterval) + } + + if local.Cluster.NodeRecoverTimeout != other.Cluster.NodeRecoverTimeout { + return fmt.Errorf("cluster.node_recover_timeout_sec: actual: %v, expected: %v", local.Cluster.NodeRecoverTimeout, other.Cluster.NodeRecoverTimeout) + } + + if local.Cluster.EmergencyLeaderTimeout != other.Cluster.EmergencyLeaderTimeout { + return fmt.Errorf("cluster.emergency_leader_timeout_sec: actual: %v, expected: %v", local.Cluster.EmergencyLeaderTimeout, other.Cluster.EmergencyLeaderTimeout) + } + + if local.Cluster.Debug.DisableFFmpegCheck != other.Cluster.Debug.DisableFFmpegCheck { + return fmt.Errorf("cluster.debug.disable_ffmpeg_check: actual: %v, expected: %v", local.Cluster.Debug.DisableFFmpegCheck, other.Cluster.Debug.DisableFFmpegCheck) + } + + if !local.API.Auth.Enable { + return fmt.Errorf("api.auth.enable must be enabled") + } + + if local.API.Auth.Enable != other.API.Auth.Enable { + return fmt.Errorf("api.auth.enable: actual: %v, expected: %v", local.API.Auth.Enable, other.API.Auth.Enable) + } + + if local.API.Auth.Username != other.API.Auth.Username { + return fmt.Errorf("api.auth.username: actual: %v, expected: %v", local.API.Auth.Username, other.API.Auth.Username) + } + + if local.API.Auth.Password != other.API.Auth.Password { + return fmt.Errorf("api.auth.password: actual: %v, expected: %v", local.API.Auth.Password, other.API.Auth.Password) + } + + if local.API.Auth.JWT.Secret != other.API.Auth.JWT.Secret { + return fmt.Errorf("api.auth.jwt.secret: actual: %v, expected: %v", local.API.Auth.JWT.Secret, other.API.Auth.JWT.Secret) + } + + if local.RTMP.Enable != other.RTMP.Enable { + return fmt.Errorf("rtmp.enable: actual: %v, expected: %v", local.RTMP.Enable, other.RTMP.Enable) + } + + if local.RTMP.Enable { + if local.RTMP.App != other.RTMP.App { + return fmt.Errorf("rtmp.app: actual: %v, expected: %v", local.RTMP.App, other.RTMP.App) + } + } + + if local.SRT.Enable != other.SRT.Enable { + return fmt.Errorf("srt.enable: actual: %v, expected: %v", local.SRT.Enable, other.SRT.Enable) + } + + if local.SRT.Enable { + if local.SRT.Passphrase != other.SRT.Passphrase { + return fmt.Errorf("srt.passphrase: actual: %v, expected: %v", local.SRT.Passphrase, other.SRT.Passphrase) + } + } + + if local.Resources.MaxCPUUsage == 0 || other.Resources.MaxCPUUsage == 0 { + return fmt.Errorf("resources.max_cpu_usage") + } + + if local.Resources.MaxMemoryUsage == 0 || other.Resources.MaxMemoryUsage == 0 { + return fmt.Errorf("resources.max_memory_usage") + } + + if local.TLS.Enable != other.TLS.Enable { + return fmt.Errorf("tls.enable: actual: %v, expected: %v", local.TLS.Enable, other.TLS.Enable) + } + + if local.TLS.Enable { + if local.TLS.Auto != other.TLS.Auto { + return fmt.Errorf("tls.auto: actual: %v, expected: %v", local.TLS.Auto, other.TLS.Auto) + } + + if len(local.Host.Name) == 0 || len(other.Host.Name) == 0 { + return fmt.Errorf("host.name must be set") + } + + if local.TLS.Auto { + if local.TLS.Email != other.TLS.Email { + return fmt.Errorf("tls.email: actual: %v, expected: %v", local.TLS.Email, other.TLS.Email) + } + + if local.TLS.Staging != other.TLS.Staging { + return fmt.Errorf("tls.staging: actual: %v, expected: %v", local.TLS.Staging, other.TLS.Staging) + } + + if local.TLS.Secret != other.TLS.Secret { + return fmt.Errorf("tls.secret: actual: %v, expected: %v", local.TLS.Secret, other.TLS.Secret) + } + } + } + + return nil +} + +func verifySkills(local, other *skills.Skills) error { + if local == nil || other == nil { + return fmt.Errorf("skills are not available") + } + + if err := local.Equal(*other); err != nil { + return err + } + + return nil +} + +func (n *Node) ping(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ticker.C: start := time.Now() - version, err := n.client.Version() - if err == nil { - n.pingLock.Lock() - n.version = version - n.lastContact = time.Now() - n.lastContactErr = nil - n.latency = n.latency*0.2 + time.Since(start).Seconds()*0.8 - n.pingLock.Unlock() - } else { - n.pingLock.Lock() - n.lastContactErr = err - n.pingLock.Unlock() + about, err := n.node.About() + n.lock.Lock() + if err == nil { + n.version = about.Version + n.nodeAbout = About{ + ID: about.ID, + Version: about.Version, + Address: about.Address, + Uptime: time.Since(about.StartedAt), + Error: err, + Resources: Resources{ + IsThrottling: about.Resources.IsThrottling, + NCPU: about.Resources.NCPU, + CPU: about.Resources.CPU, + CPULimit: about.Resources.CPULimit, + Mem: about.Resources.Mem, + MemLimit: about.Resources.MemLimit, + Error: nil, + }, + } + if len(about.Resources.Error) != 0 { + n.nodeAbout.Resources.Error = errors.New(about.Resources.Error) + } + n.nodeLastContact = time.Now() + n.nodeLastErr = nil + n.nodeLatency = n.nodeLatency*0.2 + time.Since(start).Seconds()*0.8 + } else { + n.nodeLastErr = err n.logger.Warn().WithError(err).Log("Failed to ping cluster API") } + n.lock.Unlock() case <-ctx.Done(): return } } } -func (n *node) pingCore(ctx context.Context) { - ticker := time.NewTicker(time.Second) +func (n *Node) updateCore(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ticker.C: - _, err := n.proxyNode.IsConnected() + address, config, skills, err := n.CoreEssentials() + n.lock.Lock() if err == nil { - n.pingLock.Lock() - n.lastCoreContact = time.Now() - n.lastCoreContactErr = nil - n.pingLock.Unlock() + n.config = config + n.skills = skills + n.core.SetEssentials(address, config) + n.coreLastErr = nil } else { - n.pingLock.Lock() - n.lastCoreContactErr = fmt.Errorf("not connected to core api: %w", err) - n.pingLock.Unlock() - - n.logger.Warn().WithError(err).Log("not connected to core API") + n.coreLastErr = err + n.logger.Error().WithError(err).Log("Failed to retrieve core essentials") } + n.lock.Unlock() + case <-ctx.Done(): + return + } + } +} + +func (n *Node) pingCore(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + start := time.Now() + about, err := n.core.About() + + n.lock.Lock() + if err == nil { + n.coreLastContact = time.Now() + n.coreLastErr = nil + n.coreAbout = about + n.coreLatency = n.coreLatency*0.2 + time.Since(start).Seconds()*0.8 + } else { + n.coreLastErr = fmt.Errorf("not connected to core api: %w", err) + } + n.lock.Unlock() case <-ctx.Done(): return } diff --git a/cluster/process.go b/cluster/process.go index 381ae4c9..85cab534 100644 --- a/cluster/process.go +++ b/cluster/process.go @@ -7,21 +7,9 @@ import ( "github.com/datarhei/core/v16/restream/app" ) -func (c *cluster) ListProcesses() []store.Process { - return c.store.ListProcesses() -} - -func (c *cluster) GetProcess(id app.ProcessID) (store.Process, error) { - return c.store.GetProcess(id) -} - -func (c *cluster) AddProcess(origin string, config *app.Config) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessAdd(origin string, config *app.Config) error { if !c.IsRaftLeader() { - return c.forwarder.AddProcess(origin, config) + return c.forwarder.ProcessAdd(origin, config) } cmd := &store.Command{ @@ -34,13 +22,9 @@ func (c *cluster) AddProcess(origin string, config *app.Config) error { return c.applyCommand(cmd) } -func (c *cluster) RemoveProcess(origin string, id app.ProcessID) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessRemove(origin string, id app.ProcessID) error { if !c.IsRaftLeader() { - return c.forwarder.RemoveProcess(origin, id) + return c.forwarder.ProcessRemove(origin, id) } cmd := &store.Command{ @@ -53,13 +37,9 @@ func (c *cluster) RemoveProcess(origin string, id app.ProcessID) error { return c.applyCommand(cmd) } -func (c *cluster) UpdateProcess(origin string, id app.ProcessID, config *app.Config) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessUpdate(origin string, id app.ProcessID, config *app.Config) error { if !c.IsRaftLeader() { - return c.forwarder.UpdateProcess(origin, id, config) + return c.forwarder.ProcessUpdate(origin, id, config) } cmd := &store.Command{ @@ -73,14 +53,10 @@ func (c *cluster) UpdateProcess(origin string, id app.ProcessID, config *app.Con return c.applyCommand(cmd) } -func (c *cluster) SetProcessCommand(origin string, id app.ProcessID, command string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessSetCommand(origin string, id app.ProcessID, command string) error { if command == "start" || command == "stop" { if !c.IsRaftLeader() { - return c.forwarder.SetProcessCommand(origin, id, command) + return c.forwarder.ProcessSetCommand(origin, id, command) } cmd := &store.Command{ @@ -94,21 +70,17 @@ func (c *cluster) SetProcessCommand(origin string, id app.ProcessID, command str return c.applyCommand(cmd) } - nodeid, err := c.proxy.FindNodeFromProcess(id) + nodeid, err := c.manager.ProcessFindNodeID(id) if err != nil { return fmt.Errorf("the process '%s' is not registered with any node: %w", id.String(), err) } - return c.proxy.CommandProcess(nodeid, id, command) + return c.manager.ProcessCommand(nodeid, id, command) } -func (c *cluster) RelocateProcesses(origin string, relocations map[app.ProcessID]string) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessesRelocate(origin string, relocations map[app.ProcessID]string) error { if !c.IsRaftLeader() { - return c.forwarder.RelocateProcesses(origin, relocations) + return c.forwarder.ProcessesRelocate(origin, relocations) } cmd := &store.Command{ @@ -121,13 +93,9 @@ func (c *cluster) RelocateProcesses(origin string, relocations map[app.ProcessID return c.applyCommand(cmd) } -func (c *cluster) SetProcessMetadata(origin string, id app.ProcessID, key string, data interface{}) error { - if ok, _ := c.IsDegraded(); ok { - return ErrDegraded - } - +func (c *cluster) ProcessSetMetadata(origin string, id app.ProcessID, key string, data interface{}) error { if !c.IsRaftLeader() { - return c.forwarder.SetProcessMetadata(origin, id, key, data) + return c.forwarder.ProcessSetMetadata(origin, id, key, data) } cmd := &store.Command{ @@ -142,12 +110,8 @@ func (c *cluster) SetProcessMetadata(origin string, id app.ProcessID, key string return c.applyCommand(cmd) } -func (c *cluster) GetProcessMetadata(origin string, id app.ProcessID, key string) (interface{}, error) { - if ok, _ := c.IsDegraded(); ok { - return nil, ErrDegraded - } - - p, err := c.store.GetProcess(id) +func (c *cluster) ProcessGetMetadata(origin string, id app.ProcessID, key string) (interface{}, error) { + p, err := c.store.ProcessGet(id) if err != nil { return nil, err } @@ -163,7 +127,3 @@ func (c *cluster) GetProcessMetadata(origin string, id app.ProcessID, key string return data, nil } - -func (c *cluster) GetProcessNodeMap() map[string]string { - return c.store.GetProcessNodeMap() -} diff --git a/cluster/proxy/node.go b/cluster/proxy/node.go deleted file mode 100644 index 393e9a60..00000000 --- a/cluster/proxy/node.go +++ /dev/null @@ -1,1179 +0,0 @@ -package proxy - -import ( - "context" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - "sync" - "time" - - "github.com/datarhei/core/v16/config" - "github.com/datarhei/core/v16/log" - "github.com/datarhei/core/v16/restream/app" - - client "github.com/datarhei/core-client-go/v16" - clientapi "github.com/datarhei/core-client-go/v16/api" -) - -type Node interface { - SetEssentials(address string, config *config.Config) - IsConnected() (bool, error) - Disconnect() - - AddProcess(config *app.Config, metadata map[string]interface{}) error - StartProcess(id app.ProcessID) error - StopProcess(id app.ProcessID) error - RestartProcess(id app.ProcessID) error - ReloadProcess(id app.ProcessID) error - DeleteProcess(id app.ProcessID) error - UpdateProcess(id app.ProcessID, config *app.Config, metadata map[string]interface{}) error - ProbeProcess(id app.ProcessID) (clientapi.Probe, error) - ProbeProcessConfig(config *app.Config) (clientapi.Probe, error) - - NodeReader -} - -type NodeReader interface { - About() NodeAbout - Version() NodeVersion - Resources() NodeResources - - ListFiles(storage, pattern string) ([]clientapi.FileInfo, error) - DeleteFile(storage, path string) error - PutFile(storage, path string, data io.Reader) error - ListResources() NodeFiles - - GetURL(prefix, path string) (*url.URL, error) - GetFile(prefix, path string, offset int64) (io.ReadCloser, error) - GetFileInfo(prefix, path string) (int64, time.Time, error) - - GetResourceInfo(prefix, path string) (int64, time.Time, error) - - ProcessList(ProcessListOptions) ([]clientapi.Process, error) - ProxyProcessList() ([]Process, error) -} - -type NodeFiles struct { - ID string - Files []string - LastUpdate time.Time -} - -type NodeResources struct { - IsThrottling bool // Whether this core is currently throttling - NCPU float64 // Number of CPU on this node - CPU float64 // Current CPU load, 0-100*ncpu - CPULimit float64 // Defined CPU load limit, 0-100*ncpu - Mem uint64 // Currently used memory in bytes - MemLimit uint64 // Defined memory limit in bytes - Error error // Last error -} - -type NodeAbout struct { - ID string - Name string - Address string - State string - Error error - CreatedAt time.Time - Uptime time.Duration - LastContact time.Time - Latency time.Duration - Resources NodeResources - Version string -} - -type NodeVersion struct { - Number string - Commit string - Branch string - Build time.Time - Arch string - Compiler string -} - -type nodeState string - -func (n nodeState) String() string { - return string(n) -} - -const ( - stateDisconnected nodeState = "disconnected" - stateConnected nodeState = "connected" -) - -var ErrNoPeer = errors.New("not connected to the core API: client not available") - -type node struct { - id string - address string - - peer client.RestClient - peerErr error - peerLock sync.RWMutex - peerWg sync.WaitGroup - peerAbout clientapi.About - disconnect context.CancelFunc - - lastContact time.Time - - resources struct { - throttling bool - ncpu float64 - cpu float64 - cpuLimit float64 - mem uint64 - memLimit uint64 - err error - } - - config *config.Config - - state nodeState - latency float64 // Seconds - stateLock sync.RWMutex - - secure bool - httpAddress *url.URL - hasRTMP bool - rtmpAddress *url.URL - hasSRT bool - srtAddress *url.URL - - logger log.Logger -} - -type NodeConfig struct { - ID string - Address string - Config *config.Config - - Logger log.Logger -} - -func NewNode(config NodeConfig) Node { - n := &node{ - id: config.ID, - address: config.Address, - config: config.Config, - state: stateDisconnected, - secure: strings.HasPrefix(config.Address, "https://"), - logger: config.Logger, - } - - if n.logger == nil { - n.logger = log.New("") - } - - n.resources.throttling = true - n.resources.cpu = 100 - n.resources.ncpu = 1 - n.resources.cpuLimit = 100 - n.resources.mem = 0 - n.resources.memLimit = 0 - n.resources.err = fmt.Errorf("not initialized") - - ctx, cancel := context.WithCancel(context.Background()) - n.disconnect = cancel - - err := n.connect() - if err != nil { - n.peerErr = err - } - - n.peerWg.Add(3) - - go func(ctx context.Context) { - // This tries to reconnect to the core API. If everything's - // fine, this is a no-op. - ticker := time.NewTicker(3 * time.Second) - defer ticker.Stop() - defer n.peerWg.Done() - - for { - select { - case <-ticker.C: - err := n.connect() - - n.peerLock.Lock() - n.peerErr = err - n.peerLock.Unlock() - case <-ctx.Done(): - return - } - } - }(ctx) - - go n.pingPeer(ctx, &n.peerWg) - go n.updateResources(ctx, &n.peerWg) - - return n -} - -func (n *node) SetEssentials(address string, config *config.Config) { - n.peerLock.Lock() - defer n.peerLock.Unlock() - - if address != n.address { - n.address = address - n.peer = nil // force reconnet - } - - if n.config == nil && config != nil { - n.config = config - n.peer = nil // force reconnect - } - - if n.config.UpdatedAt != config.UpdatedAt { - n.config = config - n.peer = nil // force reconnect - } -} - -func (n *node) connect() error { - n.peerLock.Lock() - defer n.peerLock.Unlock() - - if n.peer != nil && n.state != stateDisconnected { - return nil - } - - if len(n.address) == 0 { - return fmt.Errorf("no address provided") - } - - if n.config == nil { - return fmt.Errorf("config not available") - } - - u, err := url.Parse(n.address) - if err != nil { - return fmt.Errorf("invalid address (%s): %w", n.address, err) - } - - nodehost, _, err := net.SplitHostPort(u.Host) - if err != nil { - return fmt.Errorf("invalid address (%s): %w", u.Host, err) - } - - peer, err := client.New(client.Config{ - Address: u.String(), - Client: &http.Client{ - Timeout: 5 * time.Second, - }, - }) - if err != nil { - return fmt.Errorf("creating client failed (%s): %w", n.address, err) - } - - n.httpAddress = u - - if n.config.RTMP.Enable { - n.hasRTMP = true - n.rtmpAddress = &url.URL{} - n.rtmpAddress.Scheme = "rtmp" - - isHostIP := net.ParseIP(nodehost) != nil - - address := n.config.RTMP.Address - if n.secure && n.config.RTMP.EnableTLS && !isHostIP { - address = n.config.RTMP.AddressTLS - n.rtmpAddress.Scheme = "rtmps" - } - - host, port, err := net.SplitHostPort(address) - if err != nil { - return fmt.Errorf("invalid rtmp address '%s': %w", address, err) - } - - if len(host) == 0 { - n.rtmpAddress.Host = net.JoinHostPort(nodehost, port) - } else { - n.rtmpAddress.Host = net.JoinHostPort(host, port) - } - - n.rtmpAddress = n.rtmpAddress.JoinPath(n.config.RTMP.App) - } - - if n.config.SRT.Enable { - n.hasSRT = true - n.srtAddress = &url.URL{} - n.srtAddress.Scheme = "srt" - - host, port, err := net.SplitHostPort(n.config.SRT.Address) - if err != nil { - return fmt.Errorf("invalid srt address '%s': %w", n.config.SRT.Address, err) - } - - if len(host) == 0 { - n.srtAddress.Host = net.JoinHostPort(nodehost, port) - } else { - n.srtAddress.Host = net.JoinHostPort(host, port) - } - - v := url.Values{} - - v.Set("mode", "caller") - if len(n.config.SRT.Passphrase) != 0 { - v.Set("passphrase", n.config.SRT.Passphrase) - } - - n.srtAddress.RawQuery = v.Encode() - } - - n.peer = peer - - return nil -} - -func (n *node) IsConnected() (bool, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return false, ErrNoPeer - } - - if n.peerErr != nil { - return false, n.peerErr - } - - return true, nil -} - -func (n *node) Disconnect() { - n.peerLock.Lock() - if n.disconnect != nil { - n.disconnect() - n.disconnect = nil - } - n.peerLock.Unlock() - - n.peerWg.Wait() - - n.peerLock.Lock() - n.peer = nil - n.peerErr = fmt.Errorf("disconnected") - n.peerLock.Unlock() -} - -func (n *node) pingPeer(ctx context.Context, wg *sync.WaitGroup) { - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - defer wg.Done() - - for { - select { - case <-ticker.C: - about, latency, err := n.AboutPeer() - - n.peerLock.Lock() - n.peerErr = err - n.peerAbout = about - n.peerLock.Unlock() - - n.stateLock.Lock() - if err != nil { - n.state = stateDisconnected - - n.logger.Warn().WithError(err).Log("Failed to retrieve about") - } else { - n.lastContact = time.Now() - n.state = stateConnected - } - n.latency = n.latency*0.2 + latency.Seconds()*0.8 - n.stateLock.Unlock() - case <-ctx.Done(): - return - } - } -} - -func (n *node) updateResources(ctx context.Context, wg *sync.WaitGroup) { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - defer wg.Done() - - for { - select { - case <-ticker.C: - // Metrics - metrics, err := n.Metrics(clientapi.MetricsQuery{ - Metrics: []clientapi.MetricsQueryMetric{ - {Name: "cpu_ncpu"}, - {Name: "cpu_idle"}, - {Name: "cpu_limit"}, - {Name: "cpu_throttling"}, - {Name: "mem_total"}, - {Name: "mem_free"}, - {Name: "mem_limit"}, - {Name: "mem_throttling"}, - }, - }) - - if err != nil { - n.stateLock.Lock() - n.resources.throttling = true - n.resources.cpu = 100 - n.resources.ncpu = 1 - n.resources.cpuLimit = 100 - n.resources.mem = 0 - n.resources.memLimit = 0 - n.resources.err = err - n.stateLock.Unlock() - - n.logger.Warn().WithError(err).Log("Failed to retrieve metrics") - - continue - } - - cpu_ncpu := .0 - cpu_idle := .0 - cpu_limit := .0 - mem_total := uint64(0) - mem_free := uint64(0) - mem_limit := uint64(0) - throttling := .0 - - for _, x := range metrics.Metrics { - if x.Name == "cpu_idle" { - cpu_idle = x.Values[0].Value - } else if x.Name == "cpu_ncpu" { - cpu_ncpu = x.Values[0].Value - } else if x.Name == "cpu_limit" { - cpu_limit = x.Values[0].Value - } else if x.Name == "cpu_throttling" { - throttling += x.Values[0].Value - } else if x.Name == "mem_total" { - mem_total = uint64(x.Values[0].Value) - } else if x.Name == "mem_free" { - mem_free = uint64(x.Values[0].Value) - } else if x.Name == "mem_limit" { - mem_limit = uint64(x.Values[0].Value) - } else if x.Name == "mem_throttling" { - throttling += x.Values[0].Value - } - } - - n.stateLock.Lock() - if throttling > 0 { - n.resources.throttling = true - } else { - n.resources.throttling = false - } - n.resources.ncpu = cpu_ncpu - n.resources.cpu = (100 - cpu_idle) * cpu_ncpu - n.resources.cpuLimit = cpu_limit * cpu_ncpu - if mem_total != 0 { - n.resources.mem = mem_total - mem_free - n.resources.memLimit = mem_limit - } else { - n.resources.mem = 0 - n.resources.memLimit = 0 - } - n.resources.err = nil - n.stateLock.Unlock() - case <-ctx.Done(): - return - } - } -} - -func (n *node) Metrics(query clientapi.MetricsQuery) (clientapi.MetricsResponse, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return clientapi.MetricsResponse{}, ErrNoPeer - } - - return n.peer.Metrics(query) -} - -func (n *node) AboutPeer() (clientapi.About, time.Duration, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return clientapi.About{}, 0, ErrNoPeer - } - - start := time.Now() - - about, err := n.peer.About(false) - - return about, time.Since(start), err -} - -func (n *node) About() NodeAbout { - n.peerLock.RLock() - createdAt, err := time.Parse(time.RFC3339, n.peerAbout.CreatedAt) - if err != nil { - createdAt = time.Now() - } - name := n.peerAbout.Name - version := n.peerAbout.Version.Number - n.peerLock.RUnlock() - - n.stateLock.RLock() - defer n.stateLock.RUnlock() - - state := n.state - if time.Since(n.lastContact) > 3*time.Second { - state = stateDisconnected - } - - nodeAbout := NodeAbout{ - ID: n.id, - Name: name, - Address: n.address, - State: state.String(), - Error: n.peerErr, - CreatedAt: createdAt, - Uptime: time.Since(createdAt), - LastContact: n.lastContact, - Latency: time.Duration(n.latency * float64(time.Second)), - Resources: NodeResources{ - IsThrottling: n.resources.throttling, - NCPU: n.resources.ncpu, - CPU: n.resources.cpu, - CPULimit: n.resources.cpuLimit, - Mem: n.resources.mem, - MemLimit: n.resources.memLimit, - Error: n.resources.err, - }, - Version: version, - } - - if state == stateDisconnected { - nodeAbout.Uptime = 0 - nodeAbout.Latency = 0 - nodeAbout.Resources.IsThrottling = true - nodeAbout.Resources.NCPU = 1 - } - - return nodeAbout -} - -func (n *node) Resources() NodeResources { - about := n.About() - - return about.Resources -} - -func (n *node) Version() NodeVersion { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - build, err := time.Parse(time.RFC3339, n.peerAbout.Version.Build) - if err != nil { - build = time.Time{} - } - - version := NodeVersion{ - Number: n.peerAbout.Version.Number, - Commit: n.peerAbout.Version.Commit, - Branch: n.peerAbout.Version.Branch, - Build: build, - Arch: n.peerAbout.Version.Arch, - Compiler: n.peerAbout.Version.Compiler, - } - - return version -} - -func (n *node) ListResources() NodeFiles { - id := n.About().ID - - files := NodeFiles{ - ID: id, - LastUpdate: time.Now(), - } - - if ok, _ := n.IsConnected(); !ok { - return files - } - - files.Files = n.files() - - return files -} - -func (n *node) files() []string { - errorsChan := make(chan error, 8) - filesChan := make(chan string, 1024) - filesList := []string{} - errorList := []error{} - - wgList := sync.WaitGroup{} - wgList.Add(1) - - go func() { - defer wgList.Done() - - for file := range filesChan { - filesList = append(filesList, file) - } - - for err := range errorsChan { - errorList = append(errorList, err) - } - }() - - wg := sync.WaitGroup{} - wg.Add(2) - - go func(f chan<- string, e chan<- error) { - defer wg.Done() - - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - e <- ErrNoPeer - return - } - - files, err := n.peer.MemFSList("name", "asc") - if err != nil { - e <- err - return - } - - for _, file := range files { - f <- "mem:" + file.Name - } - }(filesChan, errorsChan) - - go func(f chan<- string, e chan<- error) { - defer wg.Done() - - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - e <- ErrNoPeer - return - } - - files, err := n.peer.DiskFSList("name", "asc") - if err != nil { - e <- err - return - } - - for _, file := range files { - f <- "disk:" + file.Name - } - }(filesChan, errorsChan) - - if n.hasRTMP { - wg.Add(1) - - go func(f chan<- string, e chan<- error) { - defer wg.Done() - - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - e <- ErrNoPeer - return - } - - files, err := n.peer.RTMPChannels() - if err != nil { - e <- err - return - } - - for _, file := range files { - f <- "rtmp:" + file.Name - } - }(filesChan, errorsChan) - } - - if n.hasSRT { - wg.Add(1) - - go func(f chan<- string, e chan<- error) { - defer wg.Done() - - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - e <- ErrNoPeer - return - } - - files, err := n.peer.SRTChannels() - if err != nil { - e <- err - return - } - - for _, file := range files { - f <- "srt:" + file.Name - } - }(filesChan, errorsChan) - } - - wg.Wait() - - close(filesChan) - close(errorsChan) - - wgList.Wait() - - return filesList -} - -func (n *node) ListFiles(storage, pattern string) ([]clientapi.FileInfo, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return nil, ErrNoPeer - } - - files, err := n.peer.FilesystemList(storage, pattern, "", "") - if err != nil { - return nil, err - } - - for i := range files { - files[i].CoreID = n.id - } - - return files, nil -} - -func (n *node) PutFile(storage, path string, data io.Reader) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.FilesystemAddFile(storage, path, data) -} - -func (n *node) DeleteFile(storage, path string) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.FilesystemDeleteFile(storage, path) -} - -func cloneURL(src *url.URL) *url.URL { - dst := &url.URL{ - Scheme: src.Scheme, - Opaque: src.Opaque, - User: nil, - Host: src.Host, - Path: src.Path, - RawPath: src.RawPath, - OmitHost: src.OmitHost, - ForceQuery: src.ForceQuery, - RawQuery: src.RawQuery, - Fragment: src.Fragment, - RawFragment: src.RawFragment, - } - - if src.User != nil { - username := src.User.Username() - password, ok := src.User.Password() - - if ok { - dst.User = url.UserPassword(username, password) - } else { - dst.User = url.User(username) - } - } - - return dst -} - -func (n *node) GetURL(prefix, resource string) (*url.URL, error) { - var u *url.URL - - if prefix == "mem" { - u = cloneURL(n.httpAddress) - u = u.JoinPath("memfs", resource) - } else if prefix == "disk" { - u = cloneURL(n.httpAddress) - u = u.JoinPath(resource) - } else if prefix == "rtmp" { - u = cloneURL(n.rtmpAddress) - u = u.JoinPath(resource) - } else if prefix == "srt" { - u = cloneURL(n.srtAddress) - } else { - return nil, fmt.Errorf("unknown prefix") - } - - return u, nil -} - -func (n *node) GetFile(prefix, path string, offset int64) (io.ReadCloser, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return nil, ErrNoPeer - } - - return n.peer.FilesystemGetFileOffset(prefix, path, offset) -} - -func (n *node) GetFileInfo(prefix, path string) (int64, time.Time, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return 0, time.Time{}, ErrNoPeer - } - - info, err := n.peer.FilesystemList(prefix, path, "", "") - if err != nil { - return 0, time.Time{}, fmt.Errorf("file not found: %w", err) - } - - if len(info) != 1 { - return 0, time.Time{}, fmt.Errorf("ambigous result") - } - - return info[0].Size, time.Unix(info[0].LastMod, 0), nil -} - -func (n *node) GetResourceInfo(prefix, path string) (int64, time.Time, error) { - if prefix == "disk" || prefix == "mem" { - return n.GetFileInfo(prefix, path) - } else if prefix == "rtmp" { - files, err := n.peer.RTMPChannels() - if err != nil { - return 0, time.Time{}, err - } - - for _, file := range files { - if path == file.Name { - return 0, time.Now(), nil - } - } - - return 0, time.Time{}, fmt.Errorf("resource not found") - } else if prefix == "srt" { - files, err := n.peer.SRTChannels() - if err != nil { - return 0, time.Time{}, err - } - - for _, file := range files { - if path == file.Name { - return 0, time.Now(), nil - } - } - - return 0, time.Time{}, fmt.Errorf("resource not found") - } - - return 0, time.Time{}, fmt.Errorf("unknown prefix: %s", prefix) -} - -func (n *node) ProcessList(options ProcessListOptions) ([]clientapi.Process, error) { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return nil, ErrNoPeer - } - - return n.peer.ProcessList(client.ProcessListOptions{ - ID: options.ID, - Filter: options.Filter, - Domain: options.Domain, - Reference: options.Reference, - IDPattern: options.IDPattern, - RefPattern: options.RefPattern, - OwnerPattern: options.OwnerPattern, - DomainPattern: options.DomainPattern, - }) -} - -func (n *node) ProxyProcessList() ([]Process, error) { - list, err := n.ProcessList(ProcessListOptions{ - Filter: []string{"config", "state", "metadata"}, - }) - if err != nil { - return nil, err - } - - nodeid := n.About().ID - - processes := []Process{} - - for _, p := range list { - if p.State == nil { - p.State = &clientapi.ProcessState{} - } - - if p.Config == nil { - p.Config = &clientapi.ProcessConfig{} - } - - process := Process{ - NodeID: nodeid, - Order: p.State.Order, - State: p.State.State, - Mem: p.State.Resources.Memory.Current, - CPU: p.State.Resources.CPU.Current, - Throttling: p.State.Resources.CPU.IsThrottling, - Runtime: time.Duration(p.State.Runtime) * time.Second, - UpdatedAt: time.Unix(p.UpdatedAt, 0), - Metadata: p.Metadata, - } - - cfg := &app.Config{ - ID: p.Config.ID, - Owner: p.Config.Owner, - Domain: p.Config.Domain, - Reference: p.Config.Reference, - Input: []app.ConfigIO{}, - Output: []app.ConfigIO{}, - Options: p.Config.Options, - Reconnect: p.Config.Reconnect, - ReconnectDelay: p.Config.ReconnectDelay, - Autostart: p.Config.Autostart, - StaleTimeout: p.Config.StaleTimeout, - Timeout: p.Config.Timeout, - Scheduler: p.Config.Scheduler, - LogPatterns: p.Config.LogPatterns, - LimitCPU: p.Config.Limits.CPU, - LimitMemory: p.Config.Limits.Memory * 1024 * 1024, - LimitWaitFor: p.Config.Limits.WaitFor, - } - - for _, d := range p.Config.Input { - cfg.Input = append(cfg.Input, app.ConfigIO{ - ID: d.ID, - Address: d.Address, - Options: d.Options, - }) - } - - for _, d := range p.Config.Output { - output := app.ConfigIO{ - ID: d.ID, - Address: d.Address, - Options: d.Options, - Cleanup: []app.ConfigIOCleanup{}, - } - - for _, c := range d.Cleanup { - output.Cleanup = append(output.Cleanup, app.ConfigIOCleanup{ - Pattern: c.Pattern, - MaxFiles: c.MaxFiles, - MaxFileAge: c.MaxFileAge, - PurgeOnDelete: c.PurgeOnDelete, - }) - } - - cfg.Output = append(cfg.Output, output) - } - - process.Config = cfg - - processes = append(processes, process) - } - - return processes, nil -} - -func (n *node) AddProcess(config *app.Config, metadata map[string]interface{}) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - cfg := convertConfig(config, metadata) - - return n.peer.ProcessAdd(cfg) -} - -func convertConfig(config *app.Config, metadata map[string]interface{}) clientapi.ProcessConfig { - cfg := clientapi.ProcessConfig{ - ID: config.ID, - Owner: config.Owner, - Domain: config.Domain, - Type: "ffmpeg", - Reference: config.Reference, - Input: []clientapi.ProcessConfigIO{}, - Output: []clientapi.ProcessConfigIO{}, - Options: config.Options, - Reconnect: config.Reconnect, - ReconnectDelay: config.ReconnectDelay, - Autostart: config.Autostart, - StaleTimeout: config.StaleTimeout, - Timeout: config.Timeout, - Scheduler: config.Scheduler, - LogPatterns: config.LogPatterns, - Limits: clientapi.ProcessConfigLimits{ - CPU: config.LimitCPU, - Memory: config.LimitMemory / 1024 / 1024, - WaitFor: config.LimitWaitFor, - }, - Metadata: metadata, - } - - for _, d := range config.Input { - cfg.Input = append(cfg.Input, clientapi.ProcessConfigIO{ - ID: d.ID, - Address: d.Address, - Options: d.Options, - }) - } - - for _, d := range config.Output { - output := clientapi.ProcessConfigIO{ - ID: d.ID, - Address: d.Address, - Options: d.Options, - Cleanup: []clientapi.ProcessConfigIOCleanup{}, - } - - for _, c := range d.Cleanup { - output.Cleanup = append(output.Cleanup, clientapi.ProcessConfigIOCleanup{ - Pattern: c.Pattern, - MaxFiles: c.MaxFiles, - MaxFileAge: c.MaxFileAge, - PurgeOnDelete: c.PurgeOnDelete, - }) - } - - cfg.Output = append(cfg.Output, output) - } - - return cfg -} - -func (n *node) StartProcess(id app.ProcessID) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.ProcessCommand(client.NewProcessID(id.ID, id.Domain), "start") -} - -func (n *node) StopProcess(id app.ProcessID) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.ProcessCommand(client.NewProcessID(id.ID, id.Domain), "stop") -} - -func (n *node) RestartProcess(id app.ProcessID) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.ProcessCommand(client.NewProcessID(id.ID, id.Domain), "restart") -} - -func (n *node) ReloadProcess(id app.ProcessID) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.ProcessCommand(client.NewProcessID(id.ID, id.Domain), "reload") -} - -func (n *node) DeleteProcess(id app.ProcessID) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - return n.peer.ProcessDelete(client.NewProcessID(id.ID, id.Domain)) -} - -func (n *node) UpdateProcess(id app.ProcessID, config *app.Config, metadata map[string]interface{}) error { - n.peerLock.RLock() - defer n.peerLock.RUnlock() - - if n.peer == nil { - return ErrNoPeer - } - - cfg := convertConfig(config, metadata) - - return n.peer.ProcessUpdate(client.NewProcessID(id.ID, id.Domain), cfg) -} - -func (n *node) ProbeProcess(id app.ProcessID) (clientapi.Probe, error) { - n.peerLock.RLock() - peer := n.peer - n.peerLock.RUnlock() - - if peer == nil { - probe := clientapi.Probe{ - Log: []string{fmt.Sprintf("the node %s where the process %s resides, is not connected", n.id, id.String())}, - } - return probe, ErrNoPeer - } - - probe, err := peer.ProcessProbe(client.NewProcessID(id.ID, id.Domain)) - - probe.Log = append([]string{fmt.Sprintf("probed on node: %s", n.id)}, probe.Log...) - - return probe, err -} - -func (n *node) ProbeProcessConfig(config *app.Config) (clientapi.Probe, error) { - n.peerLock.RLock() - peer := n.peer - n.peerLock.RUnlock() - - if peer == nil { - probe := clientapi.Probe{ - Log: []string{fmt.Sprintf("the node %s where the process config should be probed, is not connected", n.id)}, - } - return probe, ErrNoPeer - } - - cfg := convertConfig(config, nil) - - probe, err := peer.ProcessProbeConfig(cfg) - - probe.Log = append([]string{fmt.Sprintf("probed on node: %s", n.id)}, probe.Log...) - - return probe, err -} diff --git a/cluster/proxy/proxy.go b/cluster/proxy/proxy.go deleted file mode 100644 index 541f72e0..00000000 --- a/cluster/proxy/proxy.go +++ /dev/null @@ -1,620 +0,0 @@ -package proxy - -import ( - "errors" - "fmt" - "io" - "net/url" - "sync" - "time" - - "github.com/datarhei/core/v16/log" - "github.com/datarhei/core/v16/restream/app" - - clientapi "github.com/datarhei/core-client-go/v16/api" -) - -type Proxy interface { - Start() - Stop() - - AddNode(id string, node Node) (string, error) - RemoveNode(id string) error - - ProxyReader - Reader() ProxyReader - - AddProcess(nodeid string, config *app.Config, metadata map[string]interface{}) error - DeleteProcess(nodeid string, id app.ProcessID) error - UpdateProcess(nodeid string, id app.ProcessID, config *app.Config, metadata map[string]interface{}) error - CommandProcess(nodeid string, id app.ProcessID, command string) error -} - -type ProxyReader interface { - ListNodes() []NodeReader - GetNodeReader(id string) (NodeReader, error) - - FindNodeFromProcess(id app.ProcessID) (string, error) - FindNodeFromResources(nodeid string, cpu float64, memory uint64) string - - Resources() map[string]NodeResources - - ListProcesses(ProcessListOptions) []clientapi.Process - ListProxyProcesses() []Process - ProbeProcess(nodeid string, id app.ProcessID) (clientapi.Probe, error) - ProbeProcessConfig(nodeid string, config *app.Config) (clientapi.Probe, error) - - ListFiles(storage, pattern string) []clientapi.FileInfo - - GetURL(prefix, path string) (*url.URL, error) - GetFile(prefix, path string, offset int64) (io.ReadCloser, error) - GetFileInfo(prefix, path string) (int64, time.Time, error) -} - -type ProxyConfig struct { - ID string // ID of the node - - Logger log.Logger -} - -type proxy struct { - id string - - nodes map[string]Node // List of known nodes - nodesLock sync.RWMutex - - lock sync.RWMutex - running bool - - cache *Cache[string] - - logger log.Logger -} - -var ErrNodeNotFound = errors.New("node not found") - -func NewProxy(config ProxyConfig) (Proxy, error) { - p := &proxy{ - id: config.ID, - nodes: map[string]Node{}, - cache: NewCache[string](nil), - logger: config.Logger, - } - - if p.logger == nil { - p.logger = log.New("") - } - - return p, nil -} - -func (p *proxy) Start() { - p.lock.Lock() - defer p.lock.Unlock() - - if p.running { - return - } - - p.running = true - - p.logger.Debug().Log("Starting proxy") -} - -func (p *proxy) Stop() { - p.lock.Lock() - defer p.lock.Unlock() - - if !p.running { - return - } - - p.running = false - - p.logger.Debug().Log("Stopping proxy") - - p.nodes = map[string]Node{} -} - -func (p *proxy) Reader() ProxyReader { - return p -} - -func (p *proxy) Resources() map[string]NodeResources { - resources := map[string]NodeResources{} - - p.nodesLock.RLock() - defer p.nodesLock.RUnlock() - - for id, node := range p.nodes { - resources[id] = node.Resources() - } - - return resources -} - -func (p *proxy) AddNode(id string, node Node) (string, error) { - about := node.About() - - //if id != about.ID { - // return "", fmt.Errorf("the provided (%s) and retrieved (%s) ID's don't match", id, about.ID) - //} - - p.nodesLock.Lock() - defer p.nodesLock.Unlock() - - if n, ok := p.nodes[id]; ok { - n.Disconnect() - delete(p.nodes, id) - } - - p.nodes[id] = node - - p.logger.Info().WithFields(log.Fields{ - "address": about.Address, - "name": about.Name, - "id": id, - }).Log("Added node") - - return id, nil -} - -func (p *proxy) RemoveNode(id string) error { - p.nodesLock.Lock() - defer p.nodesLock.Unlock() - - node, ok := p.nodes[id] - if !ok { - return ErrNodeNotFound - } - - node.Disconnect() - - delete(p.nodes, id) - - p.logger.Info().WithFields(log.Fields{ - "id": id, - }).Log("Removed node") - - return nil -} - -func (p *proxy) ListNodes() []NodeReader { - list := []NodeReader{} - - p.nodesLock.RLock() - defer p.nodesLock.RUnlock() - - for _, node := range p.nodes { - list = append(list, node) - } - - return list -} - -func (p *proxy) GetNode(id string) (Node, error) { - p.nodesLock.RLock() - defer p.nodesLock.RUnlock() - - node, ok := p.nodes[id] - if !ok { - return nil, fmt.Errorf("node not found") - } - - return node, nil -} - -func (p *proxy) GetNodeReader(id string) (NodeReader, error) { - return p.GetNode(id) -} - -func (p *proxy) GetURL(prefix, path string) (*url.URL, error) { - logger := p.logger.WithFields(log.Fields{ - "path": path, - "prefix": prefix, - }) - - node, err := p.getNodeForFile(prefix, path) - if err != nil { - logger.Debug().WithError(err).Log("Unknown node") - return nil, fmt.Errorf("file not found: %w", err) - } - - url, err := node.GetURL(prefix, path) - if err != nil { - logger.Debug().Log("Invalid path") - return nil, fmt.Errorf("file not found") - } - - logger.Debug().WithField("url", url).Log("File cluster url") - - return url, nil -} - -func (p *proxy) GetFile(prefix, path string, offset int64) (io.ReadCloser, error) { - logger := p.logger.WithFields(log.Fields{ - "path": path, - "prefix": prefix, - }) - - node, err := p.getNodeForFile(prefix, path) - if err != nil { - logger.Debug().WithError(err).Log("File not available") - return nil, fmt.Errorf("file not found") - } - - data, err := node.GetFile(prefix, path, offset) - if err != nil { - logger.Debug().Log("Invalid path") - return nil, fmt.Errorf("file not found") - } - - logger.Debug().Log("File cluster path") - - return data, nil -} - -func (p *proxy) GetFileInfo(prefix, path string) (int64, time.Time, error) { - logger := p.logger.WithFields(log.Fields{ - "path": path, - "prefix": prefix, - }) - - node, err := p.getNodeForFile(prefix, path) - if err != nil { - logger.Debug().WithError(err).Log("File not available") - return 0, time.Time{}, fmt.Errorf("file not found") - } - - size, lastModified, err := node.GetFileInfo(prefix, path) - if err != nil { - logger.Debug().Log("Invalid path") - return 0, time.Time{}, fmt.Errorf("file not found") - } - - logger.Debug().Log("File cluster path") - - return size, lastModified, nil -} - -func (p *proxy) getNodeIDForFile(prefix, path string) (string, error) { - // this is only for mem and disk prefixes - nodesChan := make(chan string, 16) - nodeids := []string{} - - wgList := sync.WaitGroup{} - wgList.Add(1) - - go func() { - defer wgList.Done() - - for nodeid := range nodesChan { - if len(nodeid) == 0 { - continue - } - - nodeids = append(nodeids, nodeid) - } - }() - - wg := sync.WaitGroup{} - - p.nodesLock.RLock() - for id, node := range p.nodes { - wg.Add(1) - - go func(nodeid string, node Node, p chan<- string) { - defer wg.Done() - - _, _, err := node.GetResourceInfo(prefix, path) - if err != nil { - nodeid = "" - } - - p <- nodeid - }(id, node, nodesChan) - } - p.nodesLock.RUnlock() - - wg.Wait() - - close(nodesChan) - - wgList.Wait() - - if len(nodeids) == 0 { - return "", fmt.Errorf("file not found") - } - - return nodeids[0], nil -} - -func (p *proxy) getNodeForFile(prefix, path string) (Node, error) { - id, err := p.cache.Get(prefix + ":" + path) - if err == nil { - node, err := p.GetNode(id) - if err == nil { - return node, nil - } - } - - id, err = p.getNodeIDForFile(prefix, path) - if err != nil { - return nil, err - } - - p.cache.Put(prefix+":"+path, id, 5*time.Second) - - return p.GetNode(id) -} - -func (p *proxy) ListFiles(storage, pattern string) []clientapi.FileInfo { - filesChan := make(chan []clientapi.FileInfo, 64) - filesList := []clientapi.FileInfo{} - - wgList := sync.WaitGroup{} - wgList.Add(1) - - go func() { - defer wgList.Done() - - for list := range filesChan { - filesList = append(filesList, list...) - } - }() - - wg := sync.WaitGroup{} - - p.nodesLock.RLock() - for _, node := range p.nodes { - wg.Add(1) - - go func(node Node, p chan<- []clientapi.FileInfo) { - defer wg.Done() - - files, err := node.ListFiles(storage, pattern) - if err != nil { - return - } - - p <- files - }(node, filesChan) - } - p.nodesLock.RUnlock() - - wg.Wait() - - close(filesChan) - - wgList.Wait() - - return filesList -} - -type Process struct { - NodeID string - Order string - State string - CPU float64 // Current CPU load of this process, 0-100*ncpu - Mem uint64 // Currently consumed memory of this process in bytes - Throttling bool - Runtime time.Duration - UpdatedAt time.Time - Config *app.Config - Metadata map[string]interface{} -} - -type ProcessListOptions struct { - ID []string - Filter []string - Domain string - Reference string - IDPattern string - RefPattern string - OwnerPattern string - DomainPattern string -} - -func (p *proxy) ListProxyProcesses() []Process { - processChan := make(chan []Process, 64) - processList := []Process{} - - wgList := sync.WaitGroup{} - wgList.Add(1) - - go func() { - defer wgList.Done() - - for list := range processChan { - processList = append(processList, list...) - } - }() - - wg := sync.WaitGroup{} - - p.nodesLock.RLock() - for _, node := range p.nodes { - wg.Add(1) - - go func(node Node, p chan<- []Process) { - defer wg.Done() - - processes, err := node.ProxyProcessList() - if err != nil { - return - } - - p <- processes - }(node, processChan) - } - p.nodesLock.RUnlock() - - wg.Wait() - - close(processChan) - - wgList.Wait() - - return processList -} - -func (p *proxy) FindNodeFromProcess(id app.ProcessID) (string, error) { - procs := p.ListProxyProcesses() - nodeid := "" - - for _, p := range procs { - if p.Config.ProcessID() != id { - continue - } - - nodeid = p.NodeID - - break - } - - if len(nodeid) == 0 { - return "", fmt.Errorf("the process '%s' is not registered with any node", id.String()) - } - - return nodeid, nil -} - -func (p *proxy) FindNodeFromResources(nodeid string, cpu float64, memory uint64) string { - p.nodesLock.RLock() - defer p.nodesLock.RUnlock() - - if len(nodeid) != 0 { - node, ok := p.nodes[nodeid] - if ok { - r := node.Resources() - if r.CPU+cpu <= r.CPULimit && r.Mem+memory <= r.MemLimit && !r.IsThrottling { - return nodeid - } - } - } - - for nodeid, node := range p.nodes { - r := node.Resources() - if r.CPU+cpu <= r.CPULimit && r.Mem+memory <= r.MemLimit && !r.IsThrottling { - return nodeid - } - } - - return "" -} - -func (p *proxy) ListProcesses(options ProcessListOptions) []clientapi.Process { - processChan := make(chan []clientapi.Process, 64) - processList := []clientapi.Process{} - - wgList := sync.WaitGroup{} - wgList.Add(1) - - go func() { - defer wgList.Done() - - for list := range processChan { - processList = append(processList, list...) - } - }() - - wg := sync.WaitGroup{} - - p.nodesLock.RLock() - for _, node := range p.nodes { - wg.Add(1) - - go func(node Node, p chan<- []clientapi.Process) { - defer wg.Done() - - processes, err := node.ProcessList(options) - if err != nil { - return - } - - p <- processes - }(node, processChan) - } - p.nodesLock.RUnlock() - - wg.Wait() - - close(processChan) - - wgList.Wait() - - return processList -} - -func (p *proxy) AddProcess(nodeid string, config *app.Config, metadata map[string]interface{}) error { - node, err := p.GetNode(nodeid) - if err != nil { - return fmt.Errorf("node not found: %w", err) - } - - return node.AddProcess(config, metadata) -} - -func (p *proxy) DeleteProcess(nodeid string, id app.ProcessID) error { - node, err := p.GetNode(nodeid) - if err != nil { - return fmt.Errorf("node not found: %w", err) - } - - return node.DeleteProcess(id) -} - -func (p *proxy) UpdateProcess(nodeid string, id app.ProcessID, config *app.Config, metadata map[string]interface{}) error { - node, err := p.GetNode(nodeid) - if err != nil { - return fmt.Errorf("node not found: %w", err) - } - - return node.UpdateProcess(id, config, metadata) -} - -func (p *proxy) CommandProcess(nodeid string, id app.ProcessID, command string) error { - node, err := p.GetNode(nodeid) - if err != nil { - return fmt.Errorf("node not found: %w", err) - } - - switch command { - case "start": - err = node.StartProcess(id) - case "stop": - err = node.StopProcess(id) - case "restart": - err = node.RestartProcess(id) - case "reload": - err = node.ReloadProcess(id) - default: - err = fmt.Errorf("unknown command: %s", command) - } - - return err -} - -func (p *proxy) ProbeProcess(nodeid string, id app.ProcessID) (clientapi.Probe, error) { - node, err := p.GetNode(nodeid) - if err != nil { - probe := clientapi.Probe{ - Log: []string{fmt.Sprintf("the node %s where the process %s should reside on, doesn't exist", nodeid, id.String())}, - } - return probe, fmt.Errorf("node not found: %w", err) - } - - return node.ProbeProcess(id) -} - -func (p *proxy) ProbeProcessConfig(nodeid string, config *app.Config) (clientapi.Probe, error) { - node, err := p.GetNode(nodeid) - if err != nil { - probe := clientapi.Probe{ - Log: []string{fmt.Sprintf("the node %s where the process config should be probed on, doesn't exist", nodeid)}, - } - return probe, fmt.Errorf("node not found: %w", err) - } - - return node.ProbeProcessConfig(config) -} diff --git a/cluster/resources.go b/cluster/resources.go new file mode 100644 index 00000000..2b5bb2c9 --- /dev/null +++ b/cluster/resources.go @@ -0,0 +1,123 @@ +package cluster + +import ( + "sort" + + "github.com/datarhei/core/v16/cluster/node" +) + +type resourcePlanner struct { + nodes map[string]node.Resources + blocked map[string]struct{} +} + +func NewResourcePlanner(nodes map[string]node.About) *resourcePlanner { + r := &resourcePlanner{ + nodes: map[string]node.Resources{}, + blocked: map[string]struct{}{}, + } + + for nodeid, about := range nodes { + r.nodes[nodeid] = about.Resources + if about.State != "online" { + r.blocked[nodeid] = struct{}{} + } + } + + return r +} + +func (r *resourcePlanner) Throttling(nodeid string, throttling bool) { + res, hasNode := r.nodes[nodeid] + if !hasNode { + return + } + + res.IsThrottling = throttling + + r.nodes[nodeid] = res +} + +// HasNodeEnough returns whether a node has enough resources available for the +// requested cpu and memory consumption. +func (r *resourcePlanner) HasNodeEnough(nodeid string, cpu float64, mem uint64) bool { + res, hasNode := r.nodes[nodeid] + if !hasNode { + return false + } + + if _, hasNode := r.blocked[nodeid]; hasNode { + return false + } + + if res.Error == nil && res.CPU+cpu < res.CPULimit && res.Mem+mem < res.MemLimit && !res.IsThrottling { + return true + } + + return false +} + +// FindBestNodes returns an array of nodeids that can fit the requested cpu and memory requirements. If no +// such node is available, an empty array is returned. The array is sorted by the most suitable node first. +func (r *resourcePlanner) FindBestNodes(cpu float64, mem uint64) []string { + nodes := []string{} + + for id := range r.nodes { + if r.HasNodeEnough(id, cpu, mem) { + nodes = append(nodes, id) + } + } + + sort.SliceStable(nodes, func(i, j int) bool { + nodeA, nodeB := nodes[i], nodes[j] + + if r.nodes[nodeA].CPU != r.nodes[nodeB].CPU { + return r.nodes[nodeA].CPU < r.nodes[nodeB].CPU + } + + return r.nodes[nodeA].Mem <= r.nodes[nodeB].Mem + }) + + return nodes +} + +// Add adds the resources of the node according to the cpu and memory utilization. +func (r *resourcePlanner) Add(nodeid string, cpu float64, mem uint64) { + res, hasRes := r.nodes[nodeid] + if !hasRes { + return + } + + res.CPU += cpu + res.Mem += mem + r.nodes[nodeid] = res +} + +// Remove subtracts the resources from the node according to the cpu and memory utilization. +func (r *resourcePlanner) Remove(nodeid string, cpu float64, mem uint64) { + res, hasRes := r.nodes[nodeid] + if !hasRes { + return + } + + res.CPU -= cpu + if res.CPU < 0 { + res.CPU = 0 + } + if mem >= res.Mem { + res.Mem = 0 + } else { + res.Mem -= mem + } + r.nodes[nodeid] = res +} + +// Move adjusts the resources from the target and source node according to the cpu and memory utilization. +func (r *resourcePlanner) Move(target, source string, cpu float64, mem uint64) { + r.Add(target, cpu, mem) + r.Remove(source, cpu, mem) +} + +func (r *resourcePlanner) Map() map[string]node.Resources { + return r.nodes +} diff --git a/cluster/store/errors.go b/cluster/store/errors.go new file mode 100644 index 00000000..51d01159 --- /dev/null +++ b/cluster/store/errors.go @@ -0,0 +1,6 @@ +package store + +import "errors" + +var ErrNotFound = errors.New("") +var ErrBadRequest = errors.New("") diff --git a/cluster/store/identity.go b/cluster/store/identity.go index 17498745..020d9527 100644 --- a/cluster/store/identity.go +++ b/cluster/store/identity.go @@ -11,7 +11,7 @@ func (s *store) addIdentity(cmd CommandAddIdentity) error { err := s.data.Users.userlist.Add(cmd.Identity) if err != nil { - return fmt.Errorf("the identity with the name '%s' already exists", cmd.Identity.Name) + return fmt.Errorf("the identity with the name '%s' already exists%w", cmd.Identity.Name, ErrBadRequest) } now := time.Now() @@ -30,17 +30,17 @@ func (s *store) updateIdentity(cmd CommandUpdateIdentity) error { defer s.lock.Unlock() if cmd.Name == "$anon" { - return fmt.Errorf("the identity with the name '%s' can't be updated", cmd.Name) + return fmt.Errorf("the identity with the name '%s' can't be updated%w", cmd.Name, ErrBadRequest) } oldUser, err := s.data.Users.userlist.Get(cmd.Name) if err != nil { - return fmt.Errorf("the identity with the name '%s' doesn't exist", cmd.Name) + return fmt.Errorf("the identity with the name '%s' doesn't exist%w", cmd.Name, ErrNotFound) } o, ok := s.data.Users.Users[oldUser.Name] if !ok { - return fmt.Errorf("the identity with the name '%s' doesn't exist", cmd.Name) + return fmt.Errorf("the identity with the name '%s' doesn't exist%w", cmd.Name, ErrNotFound) } err = s.data.Users.userlist.Update(cmd.Name, cmd.Identity) @@ -50,7 +50,7 @@ func (s *store) updateIdentity(cmd CommandUpdateIdentity) error { user, err := s.data.Users.userlist.Get(cmd.Identity.Name) if err != nil { - return fmt.Errorf("the identity with the name '%s' doesn't exist", cmd.Identity.Name) + return fmt.Errorf("the identity with the name '%s' doesn't exist%w", cmd.Identity.Name, ErrNotFound) } now := time.Now() @@ -89,7 +89,7 @@ func (s *store) removeIdentity(cmd CommandRemoveIdentity) error { return nil } -func (s *store) ListUsers() Users { +func (s *store) IAMIdentityList() Users { s.lock.RLock() defer s.lock.RUnlock() @@ -104,7 +104,7 @@ func (s *store) ListUsers() Users { return u } -func (s *store) GetUser(name string) Users { +func (s *store) IAMIdentityGet(name string) Users { s.lock.RLock() defer s.lock.RUnlock() diff --git a/cluster/store/identity_test.go b/cluster/store/identity_test.go index e3d82061..0253040e 100644 --- a/cluster/store/identity_test.go +++ b/cluster/store/identity_test.go @@ -49,7 +49,7 @@ func TestAddIdentity(t *testing.T) { require.Equal(t, 1, len(s.data.Users.Users)) require.Equal(t, 0, len(s.data.Policies.Policies)) - u := s.GetUser("foobar") + u := s.IAMIdentityGet("foobar") require.Equal(t, 1, len(u.Users)) user := u.Users[0] @@ -254,7 +254,7 @@ func TestUpdateIdentity(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(s.data.Users.Users)) - foobar := s.GetUser("foobar1").Users[0] + foobar := s.IAMIdentityGet("foobar1").Users[0] require.True(t, foobar.CreatedAt.Equal(foobar.UpdatedAt)) require.False(t, time.Time{}.Equal(foobar.CreatedAt)) @@ -287,13 +287,13 @@ func TestUpdateIdentity(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(s.data.Users.Users)) - u := s.GetUser("foobar1") + u := s.IAMIdentityGet("foobar1") require.Empty(t, u.Users) - u = s.GetUser("foobar2") + u = s.IAMIdentityGet("foobar2") require.NotEmpty(t, u.Users) - u = s.GetUser("foobaz") + u = s.IAMIdentityGet("foobaz") require.NotEmpty(t, u.Users) require.True(t, u.Users[0].CreatedAt.Equal(foobar.CreatedAt)) @@ -367,22 +367,22 @@ func TestUpdateIdentityWithAlias(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(s.data.Users.Users)) - u := s.GetUser("foobar1") + u := s.IAMIdentityGet("foobar1") require.Empty(t, u.Users) - u = s.GetUser("fooalias1") + u = s.IAMIdentityGet("fooalias1") require.Empty(t, u.Users) - u = s.GetUser("foobar2") + u = s.IAMIdentityGet("foobar2") require.NotEmpty(t, u.Users) - u = s.GetUser("fooalias2") + u = s.IAMIdentityGet("fooalias2") require.NotEmpty(t, u.Users) - u = s.GetUser("foobaz") + u = s.IAMIdentityGet("foobaz") require.NotEmpty(t, u.Users) - u = s.GetUser("fooalias") + u = s.IAMIdentityGet("fooalias") require.NotEmpty(t, u.Users) } diff --git a/cluster/store/kvs.go b/cluster/store/kvs.go index 37bcb682..6e79d6f7 100644 --- a/cluster/store/kvs.go +++ b/cluster/store/kvs.go @@ -1,6 +1,7 @@ package store import ( + "errors" "io/fs" "strings" "time" @@ -25,7 +26,7 @@ func (s *store) unsetKV(cmd CommandUnsetKV) error { defer s.lock.Unlock() if _, ok := s.data.KVS[cmd.Key]; !ok { - return fs.ErrNotExist + return errors.Join(fs.ErrNotExist, ErrNotFound) } delete(s.data.KVS, cmd.Key) @@ -33,7 +34,7 @@ func (s *store) unsetKV(cmd CommandUnsetKV) error { return nil } -func (s *store) ListKVS(prefix string) map[string]Value { +func (s *store) KVSList(prefix string) map[string]Value { s.lock.RLock() defer s.lock.RUnlock() @@ -50,13 +51,13 @@ func (s *store) ListKVS(prefix string) map[string]Value { return m } -func (s *store) GetFromKVS(key string) (Value, error) { +func (s *store) KVSGetValue(key string) (Value, error) { s.lock.RLock() defer s.lock.RUnlock() value, ok := s.data.KVS[key] if !ok { - return Value{}, fs.ErrNotExist + return Value{}, errors.Join(fs.ErrNotExist, ErrNotFound) } return value, nil diff --git a/cluster/store/kvs_test.go b/cluster/store/kvs_test.go index 1b560d4c..e1f62af0 100644 --- a/cluster/store/kvs_test.go +++ b/cluster/store/kvs_test.go @@ -34,7 +34,7 @@ func TestSetKV(t *testing.T) { }) require.NoError(t, err) - value, err := s.GetFromKVS("foo") + value, err := s.KVSGetValue("foo") require.NoError(t, err) require.Equal(t, "bar", value.Value) @@ -46,7 +46,7 @@ func TestSetKV(t *testing.T) { }) require.NoError(t, err) - value, err = s.GetFromKVS("foo") + value, err = s.KVSGetValue("foo") require.NoError(t, err) require.Equal(t, "baz", value.Value) require.Greater(t, value.UpdatedAt, updatedAt) @@ -86,7 +86,7 @@ func TestUnsetKVCommand(t *testing.T) { }, }) require.Error(t, err) - require.Equal(t, fs.ErrNotExist, err) + require.ErrorIs(t, err, fs.ErrNotExist) } func TestUnsetKV(t *testing.T) { @@ -99,7 +99,7 @@ func TestUnsetKV(t *testing.T) { }) require.NoError(t, err) - _, err = s.GetFromKVS("foo") + _, err = s.KVSGetValue("foo") require.NoError(t, err) err = s.unsetKV(CommandUnsetKV{ @@ -107,7 +107,7 @@ func TestUnsetKV(t *testing.T) { }) require.NoError(t, err) - _, err = s.GetFromKVS("foo") + _, err = s.KVSGetValue("foo") require.Error(t, err) - require.Equal(t, fs.ErrNotExist, err) + require.ErrorIs(t, err, fs.ErrNotExist) } diff --git a/cluster/store/lock.go b/cluster/store/lock.go index 9ed1be4c..3c26a948 100644 --- a/cluster/store/lock.go +++ b/cluster/store/lock.go @@ -13,7 +13,7 @@ func (s *store) createLock(cmd CommandCreateLock) error { if ok { if time.Now().Before(validUntil) { - return fmt.Errorf("the lock with the ID '%s' already exists", cmd.Name) + return fmt.Errorf("the lock with the ID '%s' already exists%w", cmd.Name, ErrBadRequest) } } @@ -51,7 +51,7 @@ func (s *store) clearLocks(_ CommandClearLocks) error { return nil } -func (s *store) HasLock(name string) bool { +func (s *store) LockHasLock(name string) bool { s.lock.RLock() defer s.lock.RUnlock() @@ -60,7 +60,7 @@ func (s *store) HasLock(name string) bool { return ok } -func (s *store) ListLocks() map[string]time.Time { +func (s *store) LockList() map[string]time.Time { s.lock.RLock() defer s.lock.RUnlock() diff --git a/cluster/store/node.go b/cluster/store/node.go index a362e204..d5d93db3 100644 --- a/cluster/store/node.go +++ b/cluster/store/node.go @@ -21,7 +21,7 @@ func (s *store) setNodeState(cmd CommandSetNodeState) error { return nil } -func (s *store) ListNodes() map[string]Node { +func (s *store) NodeList() map[string]Node { s.lock.RLock() defer s.lock.RUnlock() diff --git a/cluster/store/policy.go b/cluster/store/policy.go index b3d5fe29..2060cf7f 100644 --- a/cluster/store/policy.go +++ b/cluster/store/policy.go @@ -16,12 +16,12 @@ func (s *store) setPolicies(cmd CommandSetPolicies) error { if cmd.Name != "$anon" { user, err := s.data.Users.userlist.Get(cmd.Name) if err != nil { - return fmt.Errorf("the identity with the name '%s' doesn't exist", cmd.Name) + return fmt.Errorf("unknown identity %s%w", cmd.Name, ErrNotFound) } u, ok := s.data.Users.Users[user.Name] if !ok { - return fmt.Errorf("the identity with the name '%s' doesn't exist", cmd.Name) + return fmt.Errorf("unknown identity %s%w", cmd.Name, ErrNotFound) } u.UpdatedAt = now @@ -45,7 +45,7 @@ func (s *store) setPolicies(cmd CommandSetPolicies) error { return nil } -func (s *store) ListPolicies() Policies { +func (s *store) IAMPolicyList() Policies { s.lock.RLock() defer s.lock.RUnlock() @@ -60,7 +60,7 @@ func (s *store) ListPolicies() Policies { return p } -func (s *store) ListUserPolicies(name string) Policies { +func (s *store) IAMIdentityPolicyList(name string) Policies { s.lock.RLock() defer s.lock.RUnlock() diff --git a/cluster/store/policy_test.go b/cluster/store/policy_test.go index 5c0dd3a7..dde9cfc3 100644 --- a/cluster/store/policy_test.go +++ b/cluster/store/policy_test.go @@ -88,7 +88,7 @@ func TestSetPolicies(t *testing.T) { require.Equal(t, 1, len(s.data.Users.Users)) require.Equal(t, 0, len(s.data.Policies.Policies)) - users := s.GetUser("foobar") + users := s.IAMIdentityGet("foobar") require.NotEmpty(t, users.Users) updatedAt := users.Users[0].UpdatedAt @@ -101,7 +101,7 @@ func TestSetPolicies(t *testing.T) { require.Equal(t, 1, len(s.data.Users.Users)) require.Equal(t, 2, len(s.data.Policies.Policies["foobar"])) - users = s.GetUser("foobar") + users = s.IAMIdentityGet("foobar") require.NotEmpty(t, users.Users) require.False(t, updatedAt.Equal(users.Users[0].UpdatedAt)) diff --git a/cluster/store/process.go b/cluster/store/process.go index 55e905ca..f14d4c95 100644 --- a/cluster/store/process.go +++ b/cluster/store/process.go @@ -14,12 +14,12 @@ func (s *store) addProcess(cmd CommandAddProcess) error { id := cmd.Config.ProcessID().String() if cmd.Config.LimitCPU <= 0 || cmd.Config.LimitMemory <= 0 { - return fmt.Errorf("the process with the ID '%s' must have limits defined", id) + return fmt.Errorf("the process with the ID '%s' must have limits defined%w", id, ErrBadRequest) } _, ok := s.data.Process[id] if ok { - return fmt.Errorf("the process with the ID '%s' already exists", id) + return fmt.Errorf("the process with the ID '%s' already exists%w", id, ErrBadRequest) } order := "stop" @@ -48,7 +48,7 @@ func (s *store) removeProcess(cmd CommandRemoveProcess) error { _, ok := s.data.Process[id] if !ok { - return fmt.Errorf("the process with the ID '%s' doesn't exist", id) + return fmt.Errorf("the process with the ID '%s' doesn't exist%w", id, ErrNotFound) } delete(s.data.Process, id) @@ -64,12 +64,12 @@ func (s *store) updateProcess(cmd CommandUpdateProcess) error { dstid := cmd.Config.ProcessID().String() if cmd.Config.LimitCPU <= 0 || cmd.Config.LimitMemory <= 0 { - return fmt.Errorf("the process with the ID '%s' must have limits defined", dstid) + return fmt.Errorf("the process with the ID '%s' must have limits defined%w", dstid, ErrBadRequest) } p, ok := s.data.Process[srcid] if !ok { - return fmt.Errorf("the process with the ID '%s' doesn't exists", srcid) + return fmt.Errorf("the process with the ID '%s' doesn't exists%w", srcid, ErrNotFound) } if p.Config.Equal(cmd.Config) { @@ -87,7 +87,7 @@ func (s *store) updateProcess(cmd CommandUpdateProcess) error { _, ok = s.data.Process[dstid] if ok { - return fmt.Errorf("the process with the ID '%s' already exists", dstid) + return fmt.Errorf("the process with the ID '%s' already exists%w", dstid, ErrBadRequest) } now := time.Now() @@ -134,7 +134,7 @@ func (s *store) setProcessOrder(cmd CommandSetProcessOrder) error { p, ok := s.data.Process[id] if !ok { - return fmt.Errorf("the process with the ID '%s' doesn't exists", cmd.ID) + return fmt.Errorf("the process with the ID '%s' doesn't exists%w", cmd.ID, ErrNotFound) } p.Order = cmd.Order @@ -153,7 +153,7 @@ func (s *store) setProcessMetadata(cmd CommandSetProcessMetadata) error { p, ok := s.data.Process[id] if !ok { - return fmt.Errorf("the process with the ID '%s' doesn't exists", cmd.ID) + return fmt.Errorf("the process with the ID '%s' doesn't exists%w", cmd.ID, ErrNotFound) } if p.Metadata == nil { @@ -180,7 +180,7 @@ func (s *store) setProcessError(cmd CommandSetProcessError) error { p, ok := s.data.Process[id] if !ok { - return fmt.Errorf("the process with the ID '%s' doesn't exists", cmd.ID) + return fmt.Errorf("the process with the ID '%s' doesn't exists%w", cmd.ID, ErrNotFound) } p.Error = cmd.Error @@ -199,7 +199,7 @@ func (s *store) setProcessNodeMap(cmd CommandSetProcessNodeMap) error { return nil } -func (s *store) ListProcesses() []Process { +func (s *store) ProcessList() []Process { s.lock.RLock() defer s.lock.RUnlock() @@ -219,13 +219,13 @@ func (s *store) ListProcesses() []Process { return processes } -func (s *store) GetProcess(id app.ProcessID) (Process, error) { +func (s *store) ProcessGet(id app.ProcessID) (Process, error) { s.lock.RLock() defer s.lock.RUnlock() process, ok := s.data.Process[id.String()] if !ok { - return Process{}, fmt.Errorf("not found") + return Process{}, fmt.Errorf("not found%w", ErrNotFound) } return Process{ @@ -238,7 +238,7 @@ func (s *store) GetProcess(id app.ProcessID) (Process, error) { }, nil } -func (s *store) GetProcessNodeMap() map[string]string { +func (s *store) ProcessGetNodeMap() map[string]string { s.lock.RLock() defer s.lock.RUnlock() @@ -251,7 +251,7 @@ func (s *store) GetProcessNodeMap() map[string]string { return m } -func (s *store) GetProcessRelocateMap() map[string]string { +func (s *store) ProcessGetRelocateMap() map[string]string { s.lock.RLock() defer s.lock.RUnlock() diff --git a/cluster/store/process_test.go b/cluster/store/process_test.go index 080707e3..03a33587 100644 --- a/cluster/store/process_test.go +++ b/cluster/store/process_test.go @@ -301,13 +301,13 @@ func TestUpdateProcess(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(s.data.Process)) - _, err = s.GetProcess(config1.ProcessID()) + _, err = s.ProcessGet(config1.ProcessID()) require.Error(t, err) - _, err = s.GetProcess(config2.ProcessID()) + _, err = s.ProcessGet(config2.ProcessID()) require.NoError(t, err) - _, err = s.GetProcess(config.ProcessID()) + _, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) } @@ -330,7 +330,7 @@ func TestSetProcessOrderCommand(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, s.data.Process) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "stop", p.Order) @@ -343,7 +343,7 @@ func TestSetProcessOrderCommand(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "start", p.Order) } @@ -382,7 +382,7 @@ func TestSetProcessOrder(t *testing.T) { }) require.NoError(t, err) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "stop", p.Order) @@ -392,7 +392,7 @@ func TestSetProcessOrder(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "start", p.Order) } @@ -416,7 +416,7 @@ func TestSetProcessMetadataCommand(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, s.data.Process) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Empty(t, p.Metadata) @@ -432,7 +432,7 @@ func TestSetProcessMetadataCommand(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.NotEmpty(t, p.Metadata) @@ -477,7 +477,7 @@ func TestSetProcessMetadata(t *testing.T) { }) require.NoError(t, err) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.NotEmpty(t, p.Metadata) @@ -492,7 +492,7 @@ func TestSetProcessMetadata(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.NotEmpty(t, p.Metadata) @@ -506,7 +506,7 @@ func TestSetProcessMetadata(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.NotEmpty(t, p.Metadata) @@ -533,7 +533,7 @@ func TestSetProcessErrorCommand(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, s.data.Process) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "", p.Error) @@ -546,7 +546,7 @@ func TestSetProcessErrorCommand(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "foobar", p.Error) } @@ -585,7 +585,7 @@ func TestSetProcessError(t *testing.T) { }) require.NoError(t, err) - p, err := s.GetProcess(config.ProcessID()) + p, err := s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "", p.Error) @@ -595,7 +595,7 @@ func TestSetProcessError(t *testing.T) { }) require.NoError(t, err) - p, err = s.GetProcess(config.ProcessID()) + p, err = s.ProcessGet(config.ProcessID()) require.NoError(t, err) require.Equal(t, "foobar", p.Error) } @@ -642,6 +642,6 @@ func TestSetProcessNodeMap(t *testing.T) { require.NoError(t, err) require.Equal(t, m2, s.data.ProcessNodeMap) - m := s.GetProcessNodeMap() + m := s.ProcessGetNodeMap() require.Equal(t, m2, m) } diff --git a/cluster/store/store.go b/cluster/store/store.go index f5d6c082..f6694f2b 100644 --- a/cluster/store/store.go +++ b/cluster/store/store.go @@ -20,23 +20,23 @@ type Store interface { OnApply(func(op Operation)) - ListProcesses() []Process - GetProcess(id app.ProcessID) (Process, error) - GetProcessNodeMap() map[string]string - GetProcessRelocateMap() map[string]string + ProcessList() []Process + ProcessGet(id app.ProcessID) (Process, error) + ProcessGetNodeMap() map[string]string + ProcessGetRelocateMap() map[string]string - ListUsers() Users - GetUser(name string) Users - ListPolicies() Policies - ListUserPolicies(name string) Policies + IAMIdentityList() Users + IAMIdentityGet(name string) Users + IAMIdentityPolicyList(name string) Policies + IAMPolicyList() Policies - HasLock(name string) bool - ListLocks() map[string]time.Time + LockHasLock(name string) bool + LockList() map[string]time.Time - ListKVS(prefix string) map[string]Value - GetFromKVS(key string) (Value, error) + KVSList(prefix string) map[string]Value + KVSGetValue(key string) (Value, error) - ListNodes() map[string]Node + NodeList() map[string]Node } type Process struct { diff --git a/cluster/version.go b/cluster/version.go index 0748b0b2..053a2a7a 100644 --- a/cluster/version.go +++ b/cluster/version.go @@ -38,6 +38,6 @@ func ParseClusterVersion(version string) (ClusterVersion, error) { // Version of the cluster var Version = ClusterVersion{ Major: 2, - Minor: 0, + Minor: 1, Patch: 0, } diff --git a/ffmpeg/skills/skills.go b/ffmpeg/skills/skills.go index da065c51..643ba176 100644 --- a/ffmpeg/skills/skills.go +++ b/ffmpeg/skills/skills.go @@ -23,24 +23,24 @@ type Codec struct { Decoders []string } -func (a Codec) Equal(b Codec) bool { +func (a Codec) Equal(b Codec) error { if a.Id != b.Id { - return false + return fmt.Errorf("id expected: %s, actual: %s", a.Id, b.Id) } if a.Name != b.Name { - return false + return fmt.Errorf("name expected: %s, actual: %s", a.Name, b.Name) } - if !slices.EqualComparableElements(a.Encoders, b.Encoders) { - return false + if err := slices.EqualComparableElements(a.Encoders, b.Encoders); err != nil { + return fmt.Errorf("codec %s encoders: %w", a.Name, err) } - if !slices.EqualComparableElements(a.Decoders, b.Decoders) { - return false + if err := slices.EqualComparableElements(a.Decoders, b.Decoders); err != nil { + return fmt.Errorf("codec %s decoders: %w", a.Name, err) } - return true + return nil } type ffCodecs struct { @@ -49,20 +49,20 @@ type ffCodecs struct { Subtitle []Codec } -func (a ffCodecs) Equal(b ffCodecs) bool { - if !slices.EqualEqualerElements(a.Audio, b.Audio) { - return false +func (a ffCodecs) Equal(b ffCodecs) error { + if err := slices.EqualEqualerElements(a.Audio, b.Audio); err != nil { + return fmt.Errorf("audio: %w", err) } - if !slices.EqualEqualerElements(a.Video, b.Video) { - return false + if err := slices.EqualEqualerElements(a.Video, b.Video); err != nil { + return fmt.Errorf("video: %w", err) } - if !slices.EqualEqualerElements(a.Subtitle, b.Subtitle) { - return false + if err := slices.EqualEqualerElements(a.Subtitle, b.Subtitle); err != nil { + return fmt.Errorf("subtitle: %w", err) } - return true + return nil } // HWDevice represents a hardware device (e.g. USB device) @@ -73,24 +73,24 @@ type HWDevice struct { Media string } -func (a HWDevice) Equal(b HWDevice) bool { +func (a HWDevice) Equal(b HWDevice) error { if a.Id != b.Id { - return false + return fmt.Errorf("id expected: %s, actual: %s", a.Id, b.Id) } if a.Name != b.Name { - return false + return fmt.Errorf("name expected: %s, actual: %s", a.Name, b.Name) } if a.Extra != b.Extra { - return false + return fmt.Errorf("extra expected: %s, actual: %s", a.Extra, b.Extra) } if a.Media != b.Media { - return false + return fmt.Errorf("media expected: %s, actual: %s", a.Media, b.Media) } - return true + return nil } // Device represents a type of device (e.g. V4L2) including connected actual devices @@ -100,20 +100,20 @@ type Device struct { Devices []HWDevice } -func (a Device) Equal(b Device) bool { +func (a Device) Equal(b Device) error { if a.Id != b.Id { - return false + return fmt.Errorf("id expected: %s, actual: %s", a.Id, b.Id) } if a.Name != b.Name { - return false + return fmt.Errorf("name expected: %s, actual: %s", a.Name, b.Name) } - if !slices.EqualEqualerElements(a.Devices, b.Devices) { - return false + if err := slices.EqualEqualerElements(a.Devices, b.Devices); err != nil { + return fmt.Errorf("hwdevice: %w", err) } - return true + return nil } type ffDevices struct { @@ -121,16 +121,16 @@ type ffDevices struct { Muxers []Device } -func (a ffDevices) Equal(b ffDevices) bool { - if !slices.EqualEqualerElements(a.Demuxers, b.Demuxers) { - return false +func (a ffDevices) Equal(b ffDevices) error { + if err := slices.EqualEqualerElements(a.Demuxers, b.Demuxers); err != nil { + return fmt.Errorf("demuxers: %w", err) } - if !slices.EqualEqualerElements(a.Muxers, b.Muxers) { - return false + if err := slices.EqualEqualerElements(a.Muxers, b.Muxers); err != nil { + return fmt.Errorf("muxers: %w", err) } - return true + return nil } // Format represents a supported format (e.g. flv) @@ -144,16 +144,16 @@ type ffFormats struct { Muxers []Format } -func (a ffFormats) Equal(b ffFormats) bool { - if !slices.EqualComparableElements(a.Demuxers, b.Demuxers) { - return false +func (a ffFormats) Equal(b ffFormats) error { + if err := slices.EqualComparableElements(a.Demuxers, b.Demuxers); err != nil { + return fmt.Errorf("demuxers: %w", err) } - if !slices.EqualComparableElements(a.Muxers, b.Muxers) { - return false + if err := slices.EqualComparableElements(a.Muxers, b.Muxers); err != nil { + return fmt.Errorf("muxers: %w", err) } - return true + return nil } // Protocol represents a supported protocol (e.g. rtsp) @@ -167,16 +167,16 @@ type ffProtocols struct { Output []Protocol } -func (a ffProtocols) Equal(b ffProtocols) bool { - if !slices.EqualComparableElements(a.Input, b.Input) { - return false +func (a ffProtocols) Equal(b ffProtocols) error { + if err := slices.EqualComparableElements(a.Input, b.Input); err != nil { + return fmt.Errorf("input: %w", err) } - if !slices.EqualComparableElements(a.Output, b.Output) { - return false + if err := slices.EqualComparableElements(a.Output, b.Output); err != nil { + return fmt.Errorf("output: %w", err) } - return true + return nil } type HWAccel struct { @@ -204,24 +204,24 @@ type ffmpeg struct { Libraries []Library } -func (a ffmpeg) Equal(b ffmpeg) bool { +func (a ffmpeg) Equal(b ffmpeg) error { if a.Version != b.Version { - return false + return fmt.Errorf("version expected: %s, actual: %s", a.Version, b.Version) } if a.Compiler != b.Compiler { - return false + return fmt.Errorf("compiler expected: %s, actual: %s", a.Compiler, b.Compiler) } if a.Configuration != b.Configuration { - return false + return fmt.Errorf("configuration expected: %s, actual: %s", a.Configuration, b.Configuration) } - if !slices.EqualComparableElements(a.Libraries, b.Libraries) { - return false + if err := slices.EqualComparableElements(a.Libraries, b.Libraries); err != nil { + return fmt.Errorf("libraries: %w", err) } - return true + return nil } // Skills are the detected capabilities of a ffmpeg binary @@ -237,36 +237,36 @@ type Skills struct { Protocols ffProtocols } -func (a Skills) Equal(b Skills) bool { - if !a.FFmpeg.Equal(b.FFmpeg) { - return false +func (a Skills) Equal(b Skills) error { + if err := a.FFmpeg.Equal(b.FFmpeg); err != nil { + return fmt.Errorf("ffmpeg: %w", err) } - if !slices.EqualComparableElements(a.Filters, b.Filters) { - return false + if err := slices.EqualComparableElements(a.Filters, b.Filters); err != nil { + return fmt.Errorf("filters: %w", err) } - if !slices.EqualComparableElements(a.HWAccels, b.HWAccels) { - return false + if err := slices.EqualComparableElements(a.HWAccels, b.HWAccels); err != nil { + return fmt.Errorf("hwaccels: %w", err) } - if !a.Codecs.Equal(b.Codecs) { - return false + if err := a.Codecs.Equal(b.Codecs); err != nil { + return fmt.Errorf("codecs: %w", err) } - if !a.Devices.Equal(b.Devices) { - return false + if err := a.Devices.Equal(b.Devices); err != nil { + return fmt.Errorf("devices: %w", err) } - if !a.Formats.Equal(b.Formats) { - return false + if err := a.Formats.Equal(b.Formats); err != nil { + return fmt.Errorf("formats: %w", err) } - if !a.Protocols.Equal(b.Protocols) { - return false + if err := a.Protocols.Equal(b.Protocols); err != nil { + return fmt.Errorf("protocols: %w", err) } - return true + return nil } // New returns all skills that ffmpeg provides diff --git a/ffmpeg/skills/skills_test.go b/ffmpeg/skills/skills_test.go index 7adf0a21..0ee8b74e 100644 --- a/ffmpeg/skills/skills_test.go +++ b/ffmpeg/skills/skills_test.go @@ -321,28 +321,28 @@ func TestNew(t *testing.T) { func TestEqualEmptySkills(t *testing.T) { s := Skills{} - ok := s.Equal(s) - require.True(t, ok) + err := s.Equal(s) + require.NoError(t, err) } -func TestEuqalSkills(t *testing.T) { +func TestEqualSkills(t *testing.T) { binary, err := testhelper.BuildBinary("ffmpeg", "../../internal/testhelper") require.NoError(t, err, "Failed to build helper program") s1, err := New(binary) require.NoError(t, err) - ok := s1.Equal(s1) - require.True(t, ok) + err = s1.Equal(s1) + require.NoError(t, err) s2, err := New(binary) require.NoError(t, err) - ok = s1.Equal(s2) - require.True(t, ok) + err = s1.Equal(s2) + require.NoError(t, err) - ok = s1.Equal(Skills{}) - require.False(t, ok) + err = s1.Equal(Skills{}) + require.Error(t, err) } func TestPatchVersion(t *testing.T) { diff --git a/glob/glob.go b/glob/glob.go index ea8b5762..0749bea0 100644 --- a/glob/glob.go +++ b/glob/glob.go @@ -10,6 +10,12 @@ type globber struct { glob glob.Glob } +func MustCompile(pattern string, separators ...rune) Glob { + g := glob.MustCompile(pattern, separators...) + + return &globber{glob: g} +} + func Compile(pattern string, separators ...rune) (Glob, error) { g, err := glob.Compile(pattern, separators...) if err != nil { diff --git a/go.mod b/go.mod index 442725fd..81a55f23 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,13 @@ require ( github.com/atrox/haikunatorgo/v2 v2.0.1 github.com/caddyserver/certmagic v0.21.3 github.com/casbin/casbin/v2 v2.90.0 - github.com/datarhei/core-client-go/v16 v16.11.1-0.20240429143858-23ad5985b894 github.com/datarhei/gosrt v0.6.0 github.com/datarhei/joy4 v0.0.0-20240603190808-b1407345907e github.com/fujiwara/shapeio v1.0.0 github.com/go-playground/validator/v10 v10.21.0 github.com/gobwas/glob v0.2.3 github.com/goccy/go-json v0.10.3 + github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/gops v0.3.28 github.com/google/uuid v1.6.0 @@ -76,7 +76,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index f386cc83..c4b26626 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,6 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/datarhei/core-client-go/v16 v16.11.1-0.20240429143858-23ad5985b894 h1:ZQCTobOGpzfuZxgMWsZviFSXfI5QuttkTgPQz1PKbhU= -github.com/datarhei/core-client-go/v16 v16.11.1-0.20240429143858-23ad5985b894/go.mod h1:Mu2bHqvGJe46KvAhY2ElohuQYhHB64PZeaCNDv6C5b8= github.com/datarhei/gosrt v0.6.0 h1:HrrXAw90V78ok4WMIhX6se1aTHPCn82Sg2hj+PhdmGc= github.com/datarhei/gosrt v0.6.0/go.mod h1:fsOWdLSHUHShHjgi/46h6wjtdQrtnSdAQFnlas8ONxs= github.com/datarhei/joy4 v0.0.0-20240603190808-b1407345907e h1:Qc/0D4xvXrazFkoi/4UGqO15yQ1JN5I8h7RwdzCLgTY= diff --git a/http/api/report.go b/http/api/report.go index bfdbb2f5..49c67a74 100644 --- a/http/api/report.go +++ b/http/api/report.go @@ -29,7 +29,7 @@ type ProcessReport struct { } // Unmarshal converts a restream log to a report -func (report *ProcessReport) Unmarshal(l *app.Log) { +func (report *ProcessReport) Unmarshal(l *app.Report) { if l == nil { return } diff --git a/vendor/github.com/datarhei/core-client-go/v16/client.go b/http/client/client.go similarity index 70% rename from vendor/github.com/datarhei/core-client-go/v16/client.go rename to http/client/client.go index 63396b11..7d8abc05 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/client.go +++ b/http/client/client.go @@ -1,4 +1,4 @@ -package coreclient +package client import ( "bytes" @@ -12,12 +12,12 @@ import ( "sync" "time" - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/glob" + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/restream/app" "github.com/Masterminds/semver/v3" - "github.com/gobwas/glob" jwtgo "github.com/golang-jwt/jwt/v4" "github.com/klauspost/compress/gzip" "github.com/klauspost/compress/zstd" @@ -50,24 +50,6 @@ type RestClient interface { About(cached bool) (api.About, error) // GET / - Config() (int64, api.Config, error) // GET /v3/config - ConfigSet(config interface{}) error // POST /v3/config - ConfigReload() error // GET /v3/config/reload - - Graph(query api.GraphQuery) (api.GraphResponse, error) // POST /graph - - DiskFSList(sort, order string) ([]api.FileInfo, error) // GET /v3/fs/disk - DiskFSHasFile(path string) bool // HEAD /v3/fs/disk/{path} - DiskFSGetFile(path string) (io.ReadCloser, error) // GET /v3/fs/disk/{path} - DiskFSDeleteFile(path string) error // DELETE /v3/fs/disk/{path} - DiskFSAddFile(path string, data io.Reader) error // PUT /v3/fs/disk/{path} - - MemFSList(sort, order string) ([]api.FileInfo, error) // GET /v3/fs/mem - MemFSHasFile(path string) bool // HEAD /v3/fs/mem/{path} - MemFSGetFile(path string) (io.ReadCloser, error) // GET /v3/fs/mem/{path} - MemFSDeleteFile(path string) error // DELETE /v3/fs/mem/{path} - MemFSAddFile(path string, data io.Reader) error // PUT /v3/fs/mem/{path} - FilesystemList(storage, pattern, sort, order string) ([]api.FileInfo, error) // GET /v3/fs/{storage} FilesystemHasFile(storage, path string) bool // HEAD /v3/fs/{storage}/{path} FilesystemGetFile(storage, path string) (io.ReadCloser, error) // GET /v3/fs/{storage}/{path} @@ -75,96 +57,28 @@ type RestClient interface { FilesystemDeleteFile(storage, path string) error // DELETE /v3/fs/{storage}/{path} FilesystemAddFile(storage, path string, data io.Reader) error // PUT /v3/fs/{storage}/{path} - Log() ([]api.LogEvent, error) // GET /v3/log Events(ctx context.Context, filters api.EventFilters) (<-chan api.Event, error) // POST /v3/events - Metadata(key string) (api.Metadata, error) // GET /v3/metadata/{key} - MetadataSet(key string, metadata api.Metadata) error // PUT /v3/metadata/{key} - - MetricsList() ([]api.MetricsDescription, error) // GET /v3/metrics - Metrics(query api.MetricsQuery) (api.MetricsResponse, error) // POST /v3/metrics - - ProcessList(opts ProcessListOptions) ([]api.Process, error) // GET /v3/process - ProcessAdd(p api.ProcessConfig) error // POST /v3/process - Process(id ProcessID, filter []string) (api.Process, error) // GET /v3/process/{id} - ProcessUpdate(id ProcessID, p api.ProcessConfig) error // PUT /v3/process/{id} - ProcessDelete(id ProcessID) error // DELETE /v3/process/{id} - ProcessCommand(id ProcessID, command string) error // PUT /v3/process/{id}/command - ProcessProbe(id ProcessID) (api.Probe, error) // GET /v3/process/{id}/probe - ProcessProbeConfig(config api.ProcessConfig) (api.Probe, error) // POST /v3/process/probe - ProcessConfig(id ProcessID) (api.ProcessConfig, error) // GET /v3/process/{id}/config - ProcessReport(id ProcessID) (api.ProcessReport, error) // GET /v3/process/{id}/report - ProcessState(id ProcessID) (api.ProcessState, error) // GET /v3/process/{id}/state - ProcessMetadata(id ProcessID, key string) (api.Metadata, error) // GET /v3/process/{id}/metadata/{key} - ProcessMetadataSet(id ProcessID, key string, metadata api.Metadata) error // PUT /v3/process/{id}/metadata/{key} - - PlayoutStatus(id ProcessID, inputID string) (api.PlayoutStatus, error) // GET /v3/process/{id}/playout/{inputid}/status - - IdentitiesList() ([]api.IAMUser, error) // GET /v3/iam/user - Identity(name string) (api.IAMUser, error) // GET /v3/iam/user/{name} - IdentityAdd(u api.IAMUser) error // POST /v3/iam/user - IdentityUpdate(name string, u api.IAMUser) error // PUT /v3/iam/user/{name} - IdentitySetPolicies(name string, p []api.IAMPolicy) error // PUT /v3/iam/user/{name}/policy - IdentityDelete(name string) error // DELETE /v3/iam/user/{name} - - Cluster() (*api.ClusterAboutV1, *api.ClusterAboutV2, error) // GET /v3/cluster - ClusterHealthy() (bool, error) // GET /v3/cluster/healthy - ClusterSnapshot() (io.ReadCloser, error) // GET /v3/cluster/snapshot - ClusterLeave() error // PUT /v3/cluster/leave - ClusterTransferLeadership(id string) error // PUT /v3/cluster/transfer/{id} - - ClusterNodeList() ([]api.ClusterNode, error) // GET /v3/cluster/node - ClusterNode(id string) (api.ClusterNode, error) // GET /v3/cluster/node/{id} - ClusterNodeFiles(id string) (api.ClusterNodeFiles, error) // GET /v3/cluster/node/{id}/files - ClusterNodeProcessList(id string, opts ProcessListOptions) ([]api.Process, error) // GET /v3/cluster/node/{id}/process - ClusterNodeVersion(id string) (api.Version, error) // GET /v3/cluster/node/{id}/version - ClusterNodeFilesystemList(id, storage, pattern, sort, order string) ([]api.FileInfo, error) // GET /v3/cluster/node/{id}/fs/{storage} - ClusterNodeFilesystemDeleteFile(id, storage, path string) error // DELETE /v3/cluster/node/{id}/fs/{storage}/{path} - ClusterNodeFilesystemPutFile(id, storage, path string, data io.Reader) error // PUT /v3/cluster/node/{id}/fs/{storage}/{path} - ClusterNodeFilesystemGetFile(id, storage, path string) (io.ReadCloser, error) // GET /v3/cluster/node/{id}/fs/{storage}/{path} - - ClusterDBProcessList() ([]api.Process, error) // GET /v3/cluster/db/process - ClusterDBProcess(id ProcessID) (api.Process, error) // GET /v3/cluster/db/process/{id} - ClusterDBUserList() ([]api.IAMUser, error) // GET /v3/cluster/db/user - ClusterDBUser(name string) (api.IAMUser, error) // GET /v3/cluster/db/user/{name} - ClusterDBPolicies() ([]api.IAMPolicy, error) // GET /v3/cluster/db/policies - ClusterDBLocks() ([]api.ClusterLock, error) // GET /v3/cluster/db/locks - ClusterDBKeyValues() (api.ClusterKVS, error) // GET /v3/cluster/db/kv - ClusterDBProcessMap() (api.ClusterProcessMap, error) // GET /v3/cluster/db/map/process - - ClusterFilesystemList(name, pattern, sort, order string) ([]api.FileInfo, error) // GET /v3/cluster/fs/{storage} - - ClusterProcessList(opts ProcessListOptions) ([]api.Process, error) // GET /v3/cluster/process - ClusterProcess(id ProcessID, filter []string) (api.Process, error) // POST /v3/cluster/process - ClusterProcessAdd(p api.ProcessConfig) error // GET /v3/cluster/process/{id} - ClusterProcessUpdate(id ProcessID, p api.ProcessConfig) error // PUT /v3/cluster/process/{id} - ClusterProcessDelete(id ProcessID) error // DELETE /v3/cluster/process/{id} - ClusterProcessCommand(id ProcessID, command string) error // PUT /v3/cluster/process/{id}/command - ClusterProcessMetadata(id ProcessID, key string) (api.Metadata, error) // GET /v3/cluster/process/{id}/metadata/{key} - ClusterProcessMetadataSet(id ProcessID, key string, metadata api.Metadata) error // PUT /v3/cluster/process/{id}/metadata/{key} - ClusterProcessProbe(id ProcessID) (api.Probe, error) // GET /v3/cluster/process/{id}/probe - ClusterProcessProbeConfig(config api.ProcessConfig, coreid string) (api.Probe, error) // POST /v3/cluster/process/probe - - ClusterIdentitiesList() ([]api.IAMUser, error) // GET /v3/cluster/iam/user - ClusterIdentity(name string) (api.IAMUser, error) // GET /v3/cluster/iam/user/{name} - ClusterIdentityAdd(u api.IAMUser) error // POST /v3/cluster/iam/user - ClusterIdentityUpdate(name string, u api.IAMUser) error // PUT /v3/cluster/iam/user/{name} - ClusterIdentitySetPolicies(name string, p []api.IAMPolicy) error // PUT /v3/cluster/iam/user/{name}/policy - ClusterIdentityDelete(name string) error // DELETE /v3/cluster/iam/user/{name} - ClusterIAMReload() error // PUT /v3/cluster/iam/reload + ProcessList(opts ProcessListOptions) ([]api.Process, error) // GET /v3/process + ProcessAdd(p *app.Config, metadata map[string]interface{}) error // POST /v3/process + Process(id app.ProcessID, filter []string) (api.Process, error) // GET /v3/process/{id} + ProcessUpdate(id app.ProcessID, p *app.Config, metadata map[string]interface{}) error // PUT /v3/process/{id} + ProcessDelete(id app.ProcessID) error // DELETE /v3/process/{id} + ProcessCommand(id app.ProcessID, command string) error // PUT /v3/process/{id}/command + ProcessProbe(id app.ProcessID) (api.Probe, error) // GET /v3/process/{id}/probe + ProcessProbeConfig(config *app.Config) (api.Probe, error) // POST /v3/process/probe + ProcessConfig(id app.ProcessID) (api.ProcessConfig, error) // GET /v3/process/{id}/config + ProcessReport(id app.ProcessID) (api.ProcessReport, error) // GET /v3/process/{id}/report + ProcessState(id app.ProcessID) (api.ProcessState, error) // GET /v3/process/{id}/state + ProcessMetadata(id app.ProcessID, key string) (api.Metadata, error) // GET /v3/process/{id}/metadata/{key} + ProcessMetadataSet(id app.ProcessID, key string, metadata api.Metadata) error // PUT /v3/process/{id}/metadata/{key} RTMPChannels() ([]api.RTMPChannel, error) // GET /v3/rtmp SRTChannels() ([]api.SRTChannel, error) // GET /v3/srt SRTChannelsRaw() ([]byte, error) // GET /v3/srt - Sessions(collectors []string) (api.SessionsSummary, error) // GET /v3/session - SessionsActive(collectors []string) (api.SessionsActive, error) // GET /v3/session/active - SessionToken(name string, req []api.SessionTokenRequest) ([]api.SessionTokenRequest, error) // PUT /v3/session/token/{username} - Skills() (api.Skills, error) // GET /v3/skills SkillsReload() error // GET /v3/skills/reload - - WidgetProcess(id ProcessID) (api.WidgetProcess, error) // GET /v3/widget/process/{id} } type Token struct { @@ -448,6 +362,14 @@ func New(config Config) (RestClient, error) { path: mustNewGlob("/v3/cluster/node/*/fs/*/**"), constraint: mustNewConstraint("^16.14.0"), }, + { + path: mustNewGlob("/v3/cluster/db/node"), + constraint: mustNewConstraint("^16.14.0"), + }, + { + path: mustNewGlob("/v3/cluster/node/*/state"), + constraint: mustNewConstraint("^16.14.0"), + }, }, "POST": { { @@ -524,6 +446,14 @@ func New(config Config) (RestClient, error) { path: mustNewGlob("/v3/cluster/node/*/fs/*/**"), constraint: mustNewConstraint("^16.14.0"), }, + { + path: mustNewGlob("/v3/cluster/reallocate"), + constraint: mustNewConstraint("^16.14.0"), + }, + { + path: mustNewGlob("/v3/cluster/node/*/state"), + constraint: mustNewConstraint("^16.14.0"), + }, }, "DELETE": { { @@ -614,6 +544,9 @@ func (r *restclient) Address() string { func (r *restclient) About(cached bool) (api.About, error) { if cached { + r.aboutLock.RLock() + defer r.aboutLock.RUnlock() + return r.about, nil } @@ -622,7 +555,22 @@ func (r *restclient) About(cached bool) (api.About, error) { return api.About{}, err } + if r.accessToken.IsSet() && len(about.ID) == 0 { + if err := r.refresh(); err != nil { + if err := r.login(); err != nil { + return api.About{}, err + } + } + + about, err = r.info() + if err != nil { + return api.About{}, err + } + } + + r.aboutLock.Lock() r.about = about + r.aboutLock.Unlock() return about, nil } @@ -821,7 +769,15 @@ func (r *restclient) info() (api.About, error) { return api.About{}, err } - if r.accessToken.IsSet() && !r.accessToken.IsExpired() { + if r.accessToken.IsSet() { + if r.accessToken.IsExpired() { + if err := r.refresh(); err != nil { + if err := r.login(); err != nil { + return api.About{}, err + } + } + } + req.Header.Add("Authorization", "Bearer "+r.accessToken.String()) } @@ -942,7 +898,7 @@ func (r *restclient) stream(ctx context.Context, method, path string, query *url return nil, e } - e.Body = data + //e.Body = data err = json.Unmarshal(data, &e) if err != nil { @@ -967,7 +923,7 @@ func (r *restclient) call(method, path string, query *url.Values, header http.He body, err := r.stream(ctx, method, path, query, header, contentType, data) if err != nil { - return nil, err + return nil, fmt.Errorf("%s %s: %w", method, path, err) } defer body.Close() diff --git a/vendor/github.com/datarhei/core-client-go/v16/events.go b/http/client/events.go similarity index 91% rename from vendor/github.com/datarhei/core-client-go/v16/events.go rename to http/client/events.go index 99121a52..36e7063b 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/events.go +++ b/http/client/events.go @@ -1,4 +1,4 @@ -package coreclient +package client import ( "bytes" @@ -6,9 +6,8 @@ import ( "io" "net/http" - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" ) func (r *restclient) Events(ctx context.Context, filters api.EventFilters) (<-chan api.Event, error) { diff --git a/vendor/github.com/datarhei/core-client-go/v16/fs.go b/http/client/fs.go similarity index 95% rename from vendor/github.com/datarhei/core-client-go/v16/fs.go rename to http/client/fs.go index f7a03544..5308eef8 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/fs.go +++ b/http/client/fs.go @@ -1,4 +1,4 @@ -package coreclient +package client import ( "context" @@ -8,9 +8,8 @@ import ( "path/filepath" "strconv" - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" ) const ( diff --git a/http/client/process.go b/http/client/process.go new file mode 100644 index 00000000..76fd1883 --- /dev/null +++ b/http/client/process.go @@ -0,0 +1,256 @@ +package client + +import ( + "bytes" + "net/url" + "strings" + + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" + "github.com/datarhei/core/v16/restream/app" +) + +type ProcessListOptions struct { + ID []string + Filter []string + Domain string + Reference string + IDPattern string + RefPattern string + OwnerPattern string + DomainPattern string +} + +func (p *ProcessListOptions) Query() *url.Values { + values := &url.Values{} + values.Set("id", strings.Join(p.ID, ",")) + values.Set("filter", strings.Join(p.Filter, ",")) + values.Set("domain", p.Domain) + values.Set("reference", p.Reference) + values.Set("idpattern", p.IDPattern) + values.Set("refpattern", p.RefPattern) + values.Set("ownerpattern", p.OwnerPattern) + values.Set("domainpattern", p.DomainPattern) + + return values +} + +func (r *restclient) ProcessList(opts ProcessListOptions) ([]api.Process, error) { + var processes []api.Process + + data, err := r.call("GET", "/v3/process", opts.Query(), nil, "", nil) + if err != nil { + return processes, err + } + + err = json.Unmarshal(data, &processes) + + return processes, err +} + +func (r *restclient) Process(id app.ProcessID, filter []string) (api.Process, error) { + var info api.Process + + values := &url.Values{} + values.Set("filter", strings.Join(filter, ",")) + values.Set("domain", id.Domain) + + data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID), values, nil, "", nil) + if err != nil { + return info, err + } + + err = json.Unmarshal(data, &info) + + return info, err +} + +func (r *restclient) ProcessAdd(p *app.Config, metadata map[string]interface{}) error { + var buf bytes.Buffer + + config := api.ProcessConfig{} + config.Unmarshal(p) + config.Metadata = metadata + + e := json.NewEncoder(&buf) + e.Encode(config) + + _, err := r.call("POST", "/v3/process", nil, nil, "application/json", &buf) + if err != nil { + return err + } + + return nil +} + +func (r *restclient) ProcessUpdate(id app.ProcessID, p *app.Config, metadata map[string]interface{}) error { + var buf bytes.Buffer + + config := api.ProcessConfig{} + config.Unmarshal(p) + config.Metadata = metadata + + e := json.NewEncoder(&buf) + e.Encode(config) + + query := &url.Values{} + query.Set("domain", id.Domain) + + _, err := r.call("PUT", "/v3/process/"+url.PathEscape(id.ID), query, nil, "application/json", &buf) + if err != nil { + return err + } + + return nil +} + +func (r *restclient) ProcessDelete(id app.ProcessID) error { + query := &url.Values{} + query.Set("domain", id.Domain) + + r.call("DELETE", "/v3/process/"+url.PathEscape(id.ID), query, nil, "", nil) + + return nil +} + +func (r *restclient) ProcessCommand(id app.ProcessID, command string) error { + var buf bytes.Buffer + + e := json.NewEncoder(&buf) + e.Encode(api.Command{ + Command: command, + }) + + query := &url.Values{} + query.Set("domain", id.Domain) + + _, err := r.call("PUT", "/v3/process/"+url.PathEscape(id.ID)+"/command", query, nil, "application/json", &buf) + if err != nil { + return err + } + + return nil +} + +func (r *restclient) ProcessMetadata(id app.ProcessID, key string) (api.Metadata, error) { + var m api.Metadata + + path := "/v3/process/" + url.PathEscape(id.ID) + "/metadata" + + if len(key) != 0 { + path += "/" + url.PathEscape(key) + } + + query := &url.Values{} + query.Set("domain", id.Domain) + + data, err := r.call("GET", path, query, nil, "", nil) + if err != nil { + return m, err + } + + err = json.Unmarshal(data, &m) + + return m, err +} + +func (r *restclient) ProcessMetadataSet(id app.ProcessID, key string, metadata api.Metadata) error { + var buf bytes.Buffer + + e := json.NewEncoder(&buf) + e.Encode(metadata) + + query := &url.Values{} + query.Set("domain", id.Domain) + + _, err := r.call("PUT", "/v3/process/"+url.PathEscape(id.ID)+"/metadata/"+url.PathEscape(key), query, nil, "application/json", &buf) + if err != nil { + return err + } + + return nil +} + +func (r *restclient) ProcessProbe(id app.ProcessID) (api.Probe, error) { + var p api.Probe + + query := &url.Values{} + query.Set("domain", id.Domain) + + data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/probe", query, nil, "", nil) + if err != nil { + return p, err + } + + err = json.Unmarshal(data, &p) + + return p, err +} + +func (r *restclient) ProcessProbeConfig(p *app.Config) (api.Probe, error) { + var probe api.Probe + var buf bytes.Buffer + + config := api.ProcessConfig{} + config.Unmarshal(p) + + e := json.NewEncoder(&buf) + e.Encode(config) + + data, err := r.call("POST", "/v3/process/probe", nil, nil, "application/json", &buf) + if err != nil { + return probe, err + } + + err = json.Unmarshal(data, &p) + + return probe, err +} + +func (r *restclient) ProcessConfig(id app.ProcessID) (api.ProcessConfig, error) { + var p api.ProcessConfig + + query := &url.Values{} + query.Set("domain", id.Domain) + + data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/config", query, nil, "", nil) + if err != nil { + return p, err + } + + err = json.Unmarshal(data, &p) + + return p, err +} + +func (r *restclient) ProcessReport(id app.ProcessID) (api.ProcessReport, error) { + var p api.ProcessReport + + query := &url.Values{} + query.Set("domain", id.Domain) + + data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/report", query, nil, "", nil) + if err != nil { + return p, err + } + + err = json.Unmarshal(data, &p) + + return p, err +} + +func (r *restclient) ProcessState(id app.ProcessID) (api.ProcessState, error) { + var p api.ProcessState + + query := &url.Values{} + query.Set("domain", id.Domain) + + data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/state", query, nil, "", nil) + if err != nil { + return p, err + } + + err = json.Unmarshal(data, &p) + + return p, err +} diff --git a/vendor/github.com/datarhei/core-client-go/v16/rtmp.go b/http/client/rtmp.go similarity index 71% rename from vendor/github.com/datarhei/core-client-go/v16/rtmp.go rename to http/client/rtmp.go index e8244dc9..13e309d2 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/rtmp.go +++ b/http/client/rtmp.go @@ -1,9 +1,8 @@ -package coreclient +package client import ( - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" ) func (r *restclient) RTMPChannels() ([]api.RTMPChannel, error) { diff --git a/vendor/github.com/datarhei/core-client-go/v16/skills.go b/http/client/skills.go similarity index 78% rename from vendor/github.com/datarhei/core-client-go/v16/skills.go rename to http/client/skills.go index 36add59b..6bd12fb9 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/skills.go +++ b/http/client/skills.go @@ -1,9 +1,8 @@ -package coreclient +package client import ( - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" ) func (r *restclient) Skills() (api.Skills, error) { diff --git a/vendor/github.com/datarhei/core-client-go/v16/srt.go b/http/client/srt.go similarity index 76% rename from vendor/github.com/datarhei/core-client-go/v16/srt.go rename to http/client/srt.go index b2b1da3b..cf7f2fb3 100644 --- a/vendor/github.com/datarhei/core-client-go/v16/srt.go +++ b/http/client/srt.go @@ -1,9 +1,8 @@ -package coreclient +package client import ( - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" + "github.com/datarhei/core/v16/encoding/json" + "github.com/datarhei/core/v16/http/api" ) func (r *restclient) SRTChannels() ([]api.SRTChannel, error) { diff --git a/http/fs/cluster.go b/http/fs/cluster.go index 00428c74..d8c1b109 100644 --- a/http/fs/cluster.go +++ b/http/fs/cluster.go @@ -6,7 +6,7 @@ import ( gofs "io/fs" "time" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/io/fs" ) @@ -18,10 +18,10 @@ type filesystem struct { fs.Filesystem name string - proxy proxy.ProxyReader + proxy *node.Manager } -func NewClusterFS(name string, fs fs.Filesystem, proxy proxy.ProxyReader) Filesystem { +func NewClusterFS(name string, fs fs.Filesystem, proxy *node.Manager) Filesystem { if proxy == nil { return fs } @@ -42,14 +42,14 @@ func (fs *filesystem) Open(path string) fs.File { } // Check if the file is available in the cluster - size, lastModified, err := fs.proxy.GetFileInfo(fs.name, path) + size, lastModified, err := fs.proxy.FilesystemGetFileInfo(fs.name, path) if err != nil { return nil } file := &file{ getFile: func(offset int64) (io.ReadCloser, error) { - return fs.proxy.GetFile(fs.name, path, offset) + return fs.proxy.FilesystemGetFile(fs.name, path, offset) }, name: path, size: size, diff --git a/http/graph/models/models.go b/http/graph/models/models.go index 5c1eab71..7d7b0f3e 100644 --- a/http/graph/models/models.go +++ b/http/graph/models/models.go @@ -44,7 +44,7 @@ func (s *RawAVstreamSwap) UnmarshalPlayout(status playout.Status) { s.Lasterror = status.Swap.LastError } -func (p *Process) UnmarshalRestream(process *app.Process, state *app.State, report *app.Log, metadata map[string]interface{}) { +func (p *Process) UnmarshalRestream(process *app.Process, state *app.State, report *app.Report, metadata map[string]interface{}) { p.ID = process.ID p.Type = "ffmpeg" p.Reference = process.Reference @@ -189,7 +189,7 @@ func (a *AVStreamIo) UnmarshalRestream(io app.AVstreamIO) { a.SizeKb = scalars.Uint64(io.Size) } -func (r *ProcessReport) UnmarshalRestream(report *app.Log) { +func (r *ProcessReport) UnmarshalRestream(report *app.Report) { r.CreatedAt = report.CreatedAt r.Prelude = report.Prelude r.Log = []*ProcessReportLogEntry{} @@ -210,7 +210,7 @@ func (r *ProcessReport) UnmarshalRestream(report *app.Log) { } } -func (h *ProcessReportHistoryEntry) UnmarshalRestream(entry app.LogHistoryEntry) { +func (h *ProcessReportHistoryEntry) UnmarshalRestream(entry app.ReportHistoryEntry) { h.CreatedAt = entry.CreatedAt h.Prelude = entry.Prelude h.Log = []*ProcessReportLogEntry{} diff --git a/http/handler/api/cluster.go b/http/handler/api/cluster.go index 7666650d..a4e00639 100644 --- a/http/handler/api/cluster.go +++ b/http/handler/api/cluster.go @@ -8,7 +8,7 @@ import ( "time" "github.com/datarhei/core/v16/cluster" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/encoding/json" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/handler/util" @@ -21,7 +21,7 @@ import ( // The ClusterHandler type provides handler functions for manipulating the cluster config. type ClusterHandler struct { cluster cluster.Cluster - proxy proxy.ProxyReader + proxy *node.Manager iam iam.IAM } @@ -29,7 +29,7 @@ type ClusterHandler struct { func NewCluster(cluster cluster.Cluster, iam iam.IAM) (*ClusterHandler, error) { h := &ClusterHandler{ cluster: cluster, - proxy: cluster.ProxyReader(), + proxy: cluster.Manager(), iam: iam, } @@ -68,7 +68,7 @@ func (h *ClusterHandler) About(c echo.Context) error { Address: state.Leader.Address, ElectedSince: uint64(state.Leader.ElectedSince.Seconds()), }, - Status: state.Status, + Status: state.State, Raft: api.ClusterRaft{ Address: state.Raft.Address, State: state.Raft.State, @@ -77,13 +77,13 @@ func (h *ClusterHandler) About(c echo.Context) error { LogTerm: state.Raft.LogTerm, LogIndex: state.Raft.LogIndex, }, - Nodes: []api.ClusterNode{}, - Version: state.Version.String(), - Degraded: state.Degraded, + Nodes: []api.ClusterNode{}, + Version: state.Version.String(), } - if state.DegradedErr != nil { - about.DegradedErr = state.DegradedErr.Error() + if state.Error != nil { + about.Degraded = true + about.DegradedErr = state.Error.Error() } for _, node := range state.Nodes { @@ -98,7 +98,7 @@ func (h *ClusterHandler) marshalClusterNode(node cluster.ClusterNode) api.Cluste ID: node.ID, Name: node.Name, Version: node.Version, - Status: node.Status, + Status: node.State, Voter: node.Voter, Leader: node.Leader, Address: node.Address, @@ -108,7 +108,7 @@ func (h *ClusterHandler) marshalClusterNode(node cluster.ClusterNode) api.Cluste Latency: node.Latency.Seconds() * 1000, Core: api.ClusterNodeCore{ Address: node.Core.Address, - Status: node.Core.Status, + Status: node.Core.State, LastContact: node.Core.LastContact.Seconds() * 1000, Latency: node.Core.Latency.Seconds() * 1000, Version: node.Core.Version, @@ -148,9 +148,9 @@ func (h *ClusterHandler) marshalClusterNode(node cluster.ClusterNode) api.Cluste // @Security ApiKeyAuth // @Router /api/v3/cluster/healthy [get] func (h *ClusterHandler) Healthy(c echo.Context) error { - degraded, _ := h.cluster.IsDegraded() + hasLeader := h.cluster.HasRaftLeader() - return c.JSON(http.StatusOK, !degraded) + return c.JSON(http.StatusOK, hasLeader) } // TransferLeadership transfers the leadership to another node @@ -266,7 +266,7 @@ func (h *ClusterHandler) Reallocation(c echo.Context) error { } } - err := h.cluster.RelocateProcesses("", relocations) + err := h.cluster.ProcessesRelocate("", relocations) if err != nil { return api.Err(http.StatusInternalServerError, "", "%s", err.Error()) } diff --git a/http/handler/api/cluster_fs.go b/http/handler/api/cluster_fs.go index 66ed390d..ecb4bb8a 100644 --- a/http/handler/api/cluster_fs.go +++ b/http/handler/api/cluster_fs.go @@ -9,7 +9,7 @@ import ( "github.com/labstack/echo/v4" ) -// ListFiles lists all files on a filesystem +// FilesystemListFiles lists all files on a filesystem // @Summary List all files on a filesystem // @Description List all files on a filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order. // @Tags v16.?.? @@ -23,13 +23,13 @@ import ( // @Success 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/fs/{storage} [get] -func (h *ClusterHandler) ListFiles(c echo.Context) error { +func (h *ClusterHandler) FilesystemListFiles(c echo.Context) error { name := util.PathParam(c, "storage") pattern := util.DefaultQuery(c, "glob", "") sortby := util.DefaultQuery(c, "sort", "none") order := util.DefaultQuery(c, "order", "asc") - files := h.proxy.ListFiles(name, pattern) + files := h.proxy.FilesystemList(name, pattern) var sortFunc func(i, j int) bool diff --git a/http/handler/api/cluster_iam.go b/http/handler/api/cluster_iam.go index fa9a71b4..91ed30e0 100644 --- a/http/handler/api/cluster_iam.go +++ b/http/handler/api/cluster_iam.go @@ -1,8 +1,10 @@ package api import ( + "errors" "net/http" + "github.com/datarhei/core/v16/cluster/store" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/handler/util" "github.com/datarhei/core/v16/iam/access" @@ -23,7 +25,7 @@ import ( // @Failure 403 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user [post] -func (h *ClusterHandler) AddIdentity(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityAdd(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) domain := util.DefaultQuery(c, "domain", "") @@ -50,18 +52,18 @@ func (h *ClusterHandler) AddIdentity(c echo.Context) error { return api.Err(http.StatusForbidden, "", "Only superusers can add superusers") } - if err := h.cluster.AddIdentity("", iamuser); err != nil { + if err := h.cluster.IAMIdentityAdd("", iamuser); err != nil { return api.Err(http.StatusBadRequest, "", "invalid identity: %s", err.Error()) } - if err := h.cluster.SetPolicies("", iamuser.Name, iampolicies); err != nil { + if err := h.cluster.IAMPoliciesSet("", iamuser.Name, iampolicies); err != nil { return api.Err(http.StatusBadRequest, "", "Invalid policies: %s", err.Error()) } return c.JSON(http.StatusOK, user) } -// UpdateIdentity replaces an existing user +// IAMIdentityUpdate replaces an existing user // @Summary Replace an existing user // @Description Replace an existing user. // @Tags v16.?.? @@ -78,7 +80,7 @@ func (h *ClusterHandler) AddIdentity(c echo.Context) error { // @Failure 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user/{name} [put] -func (h *ClusterHandler) UpdateIdentity(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityUpdate(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) domain := util.DefaultQuery(c, "domain", "") @@ -128,13 +130,13 @@ func (h *ClusterHandler) UpdateIdentity(c echo.Context) error { } if name != "$anon" { - err = h.cluster.UpdateIdentity("", name, iamuser) + err = h.cluster.IAMIdentityUpdate("", name, iamuser) if err != nil { return api.Err(http.StatusBadRequest, "", "%s", err.Error()) } } - err = h.cluster.SetPolicies("", iamuser.Name, iampolicies) + err = h.cluster.IAMPoliciesSet("", iamuser.Name, iampolicies) if err != nil { return api.Err(http.StatusInternalServerError, "", "set policies: %s", err.Error()) } @@ -142,7 +144,7 @@ func (h *ClusterHandler) UpdateIdentity(c echo.Context) error { return c.JSON(http.StatusOK, user) } -// UpdateIdentityPolicies replaces existing user policies +// IAMIdentityUpdatePolicies replaces existing user policies // @Summary Replace policies of an user // @Description Replace policies of an user // @Tags v16.?.? @@ -159,7 +161,7 @@ func (h *ClusterHandler) UpdateIdentity(c echo.Context) error { // @Failure 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user/{name}/policy [put] -func (h *ClusterHandler) UpdateIdentityPolicies(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityUpdatePolicies(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) domain := util.DefaultQuery(c, "domain", "") @@ -216,15 +218,18 @@ func (h *ClusterHandler) UpdateIdentityPolicies(c echo.Context) error { return api.Err(http.StatusForbidden, "", "only superusers can modify superusers") } - err = h.cluster.SetPolicies("", name, accessPolicies) + err = h.cluster.IAMPoliciesSet("", name, accessPolicies) if err != nil { + if errors.Is(err, store.ErrNotFound) { + return api.Err(http.StatusNotFound, "", "set policies: %s", err.Error()) + } return api.Err(http.StatusInternalServerError, "", "set policies: %s", err.Error()) } return c.JSON(http.StatusOK, policies) } -// ReloadIAM reloads the identities and policies from the cluster store to IAM +// IAMReload reloads the identities and policies from the cluster store to IAM // @Summary Reload identities and policies // @Description Reload identities and policies // @Tags v16.?.? @@ -234,7 +239,7 @@ func (h *ClusterHandler) UpdateIdentityPolicies(c echo.Context) error { // @Success 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/reload [get] -func (h *ClusterHandler) ReloadIAM(c echo.Context) error { +func (h *ClusterHandler) IAMReload(c echo.Context) error { err := h.iam.ReloadIndentities() if err != nil { return api.Err(http.StatusInternalServerError, "", "reload identities: %w", err.Error()) @@ -248,7 +253,7 @@ func (h *ClusterHandler) ReloadIAM(c echo.Context) error { return c.JSON(http.StatusOK, "OK") } -// ListIdentities returns the list of identities stored in IAM +// IAMIdentityList returns the list of identities stored in IAM // @Summary List of identities in IAM // @Description List of identities in IAM // @Tags v16.?.? @@ -257,7 +262,7 @@ func (h *ClusterHandler) ReloadIAM(c echo.Context) error { // @Success 200 {array} api.IAMUser // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user [get] -func (h *ClusterHandler) ListIdentities(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityList(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") @@ -292,7 +297,7 @@ func (h *ClusterHandler) ListIdentities(c echo.Context) error { return c.JSON(http.StatusOK, users) } -// ListIdentity returns the identity stored in IAM +// IAMIdentityGet returns the identity stored in IAM // @Summary Identity in IAM // @Description Identity in IAM // @Tags v16.?.? @@ -303,7 +308,7 @@ func (h *ClusterHandler) ListIdentities(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user/{name} [get] -func (h *ClusterHandler) ListIdentity(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityGet(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") name := util.PathParam(c, "name") @@ -342,7 +347,7 @@ func (h *ClusterHandler) ListIdentity(c echo.Context) error { return c.JSON(http.StatusOK, user) } -// ListPolicies returns the list of policies stored in IAM +// IAMPolicyList returns the list of policies stored in IAM // @Summary List of policies in IAM // @Description List of policies IAM // @Tags v16.?.? @@ -351,7 +356,7 @@ func (h *ClusterHandler) ListIdentity(c echo.Context) error { // @Success 200 {array} api.IAMPolicy // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/policies [get] -func (h *ClusterHandler) ListPolicies(c echo.Context) error { +func (h *ClusterHandler) IAMPolicyList(c echo.Context) error { iampolicies := h.iam.ListPolicies("", "", nil, "", nil) policies := []api.IAMPolicy{} @@ -381,7 +386,7 @@ func (h *ClusterHandler) ListPolicies(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/iam/user/{name} [delete] -func (h *ClusterHandler) RemoveIdentity(c echo.Context) error { +func (h *ClusterHandler) IAMIdentityRemove(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) domain := util.DefaultQuery(c, "domain", "$none") @@ -400,7 +405,7 @@ func (h *ClusterHandler) RemoveIdentity(c echo.Context) error { return api.Err(http.StatusForbidden, "", "Only superusers can remove superusers") } - if err := h.cluster.RemoveIdentity("", name); err != nil { + if err := h.cluster.IAMIdentityRemove("", name); err != nil { return api.Err(http.StatusBadRequest, "", "invalid identity: %s", err.Error()) } diff --git a/http/handler/api/cluster_node.go b/http/handler/api/cluster_node.go index 1ef0f2b0..c3bfee05 100644 --- a/http/handler/api/cluster_node.go +++ b/http/handler/api/cluster_node.go @@ -7,15 +7,14 @@ import ( "strings" "time" - clientapi "github.com/datarhei/core-client-go/v16/api" "github.com/datarhei/core/v16/cluster" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/handler/util" "github.com/labstack/echo/v4" ) -// GetNodes returns the list of proxy nodes in the cluster +// NodeList returns the list of proxy nodes in the cluster // @Summary List of proxy nodes in the cluster // @Description List of proxy nodes in the cluster // @Tags v16.?.? @@ -25,17 +24,17 @@ import ( // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node [get] -func (h *ClusterHandler) GetNodes(c echo.Context) error { +func (h *ClusterHandler) NodeList(c echo.Context) error { about, _ := h.cluster.About() - nodes := h.cluster.ListNodes() + nodes := h.cluster.Store().NodeList() list := []api.ClusterNode{} for _, node := range about.Nodes { if dbnode, hasNode := nodes[node.ID]; hasNode { if dbnode.State == "maintenance" { - node.Status = dbnode.State + node.State = dbnode.State } } @@ -45,7 +44,7 @@ func (h *ClusterHandler) GetNodes(c echo.Context) error { return c.JSON(http.StatusOK, list) } -// GetNode returns the proxy node with the given ID +// NodeGet returns the proxy node with the given ID // @Summary List a proxy node by its ID // @Description List a proxy node by its ID // @Tags v16.?.? @@ -56,12 +55,12 @@ func (h *ClusterHandler) GetNodes(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id} [get] -func (h *ClusterHandler) GetNode(c echo.Context) error { +func (h *ClusterHandler) NodeGet(c echo.Context) error { id := util.PathParam(c, "id") about, _ := h.cluster.About() - nodes := h.cluster.ListNodes() + nodes := h.cluster.Store().NodeList() for _, node := range about.Nodes { if node.ID != id { @@ -70,7 +69,7 @@ func (h *ClusterHandler) GetNode(c echo.Context) error { if dbnode, hasNode := nodes[node.ID]; hasNode { if dbnode.State == "maintenance" { - node.Status = dbnode.State + node.State = dbnode.State } } @@ -80,7 +79,7 @@ func (h *ClusterHandler) GetNode(c echo.Context) error { return api.Err(http.StatusNotFound, "", "node not found") } -// GetNodeVersion returns the proxy node version with the given ID +// NodeGetVersion returns the proxy node version with the given ID // @Summary List a proxy node by its ID // @Description List a proxy node by its ID // @Tags v16.?.? @@ -91,29 +90,29 @@ func (h *ClusterHandler) GetNode(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id}/version [get] -func (h *ClusterHandler) GetNodeVersion(c echo.Context) error { +func (h *ClusterHandler) NodeGetVersion(c echo.Context) error { id := util.PathParam(c, "id") - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } - v := peer.Version() + v := peer.CoreAbout() version := api.Version{ - Number: v.Number, - Commit: v.Commit, - Branch: v.Branch, - Build: v.Build.Format(time.RFC3339), - Arch: v.Arch, - Compiler: v.Compiler, + Number: v.Version.Number, + Commit: v.Version.Commit, + Branch: v.Version.Branch, + Build: v.Version.Build.Format(time.RFC3339), + Arch: v.Version.Arch, + Compiler: v.Version.Compiler, } return c.JSON(http.StatusOK, version) } -// GetNodeResources returns the resources from the proxy node with the given ID +// NodeGetMedia returns the resources from the proxy node with the given ID // @Summary List the resources of a proxy node by its ID // @Description List the resources of a proxy node by its ID // @Tags v16.?.? @@ -124,10 +123,10 @@ func (h *ClusterHandler) GetNodeVersion(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id}/files [get] -func (h *ClusterHandler) GetNodeResources(c echo.Context) error { +func (h *ClusterHandler) NodeGetMedia(c echo.Context) error { id := util.PathParam(c, "id") - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } @@ -136,7 +135,7 @@ func (h *ClusterHandler) GetNodeResources(c echo.Context) error { Files: make(map[string][]string), } - peerFiles := peer.ListResources() + peerFiles := peer.Core().MediaList() files.LastUpdate = peerFiles.LastUpdate.Unix() @@ -176,12 +175,12 @@ func (h *ClusterHandler) NodeFSListFiles(c echo.Context) error { sortby := util.DefaultQuery(c, "sort", "none") order := util.DefaultQuery(c, "order", "asc") - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } - files, err := peer.ListFiles(name, pattern) + files, err := peer.Core().FilesystemList(name, pattern) if err != nil { return api.Err(http.StatusInternalServerError, "", "retrieving file list: %s", err.Error()) } @@ -234,12 +233,12 @@ func (h *ClusterHandler) NodeFSGetFile(c echo.Context) error { storage := util.PathParam(c, "storage") path := util.PathWildcardParam(c) - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } - file, err := peer.GetFile(storage, path, 0) + file, err := peer.Core().FilesystemGetFile(storage, path, 0) if err != nil { return api.Err(http.StatusNotFound, "", "%s", err.Error()) } @@ -270,14 +269,14 @@ func (h *ClusterHandler) NodeFSPutFile(c echo.Context) error { storage := util.PathParam(c, "storage") path := util.PathWildcardParam(c) - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } req := c.Request() - err = peer.PutFile(storage, path, req.Body) + err = peer.Core().FilesystemPutFile(storage, path, req.Body) if err != nil { return api.Err(http.StatusBadRequest, "", "%s", err.Error()) } @@ -303,12 +302,12 @@ func (h *ClusterHandler) NodeFSDeleteFile(c echo.Context) error { storage := util.PathParam(c, "storage") path := util.PathWildcardParam(c) - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } - err = peer.DeleteFile(storage, path) + err = peer.Core().FilesystemDeleteFile(storage, path) if err != nil { return api.Err(http.StatusNotFound, "", "%s", err.Error()) } @@ -316,7 +315,7 @@ func (h *ClusterHandler) NodeFSDeleteFile(c echo.Context) error { return c.JSON(http.StatusOK, nil) } -// ListNodeProcesses returns the list of processes running on a node of the cluster +// NodeListProcesses returns the list of processes running on a node of the cluster // @Summary List of processes in the cluster on a node // @Description List of processes in the cluster on a node // @Tags v16.?.? @@ -336,7 +335,7 @@ func (h *ClusterHandler) NodeFSDeleteFile(c echo.Context) error { // @Failure 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id}/process [get] -func (h *ClusterHandler) ListNodeProcesses(c echo.Context) error { +func (h *ClusterHandler) NodeListProcesses(c echo.Context) error { id := util.PathParam(c, "id") ctxuser := util.DefaultContext(c, "user", "") filter := strings.FieldsFunc(util.DefaultQuery(c, "filter", ""), func(r rune) bool { @@ -352,12 +351,12 @@ func (h *ClusterHandler) ListNodeProcesses(c echo.Context) error { ownerpattern := util.DefaultQuery(c, "ownerpattern", "") domainpattern := util.DefaultQuery(c, "domainpattern", "") - peer, err := h.proxy.GetNodeReader(id) + peer, err := h.proxy.NodeGet(id) if err != nil { return api.Err(http.StatusNotFound, "", "node not found: %s", err.Error()) } - procs, err := peer.ProcessList(proxy.ProcessListOptions{ + procs, err := peer.Core().ProcessList(node.ProcessListOptions{ ID: wantids, Filter: filter, Domain: domain, @@ -371,7 +370,7 @@ func (h *ClusterHandler) ListNodeProcesses(c echo.Context) error { return api.Err(http.StatusInternalServerError, "", "node not available: %s", err.Error()) } - processes := []clientapi.Process{} + processes := []api.Process{} for _, p := range procs { if !h.iam.Enforce(ctxuser, domain, "process", p.Config.ID, "read") { @@ -384,7 +383,7 @@ func (h *ClusterHandler) ListNodeProcesses(c echo.Context) error { return c.JSON(http.StatusOK, processes) } -// GetNodeState returns the state of a node with the given ID +// NodeGetState returns the state of a node with the given ID // @Summary Get the state of a node with the given ID // @Description Get the state of a node with the given ID // @Tags v16.?.? @@ -395,7 +394,7 @@ func (h *ClusterHandler) ListNodeProcesses(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id}/state [get] -func (h *ClusterHandler) GetNodeState(c echo.Context) error { +func (h *ClusterHandler) NodeGetState(c echo.Context) error { id := util.PathParam(c, "id") about, _ := h.cluster.About() @@ -406,7 +405,7 @@ func (h *ClusterHandler) GetNodeState(c echo.Context) error { continue } - state = node.Status + state = node.State break } @@ -414,7 +413,7 @@ func (h *ClusterHandler) GetNodeState(c echo.Context) error { return api.Err(http.StatusNotFound, "", "node not found") } - nodes := h.cluster.ListNodes() + nodes := h.cluster.Store().NodeList() if node, hasNode := nodes[id]; hasNode { if node.State == "maintenance" { state = node.State @@ -426,7 +425,7 @@ func (h *ClusterHandler) GetNodeState(c echo.Context) error { }) } -// SetNodeState sets the state of a node with the given ID +// NodeSetState sets the state of a node with the given ID // @Summary Set the state of a node with the given ID // @Description Set the state of a node with the given ID // @Tags v16.?.? @@ -440,7 +439,7 @@ func (h *ClusterHandler) GetNodeState(c echo.Context) error { // @Failure 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/node/{id}/state [put] -func (h *ClusterHandler) SetNodeState(c echo.Context) error { +func (h *ClusterHandler) NodeSetState(c echo.Context) error { id := util.PathParam(c, "id") about, _ := h.cluster.About() @@ -478,7 +477,7 @@ func (h *ClusterHandler) SetNodeState(c echo.Context) error { return c.JSON(http.StatusOK, "OK") } - err := h.cluster.SetNodeState("", id, state.State) + err := h.cluster.NodeSetState("", id, state.State) if err != nil { if errors.Is(err, cluster.ErrUnsupportedNodeState) { return api.Err(http.StatusBadRequest, "", "%s", err.Error()) diff --git a/http/handler/api/cluster_process.go b/http/handler/api/cluster_process.go index 81ca80fc..d4c12931 100644 --- a/http/handler/api/cluster_process.go +++ b/http/handler/api/cluster_process.go @@ -7,8 +7,7 @@ import ( "strconv" "strings" - clientapi "github.com/datarhei/core-client-go/v16/api" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" "github.com/datarhei/core/v16/cluster/store" "github.com/datarhei/core/v16/encoding/json" "github.com/datarhei/core/v16/glob" @@ -20,7 +19,7 @@ import ( "github.com/lithammer/shortuuid/v4" ) -// GetAllProcesses returns the list of processes running on all nodes of the cluster +// ProcessList returns the list of processes running on all nodes of the cluster // @Summary List of processes in the cluster // @Description List of processes in the cluster // @Tags v16.?.? @@ -37,7 +36,7 @@ import ( // @Success 200 {array} api.Process // @Security ApiKeyAuth // @Router /api/v3/cluster/process [get] -func (h *ClusterHandler) GetAllProcesses(c echo.Context) error { +func (h *ClusterHandler) ProcessList(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") filter := newFilter(util.DefaultQuery(c, "filter", "")) reference := util.DefaultQuery(c, "reference", "") @@ -50,7 +49,7 @@ func (h *ClusterHandler) GetAllProcesses(c echo.Context) error { ownerpattern := util.DefaultQuery(c, "ownerpattern", "") domainpattern := util.DefaultQuery(c, "domainpattern", "") - procs := h.proxy.ListProcesses(proxy.ProcessListOptions{ + procs := h.proxy.ProcessList(node.ProcessListOptions{ ID: wantids, Filter: filter.Slice(), Domain: domain, @@ -61,7 +60,7 @@ func (h *ClusterHandler) GetAllProcesses(c echo.Context) error { DomainPattern: domainpattern, }) - processes := []clientapi.Process{} + processes := []api.Process{} pmap := map[app.ProcessID]struct{}{} for _, p := range procs { @@ -77,7 +76,7 @@ func (h *ClusterHandler) GetAllProcesses(c echo.Context) error { // Here we have to add those processes that are in the cluster DB and couldn't be deployed { - processes := h.cluster.ListProcesses() + processes := h.cluster.Store().ProcessList() filtered := h.getFilteredStoreProcesses(processes, wantids, domain, reference, idpattern, refpattern, ownerpattern, domainpattern) for _, p := range filtered { @@ -139,7 +138,7 @@ func (h *ClusterHandler) GetAllProcesses(c echo.Context) error { return c.Stream(http.StatusOK, "application/json", buf) } -func (h *ClusterHandler) getFilteredStoreProcesses(processes []store.Process, wantids []string, domain, reference, idpattern, refpattern, ownerpattern, domainpattern string) []store.Process { +func (h *ClusterHandler) getFilteredStoreProcesses(processes []store.Process, wantids []string, _, reference, idpattern, refpattern, ownerpattern, domainpattern string) []store.Process { filtered := []store.Process{} count := 0 @@ -293,7 +292,7 @@ func (h *ClusterHandler) convertStoreProcessToAPIProcess(p store.Process, filter return process } -// GetProcess returns the process with the given ID whereever it's running on the cluster +// ProcessGet returns the process with the given ID whereever it's running on the cluster // @Summary List a process by its ID // @Description List a process by its ID. Use the filter parameter to specifiy the level of detail of the output. // @Tags v16.?.? @@ -307,7 +306,7 @@ func (h *ClusterHandler) convertStoreProcessToAPIProcess(p store.Process, filter // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id} [get] -func (h *ClusterHandler) GetProcess(c echo.Context) error { +func (h *ClusterHandler) ProcessGet(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") id := util.PathParam(c, "id") filter := newFilter(util.DefaultQuery(c, "filter", "")) @@ -317,7 +316,7 @@ func (h *ClusterHandler) GetProcess(c echo.Context) error { return api.Err(http.StatusForbidden, "") } - procs := h.proxy.ListProcesses(proxy.ProcessListOptions{ + procs := h.proxy.ProcessList(node.ProcessListOptions{ ID: []string{id}, Filter: filter.Slice(), Domain: domain, @@ -325,7 +324,7 @@ func (h *ClusterHandler) GetProcess(c echo.Context) error { if len(procs) == 0 { // Check the store in the cluster for an undeployed process - p, err := h.cluster.GetProcess(app.NewProcessID(id, domain)) + p, err := h.cluster.Store().ProcessGet(app.NewProcessID(id, domain)) if err != nil { return api.Err(http.StatusNotFound, "", "Unknown process ID: %s", id) } @@ -355,7 +354,7 @@ func (h *ClusterHandler) GetProcess(c echo.Context) error { // @Failure 403 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process [post] -func (h *ClusterHandler) AddProcess(c echo.Context) error { +func (h *ClusterHandler) ProcessAdd(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) @@ -390,12 +389,12 @@ func (h *ClusterHandler) AddProcess(c echo.Context) error { config, metadata := process.Marshal() - if err := h.cluster.AddProcess("", config); err != nil { + if err := h.cluster.ProcessAdd("", config); err != nil { return api.Err(http.StatusBadRequest, "", "adding process config: %s", err.Error()) } for key, value := range metadata { - h.cluster.SetProcessMetadata("", config.ProcessID(), key, value) + h.cluster.ProcessSetMetadata("", config.ProcessID(), key, value) } return c.JSON(http.StatusOK, process) @@ -417,7 +416,7 @@ func (h *ClusterHandler) AddProcess(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id} [put] -func (h *ClusterHandler) UpdateProcess(c echo.Context) error { +func (h *ClusterHandler) ProcessUpdate(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") superuser := util.DefaultContext(c, "superuser", false) domain := util.DefaultQuery(c, "domain", "") @@ -437,7 +436,7 @@ func (h *ClusterHandler) UpdateProcess(c echo.Context) error { pid := process.ProcessID() - current, err := h.cluster.GetProcess(pid) + current, err := h.cluster.Store().ProcessGet(pid) if err != nil { return api.Err(http.StatusNotFound, "", "process not found: %s in domain '%s'", pid.ID, pid.Domain) } @@ -461,7 +460,7 @@ func (h *ClusterHandler) UpdateProcess(c echo.Context) error { config, metadata := process.Marshal() - if err := h.cluster.UpdateProcess("", pid, config); err != nil { + if err := h.cluster.ProcessUpdate("", pid, config); err != nil { if err == restream.ErrUnknownProcess { return api.Err(http.StatusNotFound, "", "process not found: %s in domain '%s'", pid.ID, pid.Domain) } @@ -472,7 +471,7 @@ func (h *ClusterHandler) UpdateProcess(c echo.Context) error { pid = process.ProcessID() for key, value := range metadata { - h.cluster.SetProcessMetadata("", pid, key, value) + h.cluster.ProcessSetMetadata("", pid, key, value) } return c.JSON(http.StatusOK, process) @@ -494,7 +493,7 @@ func (h *ClusterHandler) UpdateProcess(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id}/command [put] -func (h *ClusterHandler) SetProcessCommand(c echo.Context) error { +func (h *ClusterHandler) ProcessSetCommand(c echo.Context) error { id := util.PathParam(c, "id") ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") @@ -523,14 +522,14 @@ func (h *ClusterHandler) SetProcessCommand(c echo.Context) error { return api.Err(http.StatusBadRequest, "", "unknown command provided. known commands are: start, stop, reload, restart") } - if err := h.cluster.SetProcessCommand("", pid, command.Command); err != nil { + if err := h.cluster.ProcessSetCommand("", pid, command.Command); err != nil { return api.Err(http.StatusNotFound, "", "command failed: %s", err.Error()) } return c.JSON(http.StatusOK, "OK") } -// SetProcessMetadata stores metadata with a process +// ProcessSetMetadata stores metadata with a process // @Summary Add JSON metadata with a process under the given key // @Description Add arbitrary JSON metadata under the given key. If the key exists, all already stored metadata with this key will be overwritten. If the key doesn't exist, it will be created. // @Tags v16.?.? @@ -546,7 +545,7 @@ func (h *ClusterHandler) SetProcessCommand(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id}/metadata/{key} [put] -func (h *ClusterHandler) SetProcessMetadata(c echo.Context) error { +func (h *ClusterHandler) ProcessSetMetadata(c echo.Context) error { id := util.PathParam(c, "id") key := util.PathParam(c, "key") ctxuser := util.DefaultContext(c, "user", "") @@ -571,14 +570,14 @@ func (h *ClusterHandler) SetProcessMetadata(c echo.Context) error { Domain: domain, } - if err := h.cluster.SetProcessMetadata("", pid, key, data); err != nil { + if err := h.cluster.ProcessSetMetadata("", pid, key, data); err != nil { return api.Err(http.StatusNotFound, "", "setting metadata failed: %s", err.Error()) } return c.JSON(http.StatusOK, data) } -// GetProcessMetadata returns the metadata stored with a process +// ProcessGetMetadata returns the metadata stored with a process // @Summary Retrieve JSON metadata stored with a process under a key // @Description Retrieve the previously stored JSON metadata under the given key. If the key is empty, all metadata will be returned. // @Tags v16.?.? @@ -593,7 +592,7 @@ func (h *ClusterHandler) SetProcessMetadata(c echo.Context) error { // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id}/metadata/{key} [get] -func (h *ClusterHandler) GetProcessMetadata(c echo.Context) error { +func (h *ClusterHandler) ProcessGetMetadata(c echo.Context) error { id := util.PathParam(c, "id") key := util.PathParam(c, "key") ctxuser := util.DefaultContext(c, "user", "") @@ -608,7 +607,7 @@ func (h *ClusterHandler) GetProcessMetadata(c echo.Context) error { Domain: domain, } - data, err := h.cluster.GetProcessMetadata("", pid, key) + data, err := h.cluster.ProcessGetMetadata("", pid, key) if err != nil { return api.Err(http.StatusNotFound, "", "unknown process ID: %s", err.Error()) } @@ -628,7 +627,7 @@ func (h *ClusterHandler) GetProcessMetadata(c echo.Context) error { // @Failure 403 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id}/probe [get] -func (h *ClusterHandler) ProbeProcess(c echo.Context) error { +func (h *ClusterHandler) ProcessProbe(c echo.Context) error { id := util.PathParam(c, "id") ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") @@ -642,14 +641,14 @@ func (h *ClusterHandler) ProbeProcess(c echo.Context) error { Domain: domain, } - nodeid, err := h.proxy.FindNodeFromProcess(pid) + nodeid, err := h.proxy.ProcessFindNodeID(pid) if err != nil { return c.JSON(http.StatusOK, api.Probe{ Log: []string{fmt.Sprintf("the process can't be found: %s", err.Error())}, }) } - probe, _ := h.proxy.ProbeProcess(nodeid, pid) + probe, _ := h.proxy.ProcessProbe(nodeid, pid) return c.JSON(http.StatusOK, probe) } @@ -669,7 +668,7 @@ func (h *ClusterHandler) ProbeProcess(c echo.Context) error { // @Failure 500 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/probe [post] -func (h *ClusterHandler) ProbeProcessConfig(c echo.Context) error { +func (h *ClusterHandler) ProcessProbeConfig(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") coreid := util.DefaultQuery(c, "coreid", "") @@ -702,12 +701,12 @@ func (h *ClusterHandler) ProbeProcessConfig(c echo.Context) error { config, _ := process.Marshal() - coreid = h.proxy.FindNodeFromResources(coreid, config.LimitCPU, config.LimitMemory) + coreid = h.proxy.FindNodeForResources(coreid, config.LimitCPU, config.LimitMemory) if len(coreid) == 0 { - return api.Err(http.StatusInternalServerError, "", "Not enough available resources available to execute probe") + return api.Err(http.StatusInternalServerError, "", "Not enough resources available to execute probe") } - probe, _ := h.proxy.ProbeProcessConfig(coreid, config) + probe, _ := h.proxy.ProcessProbeConfig(coreid, config) return c.JSON(http.StatusOK, probe) } @@ -724,7 +723,7 @@ func (h *ClusterHandler) ProbeProcessConfig(c echo.Context) error { // @Failure 403 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/process/{id} [delete] -func (h *ClusterHandler) DeleteProcess(c echo.Context) error { +func (h *ClusterHandler) ProcessDelete(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") id := util.PathParam(c, "id") @@ -738,7 +737,7 @@ func (h *ClusterHandler) DeleteProcess(c echo.Context) error { Domain: domain, } - if err := h.cluster.RemoveProcess("", pid); err != nil { + if err := h.cluster.ProcessRemove("", pid); err != nil { return api.Err(http.StatusBadRequest, "", "%s", err.Error()) } diff --git a/http/handler/api/cluster_store.go b/http/handler/api/cluster_store.go index 5c7d121b..8d0c7349 100644 --- a/http/handler/api/cluster_store.go +++ b/http/handler/api/cluster_store.go @@ -12,7 +12,7 @@ import ( "github.com/labstack/echo/v4" ) -// ListStoreProcesses returns the list of processes stored in the DB of the cluster +// StoreListProcesses returns the list of processes stored in the DB of the cluster // @Summary List of processes in the cluster DB // @Description List of processes in the cluster DB // @Tags v16.?.? @@ -21,10 +21,10 @@ import ( // @Success 200 {array} api.Process // @Security ApiKeyAuth // @Router /api/v3/cluster/db/process [get] -func (h *ClusterHandler) ListStoreProcesses(c echo.Context) error { +func (h *ClusterHandler) StoreListProcesses(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") - procs := h.cluster.ListProcesses() + procs := h.cluster.Store().ProcessList() processes := []api.Process{} @@ -52,7 +52,7 @@ func (h *ClusterHandler) ListStoreProcesses(c echo.Context) error { // @Success 200 {object} api.Process // @Security ApiKeyAuth // @Router /api/v3/cluster/db/process/:id [get] -func (h *ClusterHandler) GetStoreProcess(c echo.Context) error { +func (h *ClusterHandler) StoreGetProcess(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") id := util.PathParam(c, "id") @@ -66,7 +66,7 @@ func (h *ClusterHandler) GetStoreProcess(c echo.Context) error { return api.Err(http.StatusForbidden, "", "API user %s is not allowed to read this process", ctxuser) } - p, err := h.cluster.GetProcess(pid) + p, err := h.cluster.Store().ProcessGet(pid) if err != nil { return api.Err(http.StatusNotFound, "", "process not found: %s in domain '%s'", pid.ID, pid.Domain) } @@ -76,7 +76,7 @@ func (h *ClusterHandler) GetStoreProcess(c echo.Context) error { return c.JSON(http.StatusOK, process) } -// GetStoreProcessNodeMap returns a map of which process is running on which node +// StoreGetProcessNodeMap returns a map of which process is running on which node // @Summary Retrieve a map of which process is running on which node // @Description Retrieve a map of which process is running on which node // @Tags v16.?.? @@ -85,13 +85,13 @@ func (h *ClusterHandler) GetStoreProcess(c echo.Context) error { // @Success 200 {object} api.ClusterProcessMap // @Security ApiKeyAuth // @Router /api/v3/cluster/map/process [get] -func (h *ClusterHandler) GetStoreProcessNodeMap(c echo.Context) error { - m := h.cluster.GetProcessNodeMap() +func (h *ClusterHandler) StoreGetProcessNodeMap(c echo.Context) error { + m := h.cluster.Store().ProcessGetNodeMap() return c.JSON(http.StatusOK, m) } -// ListStoreIdentities returns the list of identities stored in the DB of the cluster +// StoreListIdentities returns the list of identities stored in the DB of the cluster // @Summary List of identities in the cluster // @Description List of identities in the cluster // @Tags v16.?.? @@ -100,15 +100,15 @@ func (h *ClusterHandler) GetStoreProcessNodeMap(c echo.Context) error { // @Success 200 {array} api.IAMUser // @Security ApiKeyAuth // @Router /api/v3/cluster/db/user [get] -func (h *ClusterHandler) ListStoreIdentities(c echo.Context) error { +func (h *ClusterHandler) StoreListIdentities(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") - updatedAt, identities := h.cluster.ListIdentities() + identities := h.cluster.Store().IAMIdentityList() - users := make([]api.IAMUser, len(identities)) + users := make([]api.IAMUser, len(identities.Users)) - for i, iamuser := range identities { + for i, iamuser := range identities.Users { if !h.iam.Enforce(ctxuser, domain, "iam", iamuser.Name, "read") { continue } @@ -119,27 +119,27 @@ func (h *ClusterHandler) ListStoreIdentities(c echo.Context) error { } } - _, policies := h.cluster.ListUserPolicies(iamuser.Name) - users[i].Marshal(iamuser, policies) + policies := h.cluster.Store().IAMIdentityPolicyList(iamuser.Name) + users[i].Marshal(iamuser, policies.Policies) } - c.Response().Header().Set("Last-Modified", updatedAt.UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + c.Response().Header().Set("Last-Modified", identities.UpdatedAt.UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) return c.JSON(http.StatusOK, users) } -// ListStoreIdentity returns the list of identities stored in the DB of the cluster +// StoreGetIdentity returns the list of identities stored in the DB of the cluster // @Summary List of identities in the cluster // @Description List of identities in the cluster // @Tags v16.?.? -// @ID cluster-3-db-list-identity +// @ID cluster-3-db-get-identity // @Produce json // @Success 200 {object} api.IAMUser // @Failure 403 {object} api.Error // @Failure 404 {object} api.Error // @Security ApiKeyAuth // @Router /api/v3/cluster/db/user/{name} [get] -func (h *ClusterHandler) ListStoreIdentity(c echo.Context) error { +func (h *ClusterHandler) StoreGetIdentity(c echo.Context) error { ctxuser := util.DefaultContext(c, "user", "") domain := util.DefaultQuery(c, "domain", "") name := util.PathParam(c, "name") @@ -150,14 +150,15 @@ func (h *ClusterHandler) ListStoreIdentity(c echo.Context) error { var updatedAt time.Time var iamuser identity.User - var err error if name != "$anon" { - updatedAt, iamuser, err = h.cluster.ListIdentity(name) - if err != nil { - return api.Err(http.StatusNotFound, "", "%s", err.Error()) + user := h.cluster.Store().IAMIdentityGet(name) + if len(user.Users) == 0 { + return api.Err(http.StatusNotFound, "") } + updatedAt, iamuser = user.UpdatedAt, user.Users[0] + if ctxuser != iamuser.Name { if !h.iam.Enforce(ctxuser, domain, "iam", name, "write") { iamuser = identity.User{ @@ -171,20 +172,20 @@ func (h *ClusterHandler) ListStoreIdentity(c echo.Context) error { } } - policiesUpdatedAt, policies := h.cluster.ListUserPolicies(name) + policies := h.cluster.Store().IAMIdentityPolicyList(name) if updatedAt.IsZero() { - updatedAt = policiesUpdatedAt + updatedAt = policies.UpdatedAt } user := api.IAMUser{} - user.Marshal(iamuser, policies) + user.Marshal(iamuser, policies.Policies) c.Response().Header().Set("Last-Modified", updatedAt.UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) return c.JSON(http.StatusOK, user) } -// ListStorePolicies returns the list of policies stored in the DB of the cluster +// StoreListPolicies returns the list of policies stored in the DB of the cluster // @Summary List of policies in the cluster // @Description List of policies in the cluster // @Tags v16.?.? @@ -193,26 +194,27 @@ func (h *ClusterHandler) ListStoreIdentity(c echo.Context) error { // @Success 200 {array} api.IAMPolicy // @Security ApiKeyAuth // @Router /api/v3/cluster/db/policies [get] -func (h *ClusterHandler) ListStorePolicies(c echo.Context) error { - updatedAt, clusterpolicies := h.cluster.ListPolicies() +func (h *ClusterHandler) StoreListPolicies(c echo.Context) error { + clusterpolicies := h.cluster.Store().IAMPolicyList() policies := []api.IAMPolicy{} - for _, pol := range clusterpolicies { + for _, pol := range clusterpolicies.Policies { policies = append(policies, api.IAMPolicy{ Name: pol.Name, Domain: pol.Domain, Resource: pol.Resource, + Types: pol.Types, Actions: pol.Actions, }) } - c.Response().Header().Set("Last-Modified", updatedAt.UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + c.Response().Header().Set("Last-Modified", clusterpolicies.UpdatedAt.UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")) return c.JSON(http.StatusOK, policies) } -// ListStoreLocks returns the list of currently stored locks +// StoreListLocks returns the list of currently stored locks // @Summary List locks in the cluster DB // @Description List of locks in the cluster DB // @Tags v16.?.? @@ -221,8 +223,8 @@ func (h *ClusterHandler) ListStorePolicies(c echo.Context) error { // @Success 200 {array} api.ClusterLock // @Security ApiKeyAuth // @Router /api/v3/cluster/db/locks [get] -func (h *ClusterHandler) ListStoreLocks(c echo.Context) error { - clusterlocks := h.cluster.ListLocks() +func (h *ClusterHandler) StoreListLocks(c echo.Context) error { + clusterlocks := h.cluster.Store().LockList() locks := []api.ClusterLock{} @@ -236,7 +238,7 @@ func (h *ClusterHandler) ListStoreLocks(c echo.Context) error { return c.JSON(http.StatusOK, locks) } -// ListStoreKV returns the list of currently stored key/value pairs +// StoreListKV returns the list of currently stored key/value pairs // @Summary List KV in the cluster DB // @Description List of KV in the cluster DB // @Tags v16.?.? @@ -245,8 +247,8 @@ func (h *ClusterHandler) ListStoreLocks(c echo.Context) error { // @Success 200 {object} api.ClusterKVS // @Security ApiKeyAuth // @Router /api/v3/cluster/db/kv [get] -func (h *ClusterHandler) ListStoreKV(c echo.Context) error { - clusterkv := h.cluster.ListKV("") +func (h *ClusterHandler) StoreListKV(c echo.Context) error { + clusterkv := h.cluster.Store().KVSList("") kvs := api.ClusterKVS{} @@ -260,7 +262,7 @@ func (h *ClusterHandler) ListStoreKV(c echo.Context) error { return c.JSON(http.StatusOK, kvs) } -// ListStoreNodes returns the list of stored node metadata +// StoreListNodes returns the list of stored node metadata // @Summary List nodes in the cluster DB // @Description List of nodes in the cluster DB // @Tags v16.?.? @@ -269,8 +271,8 @@ func (h *ClusterHandler) ListStoreKV(c echo.Context) error { // @Success 200 {array} api.ClusterStoreNode // @Security ApiKeyAuth // @Router /api/v3/cluster/db/node [get] -func (h *ClusterHandler) ListStoreNodes(c echo.Context) error { - clusternodes := h.cluster.ListNodes() +func (h *ClusterHandler) StoreListNodes(c echo.Context) error { + clusternodes := h.cluster.Store().NodeList() nodes := []api.ClusterStoreNode{} diff --git a/http/handler/api/iam.go b/http/handler/api/iam.go index 731d809e..a6af1f68 100644 --- a/http/handler/api/iam.go +++ b/http/handler/api/iam.go @@ -84,6 +84,7 @@ func (h *IAMHandler) AddIdentity(c echo.Context) error { // @Param name path string true "Username" // @Param domain query string false "Domain of the acting user" // @Success 200 {string} string +// @Failure 400 {object} api.Error // @Failure 403 {object} api.Error // @Failure 404 {object} api.Error // @Failure 500 {object} api.Error diff --git a/http/server.go b/http/server.go index b30782ca..f58b04d0 100644 --- a/http/server.go +++ b/http/server.go @@ -206,7 +206,7 @@ func NewServer(config Config) (serverhandler.Server, error) { if config.Cluster != nil { if httpfs.Filesystem.Type() == "disk" || httpfs.Filesystem.Type() == "mem" { - httpfs.Filesystem = fs.NewClusterFS(httpfs.Filesystem.Name(), httpfs.Filesystem, config.Cluster.ProxyReader()) + httpfs.Filesystem = fs.NewClusterFS(httpfs.Filesystem.Name(), httpfs.Filesystem, config.Cluster.Manager()) } } @@ -728,59 +728,59 @@ func (s *server) setRoutesV3(v3 *echo.Group) { v3.GET("/cluster/snapshot", s.v3handler.cluster.GetSnapshot) - v3.GET("/cluster/db/process", s.v3handler.cluster.ListStoreProcesses) - v3.GET("/cluster/db/process/:id", s.v3handler.cluster.GetStoreProcess) - v3.GET("/cluster/db/user", s.v3handler.cluster.ListStoreIdentities) - v3.GET("/cluster/db/user/:name", s.v3handler.cluster.ListStoreIdentity) - v3.GET("/cluster/db/policies", s.v3handler.cluster.ListStorePolicies) - v3.GET("/cluster/db/locks", s.v3handler.cluster.ListStoreLocks) - v3.GET("/cluster/db/kv", s.v3handler.cluster.ListStoreKV) - v3.GET("/cluster/db/map/process", s.v3handler.cluster.GetStoreProcessNodeMap) - v3.GET("/cluster/db/node", s.v3handler.cluster.ListStoreNodes) + v3.GET("/cluster/db/process", s.v3handler.cluster.StoreListProcesses) + v3.GET("/cluster/db/process/:id", s.v3handler.cluster.StoreGetProcess) + v3.GET("/cluster/db/user", s.v3handler.cluster.StoreListIdentities) + v3.GET("/cluster/db/user/:name", s.v3handler.cluster.StoreGetIdentity) + v3.GET("/cluster/db/policies", s.v3handler.cluster.StoreListPolicies) + v3.GET("/cluster/db/locks", s.v3handler.cluster.StoreListLocks) + v3.GET("/cluster/db/kv", s.v3handler.cluster.StoreListKV) + v3.GET("/cluster/db/map/process", s.v3handler.cluster.StoreGetProcessNodeMap) + v3.GET("/cluster/db/node", s.v3handler.cluster.StoreListNodes) - v3.GET("/cluster/iam/user", s.v3handler.cluster.ListIdentities) - v3.GET("/cluster/iam/user/:name", s.v3handler.cluster.ListIdentity) - v3.GET("/cluster/iam/policies", s.v3handler.cluster.ListPolicies) + v3.GET("/cluster/iam/user", s.v3handler.cluster.IAMIdentityList) + v3.GET("/cluster/iam/user/:name", s.v3handler.cluster.IAMIdentityGet) + v3.GET("/cluster/iam/policies", s.v3handler.cluster.IAMPolicyList) - v3.GET("/cluster/process", s.v3handler.cluster.GetAllProcesses) - v3.GET("/cluster/process/:id", s.v3handler.cluster.GetProcess) - v3.GET("/cluster/process/:id/metadata", s.v3handler.cluster.GetProcessMetadata) - v3.GET("/cluster/process/:id/metadata/:key", s.v3handler.cluster.GetProcessMetadata) + v3.GET("/cluster/process", s.v3handler.cluster.ProcessList) + v3.GET("/cluster/process/:id", s.v3handler.cluster.ProcessGet) + v3.GET("/cluster/process/:id/metadata", s.v3handler.cluster.ProcessGetMetadata) + v3.GET("/cluster/process/:id/metadata/:key", s.v3handler.cluster.ProcessGetMetadata) - v3.GET("/cluster/node", s.v3handler.cluster.GetNodes) - v3.GET("/cluster/node/:id", s.v3handler.cluster.GetNode) - v3.GET("/cluster/node/:id/files", s.v3handler.cluster.GetNodeResources) + v3.GET("/cluster/node", s.v3handler.cluster.NodeList) + v3.GET("/cluster/node/:id", s.v3handler.cluster.NodeGet) + v3.GET("/cluster/node/:id/files", s.v3handler.cluster.NodeGetMedia) v3.GET("/cluster/node/:id/fs/:storage", s.v3handler.cluster.NodeFSListFiles) v3.GET("/cluster/node/:id/fs/:storage/*", s.v3handler.cluster.NodeFSGetFile) - v3.GET("/cluster/node/:id/process", s.v3handler.cluster.ListNodeProcesses) - v3.GET("/cluster/node/:id/version", s.v3handler.cluster.GetNodeVersion) - v3.GET("/cluster/node/:id/state", s.v3handler.cluster.GetNodeState) + v3.GET("/cluster/node/:id/process", s.v3handler.cluster.NodeListProcesses) + v3.GET("/cluster/node/:id/version", s.v3handler.cluster.NodeGetVersion) + v3.GET("/cluster/node/:id/state", s.v3handler.cluster.NodeGetState) - v3.GET("/cluster/fs/:storage", s.v3handler.cluster.ListFiles) + v3.GET("/cluster/fs/:storage", s.v3handler.cluster.FilesystemListFiles) if !s.readOnly { v3.PUT("/cluster/transfer/:id", s.v3handler.cluster.TransferLeadership) v3.PUT("/cluster/leave", s.v3handler.cluster.Leave) - v3.POST("/cluster/process", s.v3handler.cluster.AddProcess) - v3.POST("/cluster/process/probe", s.v3handler.cluster.ProbeProcessConfig) - v3.PUT("/cluster/process/:id", s.v3handler.cluster.UpdateProcess) - v3.GET("/cluster/process/:id/probe", s.v3handler.cluster.ProbeProcess) - v3.DELETE("/cluster/process/:id", s.v3handler.cluster.DeleteProcess) - v3.PUT("/cluster/process/:id/command", s.v3handler.cluster.SetProcessCommand) - v3.PUT("/cluster/process/:id/metadata/:key", s.v3handler.cluster.SetProcessMetadata) + v3.POST("/cluster/process", s.v3handler.cluster.ProcessAdd) + v3.POST("/cluster/process/probe", s.v3handler.cluster.ProcessProbeConfig) + v3.PUT("/cluster/process/:id", s.v3handler.cluster.ProcessUpdate) + v3.GET("/cluster/process/:id/probe", s.v3handler.cluster.ProcessProbe) + v3.DELETE("/cluster/process/:id", s.v3handler.cluster.ProcessDelete) + v3.PUT("/cluster/process/:id/command", s.v3handler.cluster.ProcessSetCommand) + v3.PUT("/cluster/process/:id/metadata/:key", s.v3handler.cluster.ProcessSetMetadata) v3.PUT("/cluster/reallocation", s.v3handler.cluster.Reallocation) v3.DELETE("/cluster/node/:id/fs/:storage/*", s.v3handler.cluster.NodeFSDeleteFile) v3.PUT("/cluster/node/:id/fs/:storage/*", s.v3handler.cluster.NodeFSPutFile) - v3.PUT("/cluster/node/:id/state", s.v3handler.cluster.SetNodeState) + v3.PUT("/cluster/node/:id/state", s.v3handler.cluster.NodeSetState) - v3.PUT("/cluster/iam/reload", s.v3handler.cluster.ReloadIAM) - v3.POST("/cluster/iam/user", s.v3handler.cluster.AddIdentity) - v3.PUT("/cluster/iam/user/:name", s.v3handler.cluster.UpdateIdentity) - v3.PUT("/cluster/iam/user/:name/policy", s.v3handler.cluster.UpdateIdentityPolicies) - v3.DELETE("/cluster/iam/user/:name", s.v3handler.cluster.RemoveIdentity) + v3.PUT("/cluster/iam/reload", s.v3handler.cluster.IAMReload) + v3.POST("/cluster/iam/user", s.v3handler.cluster.IAMIdentityAdd) + v3.PUT("/cluster/iam/user/:name", s.v3handler.cluster.IAMIdentityUpdate) + v3.PUT("/cluster/iam/user/:name/policy", s.v3handler.cluster.IAMIdentityUpdatePolicies) + v3.DELETE("/cluster/iam/user/:name", s.v3handler.cluster.IAMIdentityRemove) } } diff --git a/internal/testhelper/ffmpeg/ffmpeg.go b/internal/testhelper/ffmpeg/ffmpeg.go index 210673cb..23d98b0b 100644 --- a/internal/testhelper/ffmpeg/ffmpeg.go +++ b/internal/testhelper/ffmpeg/ffmpeg.go @@ -139,7 +139,7 @@ Output #0, hls, to './data/testsrc.m3u8': os.Exit(2) } - if slices.EqualComparableElements(os.Args[1:], []string{"-f", "avfoundation", "-list_devices", "true", "-i", ""}) { + if err := slices.EqualComparableElements(os.Args[1:], []string{"-f", "avfoundation", "-list_devices", "true", "-i", ""}); err == nil { fmt.Fprintf(os.Stderr, "%s\n", avfoundation) os.Exit(0) } diff --git a/resources/resources.go b/resources/resources.go index 6986cdb2..5d419824 100644 --- a/resources/resources.go +++ b/resources/resources.go @@ -10,6 +10,31 @@ import ( "github.com/datarhei/core/v16/psutil" ) +type Info struct { + Mem MemoryInfo + CPU CPUInfo +} + +type MemoryInfo struct { + Total uint64 // bytes + Available uint64 // bytes + Used uint64 // bytes + Limit uint64 // bytes + Throttling bool + Error error +} + +type CPUInfo struct { + NCPU float64 // number of cpus + System float64 // percent 0-100 + User float64 // percent 0-100 + Idle float64 // percent 0-100 + Other float64 // percent 0-100 + Limit float64 // percent 0-100 + Throttling bool + Error error +} + type resources struct { psutil psutil.Util @@ -34,16 +59,20 @@ type Resources interface { Start() Stop() - // HasLimits returns whether any limits have been set + // HasLimits returns whether any limits have been set. HasLimits() bool - // Limits returns the CPU (percent 0-100) and memory (bytes) limits + // Limits returns the CPU (percent 0-100) and memory (bytes) limits. Limits() (float64, uint64) - // ShouldLimit returns whether cpu and/or memory is currently limited + // ShouldLimit returns whether cpu and/or memory is currently limited. ShouldLimit() (bool, bool) + // Request checks whether the requested resources are available. Request(cpu float64, memory uint64) error + + // Info returns the current resource usage + Info() Info } type Config struct { @@ -290,3 +319,38 @@ func (r *resources) Request(cpu float64, memory uint64) error { return nil } + +func (r *resources) Info() Info { + cpulimit, memlimit := r.Limits() + cputhrottling, memthrottling := r.ShouldLimit() + + cpustat, cpuerr := r.psutil.CPUPercent() + memstat, memerr := r.psutil.VirtualMemory() + + cpuinfo := CPUInfo{ + NCPU: r.ncpu, + System: cpustat.System, + User: cpustat.User, + Idle: cpustat.Idle, + Other: cpustat.Other, + Limit: cpulimit, + Throttling: cputhrottling, + Error: cpuerr, + } + + meminfo := MemoryInfo{ + Total: memstat.Total, + Available: memstat.Available, + Used: memstat.Used, + Limit: memlimit, + Throttling: memthrottling, + Error: memerr, + } + + i := Info{ + CPU: cpuinfo, + Mem: meminfo, + } + + return i +} diff --git a/restream/app/metadata.go b/restream/app/metadata.go new file mode 100644 index 00000000..42e9dcc9 --- /dev/null +++ b/restream/app/metadata.go @@ -0,0 +1,3 @@ +package app + +type Metadata interface{} diff --git a/restream/app/process.go b/restream/app/process.go index b0e9e5f2..f48bec38 100644 --- a/restream/app/process.go +++ b/restream/app/process.go @@ -3,6 +3,7 @@ package app import ( "bytes" "crypto/md5" + "encoding/json" "strconv" "strings" @@ -143,6 +144,15 @@ func (config *Config) CreateCommand() []string { return command } +func (config *Config) String() string { + data, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err.Error() + } + + return string(data) +} + func (config *Config) Hash() []byte { b := bytes.Buffer{} diff --git a/restream/app/log.go b/restream/app/report.go similarity index 67% rename from restream/app/log.go rename to restream/app/report.go index 470db445..8bc80848 100644 --- a/restream/app/log.go +++ b/restream/app/report.go @@ -9,15 +9,15 @@ type LogLine struct { Data string } -type LogEntry struct { +type ReportEntry struct { CreatedAt time.Time Prelude []string Log []LogLine Matches []string } -type LogHistoryEntry struct { - LogEntry +type ReportHistoryEntry struct { + ReportEntry ExitedAt time.Time ExitState string @@ -25,12 +25,12 @@ type LogHistoryEntry struct { Usage ProcessUsage } -type Log struct { - LogEntry - History []LogHistoryEntry +type Report struct { + ReportEntry + History []ReportHistoryEntry } -type LogHistorySearchResult struct { +type ReportHistorySearchResult struct { ProcessID string Reference string ExitState string diff --git a/restream/restream.go b/restream/restream.go index 6fd5338c..365959c4 100644 --- a/restream/restream.go +++ b/restream/restream.go @@ -45,21 +45,21 @@ type Restreamer interface { SetMetadata(key string, data interface{}) error // Set general metadata GetMetadata(key string) (interface{}, error) // Get previously set general metadata - AddProcess(config *app.Config) error // Add a new process - GetProcessIDs(idpattern, refpattern, ownerpattern, domainpattern string) []app.ProcessID // Get a list of process IDs based on patterns for ID and reference - DeleteProcess(id app.ProcessID) error // Delete a process - UpdateProcess(id app.ProcessID, config *app.Config) error // Update a process - StartProcess(id app.ProcessID) error // Start a process - StopProcess(id app.ProcessID) error // Stop a process - RestartProcess(id app.ProcessID) error // Restart a process - ReloadProcess(id app.ProcessID) error // Reload a process - GetProcess(id app.ProcessID) (*app.Process, error) // Get a process - GetProcessState(id app.ProcessID) (*app.State, error) // Get the state of a process - GetProcessLog(id app.ProcessID) (*app.Log, error) // Get the logs of a process - SearchProcessLogHistory(idpattern, refpattern, state string, from, to *time.Time) []app.LogHistorySearchResult // Search the log history of all processes - GetPlayout(id app.ProcessID, inputid string) (string, error) // Get the URL of the playout API for a process - SetProcessMetadata(id app.ProcessID, key string, data interface{}) error // Set metatdata to a process - GetProcessMetadata(id app.ProcessID, key string) (interface{}, error) // Get previously set metadata from a process + AddProcess(config *app.Config) error // Add a new process + GetProcessIDs(idpattern, refpattern, ownerpattern, domainpattern string) []app.ProcessID // Get a list of process IDs based on patterns for ID and reference + DeleteProcess(id app.ProcessID) error // Delete a process + UpdateProcess(id app.ProcessID, config *app.Config) error // Update a process + StartProcess(id app.ProcessID) error // Start a process + StopProcess(id app.ProcessID) error // Stop a process + RestartProcess(id app.ProcessID) error // Restart a process + ReloadProcess(id app.ProcessID) error // Reload a process + GetProcess(id app.ProcessID) (*app.Process, error) // Get a process + GetProcessState(id app.ProcessID) (*app.State, error) // Get the state of a process + GetProcessLog(id app.ProcessID) (*app.Report, error) // Get the logs of a process + SearchProcessLogHistory(idpattern, refpattern, state string, from, to *time.Time) []app.ReportHistorySearchResult // Search the log history of all processes + GetPlayout(id app.ProcessID, inputid string) (string, error) // Get the URL of the playout API for a process + SetProcessMetadata(id app.ProcessID, key string, data interface{}) error // Set metatdata to a process + GetProcessMetadata(id app.ProcessID, key string) (interface{}, error) // Get previously set metadata from a process Probe(config *app.Config, timeout time.Duration) app.Probe // Probe a process with specific timeout } @@ -1777,8 +1777,8 @@ func convertProgressFromParser(progress *app.Progress, pprogress parse.Progress) } } -func (r *restream) GetProcessLog(id app.ProcessID) (*app.Log, error) { - log := &app.Log{} +func (r *restream) GetProcessLog(id app.ProcessID) (*app.Report, error) { + log := &app.Report{} r.lock.RLock() defer r.lock.RUnlock() @@ -1808,8 +1808,8 @@ func (r *restream) GetProcessLog(id app.ProcessID) (*app.Log, error) { history := task.parser.ReportHistory() for _, h := range history { - e := app.LogHistoryEntry{ - LogEntry: app.LogEntry{ + e := app.ReportHistoryEntry{ + ReportEntry: app.ReportEntry{ CreatedAt: h.CreatedAt, Prelude: h.Prelude, Matches: h.Matches, @@ -1849,9 +1849,9 @@ func (r *restream) GetProcessLog(id app.ProcessID) (*app.Log, error) { e.Progress.Output[i].ID = task.process.Config.Output[p.Index].ID } - e.LogEntry.Log = make([]app.LogLine, len(h.Log)) + e.ReportEntry.Log = make([]app.LogLine, len(h.Log)) for i, line := range h.Log { - e.LogEntry.Log[i] = app.LogLine{ + e.ReportEntry.Log[i] = app.LogLine{ Timestamp: line.Timestamp, Data: line.Data, } @@ -1863,8 +1863,8 @@ func (r *restream) GetProcessLog(id app.ProcessID) (*app.Log, error) { return log, nil } -func (r *restream) SearchProcessLogHistory(idpattern, refpattern, state string, from, to *time.Time) []app.LogHistorySearchResult { - result := []app.LogHistorySearchResult{} +func (r *restream) SearchProcessLogHistory(idpattern, refpattern, state string, from, to *time.Time) []app.ReportHistorySearchResult { + result := []app.ReportHistorySearchResult{} ids := r.GetProcessIDs(idpattern, refpattern, "", "") @@ -1880,7 +1880,7 @@ func (r *restream) SearchProcessLogHistory(idpattern, refpattern, state string, presult := task.parser.SearchReportHistory(state, from, to) for _, f := range presult { - result = append(result, app.LogHistorySearchResult{ + result = append(result, app.ReportHistorySearchResult{ ProcessID: task.id, Reference: task.reference, ExitState: f.ExitState, diff --git a/restream/rewrite/rewrite.go b/restream/rewrite/rewrite.go index d32f3d46..ecb4f149 100644 --- a/restream/rewrite/rewrite.go +++ b/restream/rewrite/rewrite.go @@ -7,7 +7,7 @@ import ( "github.com/datarhei/core/v16/iam" iamidentity "github.com/datarhei/core/v16/iam/identity" - "github.com/datarhei/core/v16/rtmp" + rtmpurl "github.com/datarhei/core/v16/rtmp/url" srturl "github.com/datarhei/core/v16/srt/url" ) @@ -124,7 +124,7 @@ func (g *rewrite) rtmpURL(u *url.URL, _ Access, identity iamidentity.Verifier) s token := identity.GetServiceToken() // Remove the existing token from the path - path, _, _ := rtmp.GetToken(u) + path, _, _ := rtmpurl.GetToken(u) u.Path = path q := u.Query() diff --git a/rtmp/rtmp.go b/rtmp/rtmp.go index 9a4f014f..ec842f23 100644 --- a/rtmp/rtmp.go +++ b/rtmp/rtmp.go @@ -5,17 +5,17 @@ import ( "crypto/tls" "fmt" "net" - "net/url" "path/filepath" "strings" "sync" "time" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" enctoken "github.com/datarhei/core/v16/encoding/token" "github.com/datarhei/core/v16/iam" iamidentity "github.com/datarhei/core/v16/iam/identity" "github.com/datarhei/core/v16/log" + rtmpurl "github.com/datarhei/core/v16/rtmp/url" "github.com/datarhei/core/v16/session" "github.com/datarhei/joy4/av/avutil" @@ -61,7 +61,7 @@ type Config struct { // with methods like tls.Config.SetSessionTicketKeys. TLSConfig *tls.Config - Proxy proxy.ProxyReader + Proxy *node.Manager IAM iam.IAM } @@ -98,7 +98,7 @@ type server struct { channels map[string]*channel lock sync.RWMutex - proxy proxy.ProxyReader + proxy *node.Manager iam iam.IAM } @@ -203,68 +203,14 @@ func (s *server) log(who, handler, action, resource, message string, client net. }).Log(message) } -// GetToken returns the path without the token and the token found in the URL and whether -// it was found in the path. If the token was part of the path, the token is removed from -// the path. The token in the query string takes precedence. The token in the path is -// assumed to be the last path element. -func GetToken(u *url.URL) (string, string, bool) { - q := u.Query() - if q.Has("token") { - // The token was in the query. Return the unmomdified path and the token. - return u.Path, q.Get("token"), false - } - - pathElements := splitPath(u.EscapedPath()) - nPathElements := len(pathElements) - - if nPathElements <= 1 { - return u.Path, "", false - } - - rawPath := "/" + strings.Join(pathElements[:nPathElements-1], "/") - rawToken := pathElements[nPathElements-1] - - path, err := url.PathUnescape(rawPath) - if err != nil { - path = rawPath - } - - token, err := url.PathUnescape(rawToken) - if err != nil { - token = rawToken - } - - // Return the path without the token - return path, token, true -} - -func splitPath(path string) []string { - pathElements := strings.Split(filepath.Clean(path), "/") - - if len(pathElements) == 0 { - return pathElements - } - - if len(pathElements[0]) == 0 { - pathElements = pathElements[1:] - } - - return pathElements -} - -func removePathPrefix(path, prefix string) (string, string) { - prefix = filepath.Join("/", prefix) - return filepath.Join("/", strings.TrimPrefix(path, prefix+"/")), prefix -} - // handlePlay is called when a RTMP client wants to play a stream func (s *server) handlePlay(conn *rtmp.Conn) { defer conn.Close() remote := conn.NetConn().RemoteAddr() - playpath, token, isStreamkey := GetToken(conn.URL) + playpath, token, isStreamkey := rtmpurl.GetToken(conn.URL) - playpath, _ = removePathPrefix(playpath, s.app) + playpath, _ = rtmpurl.RemovePathPrefix(playpath, s.app) identity, err := s.findIdentityFromStreamKey(token) if err != nil { @@ -293,7 +239,7 @@ func (s *server) handlePlay(conn *rtmp.Conn) { if ch == nil && s.proxy != nil { // Check in the cluster for that stream - url, err := s.proxy.GetURL("rtmp", playpath) + url, err := s.proxy.MediaGetURL("rtmp", playpath) if err != nil { s.log(identity, "PLAY", "NOTFOUND", playpath, "", remote) return @@ -390,9 +336,9 @@ func (s *server) handlePublish(conn *rtmp.Conn) { defer conn.Close() remote := conn.NetConn().RemoteAddr() - playpath, token, isStreamkey := GetToken(conn.URL) + playpath, token, isStreamkey := rtmpurl.GetToken(conn.URL) - playpath, app := removePathPrefix(playpath, s.app) + playpath, app := rtmpurl.RemovePathPrefix(playpath, s.app) identity, err := s.findIdentityFromStreamKey(token) if err != nil { @@ -534,7 +480,7 @@ func (s *server) findIdentityFromStreamKey(key string) (string, error) { // considered the domain. It is assumed that the app is not part of // the provided path. func (s *server) findDomainFromPlaypath(path string) string { - elements := splitPath(path) + elements := rtmpurl.SplitPath(path) if len(elements) == 1 { return "$none" } diff --git a/rtmp/rtmp_test.go b/rtmp/rtmp_test.go index bdb36676..aab346e6 100644 --- a/rtmp/rtmp_test.go +++ b/rtmp/rtmp_test.go @@ -4,6 +4,8 @@ import ( "net/url" "testing" + rtmpurl "github.com/datarhei/core/v16/rtmp/url" + "github.com/stretchr/testify/require" ) @@ -20,7 +22,7 @@ func TestToken(t *testing.T) { u, err := url.Parse(d[0]) require.NoError(t, err) - path, token, _ := GetToken(u) + path, token, _ := rtmpurl.GetToken(u) require.Equal(t, d[1], path, "url=%s", u.String()) require.Equal(t, d[2], token, "url=%s", u.String()) @@ -35,7 +37,7 @@ func TestSplitPath(t *testing.T) { } for path, split := range data { - elms := splitPath(path) + elms := rtmpurl.SplitPath(path) require.ElementsMatch(t, split, elms, "%s", path) } @@ -49,7 +51,7 @@ func TestRemovePathPrefix(t *testing.T) { } for _, d := range data { - x, _ := removePathPrefix(d[0], d[1]) + x, _ := rtmpurl.RemovePathPrefix(d[0], d[1]) require.Equal(t, d[2], x, "path=%s prefix=%s", d[0], d[1]) } diff --git a/rtmp/url/url.go b/rtmp/url/url.go new file mode 100644 index 00000000..229cc635 --- /dev/null +++ b/rtmp/url/url.go @@ -0,0 +1,61 @@ +package url + +import ( + "net/url" + "path/filepath" + "strings" +) + +// GetToken returns the path without the token and the token found in the URL and whether +// it was found in the path. If the token was part of the path, the token is removed from +// the path. The token in the query string takes precedence. The token in the path is +// assumed to be the last path element. +func GetToken(u *url.URL) (string, string, bool) { + q := u.Query() + if q.Has("token") { + // The token was in the query. Return the unmomdified path and the token. + return u.Path, q.Get("token"), false + } + + pathElements := SplitPath(u.EscapedPath()) + nPathElements := len(pathElements) + + if nPathElements <= 1 { + return u.Path, "", false + } + + rawPath := "/" + strings.Join(pathElements[:nPathElements-1], "/") + rawToken := pathElements[nPathElements-1] + + path, err := url.PathUnescape(rawPath) + if err != nil { + path = rawPath + } + + token, err := url.PathUnescape(rawToken) + if err != nil { + token = rawToken + } + + // Return the path without the token + return path, token, true +} + +func SplitPath(path string) []string { + pathElements := strings.Split(filepath.Clean(path), "/") + + if len(pathElements) == 0 { + return pathElements + } + + if len(pathElements[0]) == 0 { + pathElements = pathElements[1:] + } + + return pathElements +} + +func RemovePathPrefix(path, prefix string) (string, string) { + prefix = filepath.Join("/", prefix) + return filepath.Join("/", strings.TrimPrefix(path, prefix+"/")), prefix +} diff --git a/slices/diff.go b/slices/diff.go index 4d271571..1ef398d5 100644 --- a/slices/diff.go +++ b/slices/diff.go @@ -59,7 +59,7 @@ func DiffEqualer[T any, X Equaler[T]](listA []T, listB []X) ([]T, []X) { if visited[j] { continue } - if listB[j].Equal(element) { + if listB[j].Equal(element) == nil { visited[j] = true found = true break diff --git a/slices/equal.go b/slices/equal.go index 29ed6468..66428024 100644 --- a/slices/equal.go +++ b/slices/equal.go @@ -1,28 +1,54 @@ package slices +import ( + "errors" + "fmt" + "strings" +) + // EqualComparableElements returns whether two slices have the same elements. -func EqualComparableElements[T comparable](a, b []T) bool { +func EqualComparableElements[T comparable](a, b []T) error { extraA, extraB := DiffComparable(a, b) if len(extraA) == 0 && len(extraB) == 0 { - return true + return nil } - return false + diff := []string{} + + for _, e := range extraA { + diff = append(diff, fmt.Sprintf("+ %v", e)) + } + + for _, e := range extraB { + diff = append(diff, fmt.Sprintf("- %v", e)) + } + + return errors.New(strings.Join(diff, ",")) } // Equaler defines a type that implements the Equal function. type Equaler[T any] interface { - Equal(T) bool + Equal(T) error } // EqualEqualerElements returns whether two slices of Equaler have the same elements. -func EqualEqualerElements[T any, X Equaler[T]](a []T, b []X) bool { +func EqualEqualerElements[T any, X Equaler[T]](a []T, b []X) error { extraA, extraB := DiffEqualer(a, b) if len(extraA) == 0 && len(extraB) == 0 { - return true + return nil } - return false + diff := []string{} + + for _, e := range extraA { + diff = append(diff, fmt.Sprintf("- %v", e)) + } + + for _, e := range extraB { + diff = append(diff, fmt.Sprintf("+ %v", e)) + } + + return errors.New(strings.Join(diff, ",")) } diff --git a/slices/equal_test.go b/slices/equal_test.go index 7769f843..8ab1181a 100644 --- a/slices/equal_test.go +++ b/slices/equal_test.go @@ -1,6 +1,7 @@ package slices import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -10,42 +11,46 @@ func TestEqualComparableElements(t *testing.T) { a := []string{"a", "b", "c", "d"} b := []string{"b", "c", "a", "d"} - ok := EqualComparableElements(a, b) - require.True(t, ok) + err := EqualComparableElements(a, b) + require.NoError(t, err) - ok = EqualComparableElements(b, a) - require.True(t, ok) + err = EqualComparableElements(b, a) + require.NoError(t, err) a = append(a, "z") - ok = EqualComparableElements(a, b) - require.False(t, ok) + err = EqualComparableElements(a, b) + require.Error(t, err) - ok = EqualComparableElements(b, a) - require.False(t, ok) + err = EqualComparableElements(b, a) + require.Error(t, err) } type String string -func (a String) Equal(b String) bool { - return string(a) == string(b) +func (a String) Equal(b String) error { + if string(a) == string(b) { + return nil + } + + return fmt.Errorf("%s != %s", a, b) } func TestEqualEqualerElements(t *testing.T) { a := []String{"a", "b", "c", "d"} b := []String{"b", "c", "a", "d"} - ok := EqualEqualerElements(a, b) - require.True(t, ok) + err := EqualEqualerElements(a, b) + require.NoError(t, err) - ok = EqualEqualerElements(b, a) - require.True(t, ok) + err = EqualEqualerElements(b, a) + require.NoError(t, err) a = append(a, "z") - ok = EqualEqualerElements(a, b) - require.False(t, ok) + err = EqualEqualerElements(a, b) + require.Error(t, err) - ok = EqualEqualerElements(b, a) - require.False(t, ok) + err = EqualEqualerElements(b, a) + require.Error(t, err) } diff --git a/srt/srt.go b/srt/srt.go index ee4bae19..1974adda 100644 --- a/srt/srt.go +++ b/srt/srt.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/datarhei/core/v16/cluster/proxy" + "github.com/datarhei/core/v16/cluster/node" enctoken "github.com/datarhei/core/v16/encoding/token" "github.com/datarhei/core/v16/iam" iamidentity "github.com/datarhei/core/v16/iam/identity" @@ -45,7 +45,7 @@ type Config struct { SRTLogTopics []string - Proxy proxy.ProxyReader + Proxy *node.Manager IAM iam.IAM } @@ -84,7 +84,7 @@ type server struct { srtlog map[string]*ring.Ring // Per logtopic a dedicated ring buffer srtlogLock sync.RWMutex - proxy proxy.ProxyReader + proxy *node.Manager iam iam.IAM } @@ -423,7 +423,7 @@ func (s *server) handleSubscribe(conn srt.Conn) { } // Check in the cluster for the stream and proxy it - srturl, err := s.proxy.GetURL("srt", si.Resource) + srturl, err := s.proxy.MediaGetURL("srt", si.Resource) if err != nil { s.log(identity, "PLAY", "NOTFOUND", si.Resource, "no publisher for this resource found", client) return diff --git a/srt/url/url.go b/srt/url/url.go index 76f036cc..3345a7f5 100644 --- a/srt/url/url.go +++ b/srt/url/url.go @@ -2,7 +2,6 @@ package url import ( "fmt" - "net/url" neturl "net/url" "regexp" "strings" @@ -111,7 +110,7 @@ func (si *StreamInfo) String() string { func ParseStreamId(streamid string) (StreamInfo, error) { si := StreamInfo{Mode: "request"} - if decodedStreamid, err := url.QueryUnescape(streamid); err == nil { + if decodedStreamid, err := neturl.QueryUnescape(streamid); err == nil { streamid = decodedStreamid } diff --git a/vendor/github.com/datarhei/core-client-go/v16/LICENSE b/vendor/github.com/datarhei/core-client-go/v16/LICENSE deleted file mode 100644 index 3861dabb..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 FOSS GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/datarhei/core-client-go/v16/README.md b/vendor/github.com/datarhei/core-client-go/v16/README.md deleted file mode 100644 index bbbd321d..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/README.md +++ /dev/null @@ -1,312 +0,0 @@ -# core-client-go - -A golang client for the `github.com/datarhei/core` API. - ---- - -- [Quick Start](#quick-start) -- [API definitions](#api-definitions) - - [General](#general) - - [Config](#config) - - [Disk filesystem](#disk-filesystem) - - [In-memory filesystem](#in-memory-filesystem) - - [Log](#log) - - [Metadata](#metadata) - - [Metrics](#metrics) - - [Process](#process) - - [RTMP](#rtmp) - - [Session](#session) - - [Skills](#skills) -- [Versioning](#versioning) -- [Contributing](#contributing) -- [Licence](#licence) - -## Quick Start - -Example for retrieving a list of all processes: - -``` -import "github.com/datarhei/core-client-go/v16" - -client, err := coreclient.New(coreclient.Config{ - Address: "https://example.com:8080", - Username: "foo", - Password: "bar", -}) -if err != nil { - ... -} - -processes, err := client.ProcessList(coreclient.ProcessListOptions{}) -if err != nil { - ... -} -``` - -## API definitions - -### General - -- `GET` /api - - ```golang - About() api.About - ``` - -### Config - -- `GET` /api/v3/config - - ```golang - Config() (api.Config, error) - ``` - -- `PUT` /api/v3/config - - ```golang - ConfigSet(config api.ConfigSet) error - ``` - -- `GET` /api/v3/config/reload - - ```golang - ConfigReload() error - ``` - -### Disk filesystem - -- `GET` /api/v3/fs/disk - - ```golang - DiskFSList(sort, order string) ([]api.FileInfo, error) - ``` - -- `HEAD` /api/v3/fs/disk/{path} - - ```golang - DiskFSHasFile(path string) bool - ``` - -- `GET` /api/v3/fs/disk/{path} - - ```golang - DiskFSGetFile(path string) (io.ReadCloser, error) - ``` - -- `DELETE` /api/v3/fs/disk/{path} - - ```golang - DiskFSDeleteFile(path string) error - ``` - -- `PUT` /api/v3/fs/disk/{path} - ```golang - DiskFSAddFile(path string, data io.Reader) error - ``` - -### In-memory filesystem - -- `GET` /api/v3/fs/mem - - ```golang - MemFSList(sort, order string) ([]api.FileInfo, error) - ``` - -- `HEAD` /api/v3/fs/mem/{path} - - ```golang - MemFSHasFile(path string) bool - ``` - -- `GET` /api/v3/fs/mem/{path} - - ```golang - MemFSGetFile(path string) (io.ReadCloser, error) - ``` - -- `DELETE` /api/v3/fs/mem/{path} - - ```golang - MemFSDeleteFile(path string) error - ``` - -- `PUT` /api/v3/fs/mem/{path} - ```golang - MemFSAddFile(path string, data io.Reader) error - ``` - -### Log - -- `GET` /api/v3/log - - ```golang - Log() ([]api.LogEvent, error) - ``` - -### Metadata - -- `GET` /api/v3/metadata/{key} - - ```golang - Metadata(id, key string) (api.Metadata, error) - ``` - -- `PUT` /api/v3/metadata/{key} - ```golang - MetadataSet(id, key string, metadata api.Metadata) error - ``` - -### Metrics - -- `GET` /api/v3/metrics - - ```golang - MetricsList() ([]api.MetricsDescription, error) - ``` - -- `POST` /api/v3/metrics - - ```golang - Metrics(query api.MetricsQuery) (api.MetricsResponse, error) - ``` - -### Process - -- `GET` /api/v3/process - - ```golang - ProcessList(opts ProcessListOptions) ([]api.Process, error) - ``` - -- `POST` /api/v3/process - - ```golang - ProcessAdd(p api.ProcessConfig) error - ``` - -- `GET` /api/v3/process/{id} - - ```golang - Process(id string, filter []string) (api.Process, error) - ``` - -- `PUT` /api/v3/process/{id} - - ```golang - ProcessUpdate(id string, p api.ProcessConfig) error - ``` - -- `DELETE` /api/v3/process/{id} - - ```golang - ProcessDelete(id string) error - ``` - -- `PUT` /api/v3/process/{id}/command - - ```golang - ProcessCommand(id, command string) error - ``` - -- `GET` /api/v3/process/{id}/probe - - ```golang - ProcessProbe(id string) (api.Probe, error) - ``` - -- `GET` /api/v3/process/{id}/config - - ```golang - ProcessConfig(id string) (api.ProcessConfig, error) - ``` - -- `GET` /api/v3/process/{id}/report - - ```golang - ProcessReport(id string) (api.ProcessReport, error) - ``` - -- `GET` /api/v3/process/{id}/state - - ```golang - ProcessState(id string) (api.ProcessState, error) - ``` - -- `GET` /api/v3/process/{id}/metadata/{key} - - ```golang - ProcessMetadata(id, key string) (api.Metadata, error) - ``` - -- `PUT` /api/v3/process/{id}/metadata/{key} - ```golang - ProcessMetadataSet(id, key string, metadata api.Metadata) error - ``` - -### RTMP - -- `GET` /api/v3/rtmp - - ```golang - RTMPChannels() ([]api.RTMPChannel, error) - ``` - -### SRT - -- `GET` /api/v3/srt - - ```golang - SRTChannels() (api.SRTChannels, error) - ``` - -### Session - -- `GET` /api/v3/session - - ```golang - Sessions(collectors []string) (api.SessionsSummary, error) - ``` - -- `GET` /api/v3/session/active - ```golang - SessionsActive(collectors []string) (api.SessionsActive, error) - ``` - -### Skills - -- `GET` /api/v3/skills - - ```golang - Skills() (api.Skills, error) - ``` - -- `GET` /api/v3/skills/reload - ```golang - SkillsReload() error - ``` - -### Widget - -- `GET` /api/v3/widget - - ```golang - WidgetProcess(id string) (api.WidgetProcess, error) - ``` - -## Versioning - -The version of this module is according to which version of the datarhei Core API -you want to connect to. Check the branches to find out which other versions are -implemented. If you want to connect to an API version 12, you have to import the client -module of the version 12, i.e. `import "github.com/datarhei/core-client-go/v12"`. - -The latest implementation is on the `main` branch. - -## Contributing - -Found a mistake or misconduct? Create a [issue](https://github.com/datarhei/core-client-go/issues) or send a pull-request. -Suggestions for improvement are welcome. - -## Licence - -[MIT](https://github.com/datarhei/core-client-go/blob/main/LICENSE) diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/about.go b/vendor/github.com/datarhei/core-client-go/v16/api/about.go deleted file mode 100644 index c5fba8f9..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/about.go +++ /dev/null @@ -1,33 +0,0 @@ -package api - -// About is some general information about the API -type About struct { - App string `json:"app"` - Auths []string `json:"auths"` - Name string `json:"name"` - ID string `json:"id"` - CreatedAt string `json:"created_at"` - Uptime uint64 `json:"uptime_seconds"` - Version Version `json:"version"` -} - -// Version is some information about the binary -type Version struct { - Number string `json:"number"` - Commit string `json:"repository_commit"` - Branch string `json:"repository_branch"` - Build string `json:"build_date"` - Arch string `json:"arch"` - Compiler string `json:"compiler"` -} - -// MinimalAbout is the minimal information about the API -type MinimalAbout struct { - App string `json:"app"` - Auths []string `json:"auths"` - Version VersionMinimal `json:"version"` -} - -type VersionMinimal struct { - Number string `json:"number"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/avstream.go b/vendor/github.com/datarhei/core-client-go/v16/api/avstream.go deleted file mode 100644 index a98d03cb..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/avstream.go +++ /dev/null @@ -1,23 +0,0 @@ -package api - -type AVstreamIO struct { - State string `json:"state" enums:"running,idle" jsonschema:"enum=running,enum=idle"` - Packet uint64 `json:"packet" format:"uint64"` - Time uint64 `json:"time"` - Size uint64 `json:"size_kb"` -} - -type AVstream struct { - Input AVstreamIO `json:"input"` - Output AVstreamIO `json:"output"` - Aqueue uint64 `json:"aqueue" format:"uint64"` - Queue uint64 `json:"queue" format:"uint64"` - Dup uint64 `json:"dup" format:"uint64"` - Drop uint64 `json:"drop" format:"uint64"` - Enc uint64 `json:"enc" format:"uint64"` - Looping bool `json:"looping"` - LoopingRuntime uint64 `json:"looping_runtime" format:"uint64"` - Duplicating bool `json:"duplicating"` - GOP string `json:"gop"` - Mode string `json:"mode"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/cluster.go b/vendor/github.com/datarhei/core-client-go/v16/api/cluster.go deleted file mode 100644 index ad2461d1..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/cluster.go +++ /dev/null @@ -1,99 +0,0 @@ -package api - -import ( - "time" -) - -type ClusterNode struct { - ID string `json:"id"` - Name string `json:"name"` - Version string `json:"version"` - Status string `json:"status"` - Error string `json:"error"` - Voter bool `json:"voter"` - Leader bool `json:"leader"` - Address string `json:"address"` - CreatedAt string `json:"created_at"` // RFC 3339 - Uptime int64 `json:"uptime_seconds"` // seconds - LastContact float64 `json:"last_contact_ms"` // milliseconds - Latency float64 `json:"latency_ms"` // milliseconds - Core ClusterNodeCore `json:"core"` - Resources ClusterNodeResources `json:"resources"` -} - -type ClusterNodeCore struct { - Address string `json:"address"` - Status string `json:"status"` - Error string `json:"error"` - LastContact float64 `json:"last_contact_ms"` // milliseconds - Latency float64 `json:"latency_ms"` // milliseconds - Version string `json:"version"` -} - -type ClusterNodeResources struct { - IsThrottling bool `json:"is_throttling"` - NCPU float64 `json:"ncpu"` - CPU float64 `json:"cpu_used"` // percent 0-100*npcu - CPULimit float64 `json:"cpu_limit"` // percent 0-100*npcu - Mem uint64 `json:"memory_used_bytes"` // bytes - MemLimit uint64 `json:"memory_limit_bytes"` // bytes - Error string `json:"error"` -} - -type ClusterRaft struct { - Address string `json:"address"` - State string `json:"state"` - LastContact float64 `json:"last_contact_ms"` // milliseconds - NumPeers uint64 `json:"num_peers"` - LogTerm uint64 `json:"log_term"` - LogIndex uint64 `json:"log_index"` -} - -type ClusterAboutLeader struct { - ID string `json:"id"` - Address string `json:"address"` - ElectedSince uint64 `json:"elected_seconds"` -} - -type ClusterAbout struct { - Raft ClusterRaft `json:"raft"` - Nodes []ClusterNode `json:"nodes"` - Version string `json:"version"` - Degraded bool `json:"degraded"` - DegradedErr string `json:"degraded_error"` -} - -type ClusterAboutV1 struct { - ID string `json:"id"` - Name string `json:"name"` - Leader bool `json:"leader"` - Address string `json:"address"` - ClusterAbout -} - -type ClusterAboutV2 struct { - ID string `json:"id"` - Domains []string `json:"public_domains"` - Leader ClusterAboutLeader `json:"leader"` - Status string `json:"status"` - ClusterAbout -} - -type ClusterNodeFiles struct { - LastUpdate int64 `json:"last_update"` // unix timestamp - Files map[string][]string `json:"files"` -} - -type ClusterLock struct { - Name string `json:"name"` - ValidUntil time.Time `json:"valid_until"` -} - -type ClusterKVSValue struct { - Value string `json:"value"` - UpdatedAt time.Time `json:"updated_at"` -} - -type ClusterKVS map[string]ClusterKVSValue - -type ClusterProcessMap map[string]string diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/command.go b/vendor/github.com/datarhei/core-client-go/v16/api/command.go deleted file mode 100644 index 9fc556ad..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/command.go +++ /dev/null @@ -1,6 +0,0 @@ -package api - -// Command is a command to send to a process -type Command struct { - Command string `json:"command" validate:"required" enums:"start,stop,restart,reload" jsonschema:"enum=start,enum=stop,enum=restart,enum=reload"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/config.go b/vendor/github.com/datarhei/core-client-go/v16/api/config.go deleted file mode 100644 index fcfffe50..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/config.go +++ /dev/null @@ -1,524 +0,0 @@ -package api - -import ( - "fmt" - "strings" - "time" -) - -type ConfigV1 struct { - Version int64 `json:"version" jsonschema:"minimum=1,maximum=1"` - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - CheckForUpdates bool `json:"update_check"` - Log struct { - Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` - Topics []string `json:"topics"` - MaxLines int `json:"max_lines"` - } `json:"log"` - DB struct { - Dir string `json:"dir"` - } `json:"db"` - Host struct { - Name []string `json:"name"` - Auto bool `json:"auto"` - } `json:"host"` - API struct { - ReadOnly bool `json:"read_only"` - Access struct { - HTTP struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"http"` - HTTPS struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"https"` - } `json:"access"` - Auth struct { - Enable bool `json:"enable"` - DisableLocalhost bool `json:"disable_localhost"` - Username string `json:"username"` - Password string `json:"password"` - JWT struct { - Secret string `json:"secret"` - } `json:"jwt"` - Auth0 struct { - Enable bool `json:"enable"` - Tenants []Auth0Tenant `json:"tenants"` - } `json:"auth0"` - } `json:"auth"` - } `json:"api"` - TLS struct { - Address string `json:"address"` - Enable bool `json:"enable"` - Auto bool `json:"auto"` - CertFile string `json:"cert_file"` - KeyFile string `json:"key_file"` - } `json:"tls"` - Storage struct { - Disk struct { - Dir string `json:"dir"` - Size int64 `json:"max_size_mbytes"` - Cache struct { - Enable bool `json:"enable"` - Size uint64 `json:"max_size_mbytes"` - TTL int64 `json:"ttl_seconds"` - FileSize uint64 `json:"max_file_size_mbytes"` - Types []string `json:"types"` - } `json:"cache"` - } `json:"disk"` - Memory struct { - Auth struct { - Enable bool `json:"enable"` - Username string `json:"username"` - Password string `json:"password"` - } `json:"auth"` - Size int64 `json:"max_size_mbytes"` - Purge bool `json:"purge"` - } `json:"memory"` - CORS struct { - Origins []string `json:"origins"` - } `json:"cors"` - MimeTypes string `json:"mimetypes_file"` - } `json:"storage"` - RTMP struct { - Enable bool `json:"enable"` - EnableTLS bool `json:"enable_tls"` - Address string `json:"address"` - App string `json:"app"` - Token string `json:"token"` - } `json:"rtmp"` - FFmpeg struct { - Binary string `json:"binary"` - MaxProcesses int64 `json:"max_processes"` - Access struct { - Input struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"input"` - Output struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"output"` - } `json:"access"` - Log struct { - MaxLines int `json:"max_lines"` - MaxHistory int `json:"max_history"` - } `json:"log"` - } `json:"ffmpeg"` - Playout struct { - Enable bool `json:"enable"` - MinPort int `json:"min_port"` - MaxPort int `json:"max_port"` - } `json:"playout"` - Debug struct { - Profiling bool `json:"profiling"` - ForceGC int `json:"force_gc"` - } `json:"debug"` - Metrics struct { - Enable bool `json:"enable"` - EnablePrometheus bool `json:"enable_prometheus"` - Range int64 `json:"range_sec"` // seconds - Interval int64 `json:"interval_sec"` // seconds - } `json:"metrics"` - Sessions struct { - Enable bool `json:"enable"` - IPIgnoreList []string `json:"ip_ignorelist"` - SessionTimeout int `json:"session_timeout_sec"` - Persist bool `json:"persist"` - PersistInterval int `json:"persist_interval_sec"` - MaxBitrate uint64 `json:"max_bitrate_mbit"` - MaxSessions uint64 `json:"max_sessions"` - } `json:"sessions"` - Service struct { - Enable bool `json:"enable"` - Token string `json:"token"` - URL string `json:"url"` - } `json:"service"` - Router struct { - BlockedPrefixes []string `json:"blocked_prefixes"` - Routes map[string]string `json:"routes"` - UIPath string `json:"ui_path"` - } `json:"router"` -} - -type ConfigV2 struct { - Version int64 `json:"version" jsonschema:"minimum=2,maximum=2"` - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - CheckForUpdates bool `json:"update_check"` - Log struct { - Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` - Topics []string `json:"topics"` - MaxLines int `json:"max_lines"` - } `json:"log"` - DB struct { - Dir string `json:"dir"` - } `json:"db"` - Host struct { - Name []string `json:"name"` - Auto bool `json:"auto"` - } `json:"host"` - API struct { - ReadOnly bool `json:"read_only"` - Access struct { - HTTP struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"http"` - HTTPS struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"https"` - } `json:"access"` - Auth struct { - Enable bool `json:"enable"` - DisableLocalhost bool `json:"disable_localhost"` - Username string `json:"username"` - Password string `json:"password"` - JWT struct { - Secret string `json:"secret"` - } `json:"jwt"` - Auth0 struct { - Enable bool `json:"enable"` - Tenants []Auth0Tenant `json:"tenants"` - } `json:"auth0"` - } `json:"auth"` - } `json:"api"` - TLS struct { - Address string `json:"address"` - Enable bool `json:"enable"` - Auto bool `json:"auto"` - CertFile string `json:"cert_file"` - KeyFile string `json:"key_file"` - } `json:"tls"` - Storage struct { - Disk struct { - Dir string `json:"dir"` - Size int64 `json:"max_size_mbytes"` - Cache struct { - Enable bool `json:"enable"` - Size uint64 `json:"max_size_mbytes"` - TTL int64 `json:"ttl_seconds"` - FileSize uint64 `json:"max_file_size_mbytes"` - Types []string `json:"types"` - } `json:"cache"` - } `json:"disk"` - Memory struct { - Auth struct { - Enable bool `json:"enable"` - Username string `json:"username"` - Password string `json:"password"` - } `json:"auth"` - Size int64 `json:"max_size_mbytes"` - Purge bool `json:"purge"` - } `json:"memory"` - CORS struct { - Origins []string `json:"origins"` - } `json:"cors"` - MimeTypes string `json:"mimetypes_file"` - } `json:"storage"` - RTMP struct { - Enable bool `json:"enable"` - EnableTLS bool `json:"enable_tls"` - Address string `json:"address"` - AddressTLS string `json:"address_tls"` - App string `json:"app"` - Token string `json:"token"` - } `json:"rtmp"` - SRT struct { - Enable bool `json:"enable"` - Address string `json:"address"` - Passphrase string `json:"passphrase"` - Token string `json:"token"` - Log struct { - Enable bool `json:"enable"` - Topics []string `json:"topics"` - } `json:"log"` - } `json:"srt"` - FFmpeg struct { - Binary string `json:"binary"` - MaxProcesses int64 `json:"max_processes"` - Access struct { - Input struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"input"` - Output struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"output"` - } `json:"access"` - Log struct { - MaxLines int `json:"max_lines"` - MaxHistory int `json:"max_history"` - } `json:"log"` - } `json:"ffmpeg"` - Playout struct { - Enable bool `json:"enable"` - MinPort int `json:"min_port"` - MaxPort int `json:"max_port"` - } `json:"playout"` - Debug struct { - Profiling bool `json:"profiling"` - ForceGC int `json:"force_gc"` - } `json:"debug"` - Metrics struct { - Enable bool `json:"enable"` - EnablePrometheus bool `json:"enable_prometheus"` - Range int64 `json:"range_sec"` // seconds - Interval int64 `json:"interval_sec"` // seconds - } `json:"metrics"` - Sessions struct { - Enable bool `json:"enable"` - IPIgnoreList []string `json:"ip_ignorelist"` - SessionTimeout int `json:"session_timeout_sec"` - Persist bool `json:"persist"` - PersistInterval int `json:"persist_interval_sec"` - MaxBitrate uint64 `json:"max_bitrate_mbit"` - MaxSessions uint64 `json:"max_sessions"` - } `json:"sessions"` - Service struct { - Enable bool `json:"enable"` - Token string `json:"token"` - URL string `json:"url"` - } `json:"service"` - Router struct { - BlockedPrefixes []string `json:"blocked_prefixes"` - Routes map[string]string `json:"routes"` - UIPath string `json:"ui_path"` - } `json:"router"` -} - -type ConfigV3 struct { - Version int64 `json:"version" jsonschema:"minimum=3,maximum=3" format:"int64"` - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - CheckForUpdates bool `json:"update_check"` - Log struct { - Level string `json:"level" enums:"debug,info,warn,error,silent" jsonschema:"enum=debug,enum=info,enum=warn,enum=error,enum=silent"` - Topics []string `json:"topics"` - MaxLines int `json:"max_lines" format:"int"` - } `json:"log"` - DB struct { - Dir string `json:"dir"` - } `json:"db"` - Host struct { - Name []string `json:"name"` - Auto bool `json:"auto"` - } `json:"host"` - API struct { - ReadOnly bool `json:"read_only"` - Access struct { - HTTP struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"http"` - HTTPS struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"https"` - } `json:"access"` - Auth struct { - Enable bool `json:"enable"` - DisableLocalhost bool `json:"disable_localhost"` - Username string `json:"username"` - Password string `json:"password"` - JWT struct { - Secret string `json:"secret"` - } `json:"jwt"` - Auth0 struct { - Enable bool `json:"enable"` - Tenants []Auth0Tenant `json:"tenants"` - } `json:"auth0"` - } `json:"auth"` - } `json:"api"` - TLS struct { - Address string `json:"address"` - Enable bool `json:"enable"` - Auto bool `json:"auto"` - Email string `json:"email"` - Staging bool `json:"staging"` - Secret string `json:"secret"` - CertFile string `json:"cert_file"` - KeyFile string `json:"key_file"` - } `json:"tls"` - Storage struct { - Disk struct { - Dir string `json:"dir"` - Size int64 `json:"max_size_mbytes" format:"int64"` - Cache struct { - Enable bool `json:"enable"` - Size uint64 `json:"max_size_mbytes" format:"uint64"` - TTL int64 `json:"ttl_seconds" format:"int64"` - FileSize uint64 `json:"max_file_size_mbytes" format:"uint64"` - Types struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"types"` - } `json:"cache"` - } `json:"disk"` - Memory struct { - Auth struct { - Enable bool `json:"enable"` // Deprecated, use IAM - Username string `json:"username"` // Deprecated, use IAM - Password string `json:"password"` // Deprecated, use IAM - } `json:"auth"` // Deprecated, use IAM - Size int64 `json:"max_size_mbytes" format:"int64"` - Purge bool `json:"purge"` - Backup struct { - Dir string `json:"dir"` - Patterns []string `json:"patterns"` - } `json:"backup"` - } `json:"memory"` - S3 []S3Storage `json:"s3"` - CORS struct { - Origins []string `json:"origins"` - } `json:"cors"` - MimeTypes string `json:"mimetypes_file"` - } `json:"storage"` - RTMP struct { - Enable bool `json:"enable"` - EnableTLS bool `json:"enable_tls"` - Address string `json:"address"` - AddressTLS string `json:"address_tls"` - App string `json:"app"` - Token string `json:"token"` // Deprecated, use IAM - } `json:"rtmp"` - SRT struct { - Enable bool `json:"enable"` - Address string `json:"address"` - Passphrase string `json:"passphrase"` - Token string `json:"token"` // Deprecated, use IAM - Log struct { - Enable bool `json:"enable"` - Topics []string `json:"topics"` - } `json:"log"` - } `json:"srt"` - FFmpeg struct { - Binary string `json:"binary"` - MaxProcesses int64 `json:"max_processes" format:"int64"` - Access struct { - Input struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"input"` - Output struct { - Allow []string `json:"allow"` - Block []string `json:"block"` - } `json:"output"` - } `json:"access"` - Log struct { - MaxLines int `json:"max_lines" format:"int"` - MaxHistory int `json:"max_history" format:"int"` - MaxMinimalHistory int `json:"max_minimal_history" format:"int"` - } `json:"log"` - } `json:"ffmpeg"` - Playout struct { - Enable bool `json:"enable"` - MinPort int `json:"min_port" format:"int"` - MaxPort int `json:"max_port" format:"int"` - } `json:"playout"` - Debug struct { - Profiling bool `json:"profiling"` - ForceGC int `json:"force_gc" format:"int"` // deprecated, use MemoryLimit instead - MemoryLimit int64 `json:"memory_limit_mbytes" format:"int64"` - AutoMaxProcs bool `json:"auto_max_procs"` - AgentAddress string `json:"agent_address"` - } `json:"debug"` - Metrics struct { - Enable bool `json:"enable"` - EnablePrometheus bool `json:"enable_prometheus"` - Range int64 `json:"range_sec" format:"int64"` // seconds - Interval int64 `json:"interval_sec" format:"int64"` // seconds - } `json:"metrics"` - Sessions struct { - Enable bool `json:"enable"` - IPIgnoreList []string `json:"ip_ignorelist"` - Persist bool `json:"persist"` - PersistInterval int `json:"persist_interval_sec" format:"int"` - SessionTimeout int `json:"session_timeout_sec" format:"int"` - SessionLogPathPattern string `json:"session_log_path_pattern"` - SessionLogBuffer int `json:"session_log_buffer_sec" format:"int"` - MaxBitrate uint64 `json:"max_bitrate_mbit" format:"uint64"` - MaxSessions uint64 `json:"max_sessions" format:"uint64"` - } `json:"sessions"` - Service struct { - Enable bool `json:"enable"` - Token string `json:"token"` - URL string `json:"url"` - } `json:"service"` - Router struct { - BlockedPrefixes []string `json:"blocked_prefixes"` - Routes map[string]string `json:"routes"` - UIPath string `json:"ui_path"` - } `json:"router"` - Resources struct { - MaxCPUUsage float64 `json:"max_cpu_usage"` // percent 0-100 - MaxMemoryUsage float64 `json:"max_memory_usage"` // percent 0-100 - } `json:"resources"` - Cluster struct { - Enable bool `json:"enable"` - Address string `json:"address"` // ip:port - Peers []string `json:"peers"` - StartupTimeout int64 `json:"startup_timeout_sec" format:"int64"` // seconds - SyncInterval int64 `json:"sync_interval_sec" format:"int64"` // seconds - NodeRecoverTimeout int64 `json:"node_recover_timeout_sec" format:"int64"` // seconds - EmergencyLeaderTimeout int64 `json:"emergency_leader_timeout_sec" format:"int64"` // seconds - Debug struct { - DisableFFmpegCheck bool `json:"disable_ffmpeg_check"` - } `json:"debug"` - } `json:"cluster"` -} - -type Config struct { - CreatedAt time.Time `json:"created_at"` - LoadedAt time.Time `json:"loaded_at"` - UpdatedAt time.Time `json:"updated_at"` - - Config interface{} `json:"config"` - - Overrides []string `json:"overrides"` -} - -type Auth0Tenant struct { - Domain string `json:"domain"` - Audience string `json:"audience"` - ClientID string `json:"clientid"` - Users []string `json:"users"` -} - -type S3Storage struct { - Name string `json:"name"` - Mountpoint string `json:"mountpoint"` - Auth S3StorageAuth `json:"auth"` - Endpoint string `json:"endpoint"` - AccessKeyID string `json:"access_key_id"` - SecretAccessKey string `json:"secret_access_key"` - Bucket string `json:"bucket"` - Region string `json:"region"` - UseSSL bool `json:"use_ssl"` -} - -type S3StorageAuth struct { - Enable bool `json:"enable"` - Username string `json:"username"` - Password string `json:"password"` -} - -// ConfigError is used to return error messages when uploading a new config -type ConfigError map[string][]string - -func (c ConfigError) Error() string { - s := strings.Builder{} - - for key, messages := range map[string][]string(c) { - s.WriteString(fmt.Sprintf("%s: %s", key, strings.Join(messages, ","))) - } - - return s.String() -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/doc.go b/vendor/github.com/datarhei/core-client-go/v16/api/doc.go deleted file mode 100644 index 8bbee674..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package api provides types for communicating with the REST API -package api diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/error.go b/vendor/github.com/datarhei/core-client-go/v16/api/error.go deleted file mode 100644 index 3f9c610b..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/error.go +++ /dev/null @@ -1,19 +0,0 @@ -package api - -import ( - "fmt" - "strings" -) - -// Error represents an error response of the API -type Error struct { - Code int `json:"code" jsonschema:"required"` - Message string `json:"message" jsonschema:""` - Details []string `json:"details" jsonschema:""` - Body []byte `json:"-"` -} - -// Error returns the string representation of the error -func (e Error) Error() string { - return fmt.Sprintf("code=%d, message=%s, details=%s", e.Code, e.Message, strings.Join(e.Details, " ")) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/event.go b/vendor/github.com/datarhei/core-client-go/v16/api/event.go deleted file mode 100644 index 095acd48..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/event.go +++ /dev/null @@ -1,22 +0,0 @@ -package api - -type Event struct { - Timestamp int64 `json:"ts" format:"int64"` - Level int `json:"level"` - Component string `json:"event"` - Message string `json:"message"` - Caller string `json:"caller"` - - Data map[string]string `json:"data"` -} - -type EventFilter struct { - Component string `json:"event"` - Message string `json:"message"` - Level string `json:"level"` - Data map[string]string `json:"data"` -} - -type EventFilters struct { - Filters []EventFilter `json:"filters"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/fs.go b/vendor/github.com/datarhei/core-client-go/v16/api/fs.go deleted file mode 100644 index a0acb1e1..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/fs.go +++ /dev/null @@ -1,23 +0,0 @@ -package api - -// FileInfo represents informatiion about a file on a filesystem -type FileInfo struct { - Name string `json:"name" jsonschema:"minLength=1"` - Size int64 `json:"size_bytes" jsonschema:"minimum=0"` - LastMod int64 `json:"last_modified" jsonschema:"minimum=0"` - CoreID string `json:"core_id,omitempty"` -} - -type FilesystemInfo struct { - Name string `json:"name"` - Type string `json:"type"` - Mount string `json:"mount"` -} - -// FilesystemOperation represents a file operation on one or more filesystems -type FilesystemOperation struct { - Operation string `json:"operation" validate:"required" enums:"copy,move" jsonschema:"enum=copy,enum=move"` - Source string `json:"source"` - Target string `json:"target"` - RateLimit uint64 `json:"bandwidth_limit_kbit"` // kbit/s -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/graph.go b/vendor/github.com/datarhei/core-client-go/v16/api/graph.go deleted file mode 100644 index b5920287..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/graph.go +++ /dev/null @@ -1,11 +0,0 @@ -package api - -type GraphQuery struct { - Query string `json:"query"` - Variables interface{} `json:"variables"` -} - -type GraphResponse struct { - Data interface{} `json:"data"` - Errors []interface{} `json:"errors"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/iam.go b/vendor/github.com/datarhei/core-client-go/v16/api/iam.go deleted file mode 100644 index 18c9217e..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/iam.go +++ /dev/null @@ -1,46 +0,0 @@ -package api - -type IAMUser struct { - CreatedAt int64 `json:"created_at" format:"int64"` - UpdatedAt int64 `json:"updated_at" format:"int64"` - Name string `json:"name"` - Alias string `json:"alias"` - Superuser bool `json:"superuser"` - Auth IAMUserAuth `json:"auth"` - Policies []IAMPolicy `json:"policies"` -} - -type IAMUserAuth struct { - API IAMUserAuthAPI `json:"api"` - Services IAMUserAuthServices `json:"services"` -} - -type IAMUserAuthAPI struct { - Password string `json:"userpass"` - Auth0 IAMUserAuthAPIAuth0 `json:"auth0"` -} - -type IAMUserAuthAPIAuth0 struct { - User string `json:"user"` - Tenant IAMAuth0Tenant `json:"tenant"` -} - -type IAMUserAuthServices struct { - Basic []string `json:"basic"` - Token []string `json:"token"` - Session []string `json:"session"` -} - -type IAMAuth0Tenant struct { - Domain string `json:"domain"` - Audience string `json:"audience"` - ClientID string `json:"client_id"` -} - -type IAMPolicy struct { - Name string `json:"name,omitempty"` - Domain string `json:"domain"` - Types []string `json:"types"` - Resource string `json:"resource"` - Actions []string `json:"actions"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/jwt.go b/vendor/github.com/datarhei/core-client-go/v16/api/jwt.go deleted file mode 100644 index 85028eae..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/jwt.go +++ /dev/null @@ -1,11 +0,0 @@ -package api - -// JWT is the JWT token and its expiry date -type JWT struct { - AccessToken string `json:"access_token" jsonschema:"minLength=1"` - RefreshToken string `json:"refresh_token" jsonschema:"minLength=1"` -} - -type JWTRefresh struct { - AccessToken string `json:"access_token" jsonschema:"minLength=1"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/log.go b/vendor/github.com/datarhei/core-client-go/v16/api/log.go deleted file mode 100644 index 137f2abc..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/log.go +++ /dev/null @@ -1,4 +0,0 @@ -package api - -// LogEvent represents a log event from the app -type LogEvent map[string]interface{} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/login.go b/vendor/github.com/datarhei/core-client-go/v16/api/login.go deleted file mode 100644 index a81a9b29..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/login.go +++ /dev/null @@ -1,7 +0,0 @@ -package api - -// Login are the requires login credentials -type Login struct { - Username string `json:"username" validate:"required" jsonschema:"minLength=1"` - Password string `json:"password" validate:"required" jsonschema:"minLength=1"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/metadata.go b/vendor/github.com/datarhei/core-client-go/v16/api/metadata.go deleted file mode 100644 index 306b7c06..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/metadata.go +++ /dev/null @@ -1,9 +0,0 @@ -package api - -// Metadata represents arbitrary metadata for a process of for the app -type Metadata interface{} - -// NewMetadata takes an interface and converts it to a Metadata type. -func NewMetadata(data interface{}) Metadata { - return Metadata(data) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/metrics.go b/vendor/github.com/datarhei/core-client-go/v16/api/metrics.go deleted file mode 100644 index 1f2a7fc2..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/metrics.go +++ /dev/null @@ -1,56 +0,0 @@ -package api - -import ( - "time" - - "github.com/goccy/go-json" -) - -type MetricsDescription struct { - Name string `json:"name"` - Description string `json:"description"` - Labels []string `json:"labels"` -} - -type MetricsQueryMetric struct { - Name string `json:"name"` - Labels map[string]string `json:"labels"` -} - -type MetricsQuery struct { - Timerange int64 `json:"timerange_sec"` - Interval int64 `json:"interval_sec"` - Metrics []MetricsQueryMetric `json:"metrics"` -} - -type MetricsResponseMetric struct { - Name string `json:"name"` - Labels map[string]string `json:"labels"` - Values []MetricsResponseValue `json:"values"` -} - -type MetricsResponseValue struct { - TS time.Time `json:"ts"` - Value float64 `json:"value"` -} - -// MarshalJSON unmarshals a JSON to MetricsResponseValue -func (v *MetricsResponseValue) UnmarshalJSON(data []byte) error { - x := []float64{} - - err := json.Unmarshal(data, &x) - if err != nil { - return err - } - - v.TS = time.Unix(int64(x[0]), 0) - v.Value = x[1] - - return nil -} - -type MetricsResponse struct { - Timerange int64 `json:"timerange_sec"` - Interval int64 `json:"interval_sec"` - Metrics []MetricsResponseMetric `json:"metrics"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/playout.go b/vendor/github.com/datarhei/core-client-go/v16/api/playout.go deleted file mode 100644 index bf3e7692..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/playout.go +++ /dev/null @@ -1,33 +0,0 @@ -package api - -type PlayoutStatusIO struct { - State string `json:"state" enums:"running,idle" jsonschema:"enum=running,enum=idle"` - Packet uint64 `json:"packet" format:"uint64"` - Time uint64 `json:"time" format:"uint64"` - Size uint64 `json:"size_kb" format:"uint64"` -} - -type PlayoutStatusSwap struct { - Address string `json:"url"` - Status string `json:"status"` - LastAddress string `json:"lasturl"` - LastError string `json:"lasterror"` -} - -type PlayoutStatus struct { - ID string `json:"id"` - Address string `json:"url"` - Stream uint64 `json:"stream" format:"uint64"` - Queue uint64 `json:"queue" format:"uint64"` - AQueue uint64 `json:"aqueue" format:"uint64"` - Dup uint64 `json:"dup" format:"uint64"` - Drop uint64 `json:"drop" format:"uint64"` - Enc uint64 `json:"enc" format:"uint64"` - Looping bool `json:"looping"` - Duplicating bool `json:"duplicating"` - GOP string `json:"gop"` - Debug interface{} `json:"debug"` - Input PlayoutStatusIO `json:"input"` - Output PlayoutStatusIO `json:"output"` - Swap PlayoutStatusSwap `json:"swap"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/probe.go b/vendor/github.com/datarhei/core-client-go/v16/api/probe.go deleted file mode 100644 index 4f33932d..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/probe.go +++ /dev/null @@ -1,34 +0,0 @@ -package api - -// ProbeIO represents a stream of a probed file -type ProbeIO struct { - // common - Address string `json:"url"` - Format string `json:"format"` - Index uint64 `json:"index" format:"uint64"` - Stream uint64 `json:"stream" format:"uint64"` - Language string `json:"language"` - Type string `json:"type"` - Codec string `json:"codec"` - Coder string `json:"coder"` - Bitrate float64 `json:"bitrate_kbps" swaggertype:"number" jsonschema:"type=number"` - Duration float64 `json:"duration_sec" swaggertype:"number" jsonschema:"type=number"` - - // video - FPS float64 `json:"fps" swaggertype:"number" jsonschema:"type=number"` - Pixfmt string `json:"pix_fmt"` - Width uint64 `json:"width" format:"uint64"` - Height uint64 `json:"height" format:"uint64"` - - // audio - Sampling uint64 `json:"sampling_hz" format:"uint64"` - Layout string `json:"layout"` - Channels uint64 `json:"channels" format:"uint64"` -} - -// Probe represents the result of probing a file. It has a list of detected streams -// and a list of log lone from the probe process. -type Probe struct { - Streams []ProbeIO `json:"streams"` - Log []string `json:"log"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/process.go b/vendor/github.com/datarhei/core-client-go/v16/api/process.go deleted file mode 100644 index 683c2246..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/process.go +++ /dev/null @@ -1,95 +0,0 @@ -package api - -// Process represents all information on a process -type Process struct { - ID string `json:"id" jsonschema:"minLength=1"` - Owner string `json:"owner"` - Domain string `json:"domain"` - Type string `json:"type" jsonschema:"enum=ffmpeg"` - Reference string `json:"reference"` - CoreID string `json:"core_id"` - CreatedAt int64 `json:"created_at" jsonschema:"minimum=0" format:"int64"` // Unix timestamp - UpdatedAt int64 `json:"updated_at" jsonschema:"minimum=0" format:"int64"` // Unix timestamp - Config *ProcessConfig `json:"config,omitempty"` - State *ProcessState `json:"state,omitempty"` - Report *ProcessReport `json:"report,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` -} - -// ProcessConfigIO represents an input or output of an ffmpeg process config -type ProcessConfigIO struct { - ID string `json:"id"` - Address string `json:"address" validate:"required" jsonschema:"minLength=1"` - Options []string `json:"options"` - Cleanup []ProcessConfigIOCleanup `json:"cleanup"` -} - -type ProcessConfigIOCleanup struct { - Pattern string `json:"pattern" validate:"required"` - MaxFiles uint `json:"max_files" format:"uint"` - MaxFileAge uint `json:"max_file_age_seconds" format:"uint"` // seconds - PurgeOnDelete bool `json:"purge_on_delete"` -} - -type ProcessConfigLimits struct { - CPU float64 `json:"cpu_usage" jsonschema:"minimum=0"` // percent 0-100*ncpu - Memory uint64 `json:"memory_mbytes" jsonschema:"minimum=0" format:"uint64"` // megabytes - WaitFor uint64 `json:"waitfor_seconds" jsonschema:"minimum=0" format:"uint64"` // seconds -} - -// ProcessConfig represents the configuration of an ffmpeg process -type ProcessConfig struct { - ID string `json:"id"` - Owner string `json:"owner"` - Domain string `json:"domain"` - Type string `json:"type" validate:"oneof='ffmpeg' ''" jsonschema:"enum=ffmpeg,enum="` - Reference string `json:"reference"` - Input []ProcessConfigIO `json:"input" validate:"required"` - Output []ProcessConfigIO `json:"output" validate:"required"` - Options []string `json:"options"` - Reconnect bool `json:"reconnect"` - ReconnectDelay uint64 `json:"reconnect_delay_seconds" format:"uint64"` // seconds - Autostart bool `json:"autostart"` - StaleTimeout uint64 `json:"stale_timeout_seconds" format:"uint64"` // seconds - Timeout uint64 `json:"runtime_duration_seconds" format:"uint64"` // seconds - Scheduler string `json:"scheduler"` - LogPatterns []string `json:"log_patterns"` - Limits ProcessConfigLimits `json:"limits"` - Metadata map[string]interface{} `json:"metadata,omitempty"` -} - -// ProcessState represents the current state of an ffmpeg process -type ProcessState struct { - Order string `json:"order" jsonschema:"enum=start,enum=stop"` - State string `json:"exec" jsonschema:"enum=finished,enum=starting,enum=running,enum=finishing,enum=killed,enum=failed"` - Runtime int64 `json:"runtime_seconds" jsonschema:"minimum=0" format:"int64"` // seconds - Reconnect int64 `json:"reconnect_seconds" format:"int64"` // seconds - LastLog string `json:"last_logline"` - Progress *Progress `json:"progress"` - Memory uint64 `json:"memory_bytes" format:"uint64"` // bytes - CPU float64 `json:"cpu_usage" swaggertype:"number" jsonschema:"type=number"` // percent 0-100*ncpu - LimitMode string `json:"limit_mode"` - Resources ProcessUsage `json:"resources"` - Command []string `json:"command"` -} - -type ProcessUsageCPU struct { - NCPU float64 `json:"ncpu" swaggertype:"number" jsonschema:"type=number"` - Current float64 `json:"cur" swaggertype:"number" jsonschema:"type=number"` // percent 0-100*ncpu - Average float64 `json:"avg" swaggertype:"number" jsonschema:"type=number"` // percent 0-100*ncpu - Max float64 `json:"max" swaggertype:"number" jsonschema:"type=number"` // percent 0-100*ncpu - Limit float64 `json:"limit" swaggertype:"number" jsonschema:"type=number"` // percent 0-100*ncpu - IsThrottling bool `json:"throttling"` -} - -type ProcessUsageMemory struct { - Current uint64 `json:"cur" format:"uint64"` // bytes - Average float64 `json:"avg" swaggertype:"number" jsonschema:"type=number"` // bytes - Max uint64 `json:"max" format:"uint64"` // bytes - Limit uint64 `json:"limit" format:"uint64"` // bytes -} - -type ProcessUsage struct { - CPU ProcessUsageCPU `json:"cpu_usage"` - Memory ProcessUsageMemory `json:"memory_bytes"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/progress.go b/vendor/github.com/datarhei/core-client-go/v16/api/progress.go deleted file mode 100644 index 7c456d0e..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/progress.go +++ /dev/null @@ -1,92 +0,0 @@ -package api - -type ProgressIOFramerate struct { - Min float64 `json:"min" swaggertype:"number" jsonschema:"type=number"` - Max float64 `json:"max" swaggertype:"number" jsonschema:"type=number"` - Average float64 `json:"avg" swaggertype:"number" jsonschema:"type=number"` -} - -// ProgressIO represents the progress of an ffmpeg input or output -type ProgressIO struct { - ID string `json:"id" jsonschema:"minLength=1"` - Address string `json:"address" jsonschema:"minLength=1"` - - // General - Index uint64 `json:"index" format:"uint64"` - Stream uint64 `json:"stream" format:"uint64"` - Format string `json:"format"` - Type string `json:"type"` - Codec string `json:"codec"` - Coder string `json:"coder"` - Frame uint64 `json:"frame" format:"uint64"` - Keyframe uint64 `json:"keyframe" format:"uint64"` - Framerate ProgressIOFramerate `json:"framerate"` - FPS float64 `json:"fps" swaggertype:"number" jsonschema:"type=number"` - Packet uint64 `json:"packet" format:"uint64"` - PPS float64 `json:"pps" swaggertype:"number" jsonschema:"type=number"` - Size uint64 `json:"size_kb" format:"uint64"` // kbytes - Bitrate float64 `json:"bitrate_kbit" swaggertype:"number" jsonschema:"type=number"` // kbit/s - Extradata uint64 `json:"extradata_size_bytes" format:"uint64"` // bytes - - // Video - Pixfmt string `json:"pix_fmt,omitempty"` - Quantizer float64 `json:"q,omitempty" swaggertype:"number" jsonschema:"type=number"` - Width uint64 `json:"width,omitempty" format:"uint64"` - Height uint64 `json:"height,omitempty" format:"uint64"` - - // Audio - Sampling uint64 `json:"sampling_hz,omitempty" format:"uint64"` - Layout string `json:"layout,omitempty"` - Channels uint64 `json:"channels,omitempty" format:"uint64"` - - // avstream - AVstream *AVstream `json:"avstream"` -} - -// Progress represents the progress of an ffmpeg process -type Progress struct { - Started bool `json:"started"` - Input []ProgressIO `json:"inputs"` - Output []ProgressIO `json:"outputs"` - Mapping StreamMapping `json:"mapping"` - Frame uint64 `json:"frame" format:"uint64"` - Packet uint64 `json:"packet" format:"uint64"` - FPS float64 `json:"fps" swaggertype:"number" jsonschema:"type=number"` - Quantizer float64 `json:"q" swaggertype:"number" jsonschema:"type=number"` - Size uint64 `json:"size_kb" format:"uint64"` // kbytes - Time float64 `json:"time" swaggertype:"number" jsonschema:"type=number"` - Bitrate float64 `json:"bitrate_kbit" swaggertype:"number" jsonschema:"type=number"` // kbit/s - Speed float64 `json:"speed" swaggertype:"number" jsonschema:"type=number"` - Drop uint64 `json:"drop" format:"uint64"` - Dup uint64 `json:"dup" format:"uint64"` -} - -type GraphElement struct { - Index int `json:"index"` - Name string `json:"name"` - Filter string `json:"filter"` - DstName string `json:"dst_name"` - DstFilter string `json:"dst_filter"` - Inpad string `json:"inpad"` - Outpad string `json:"outpad"` - Timebase string `json:"timebase"` - Type string `json:"type"` // audio or video - Format string `json:"format"` - Sampling uint64 `json:"sampling"` // Hz - Layout string `json:"layout"` - Width uint64 `json:"width"` - Height uint64 `json:"height"` -} - -type GraphMapping struct { - Input int `json:"input"` - Output int `json:"output"` - Index int `json:"index"` - Name string `json:"name"` - Copy bool `json:"copy"` -} - -type StreamMapping struct { - Graphs []GraphElement `json:"graphs"` - Mapping []GraphMapping `json:"mapping"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/report.go b/vendor/github.com/datarhei/core-client-go/v16/api/report.go deleted file mode 100644 index ff5c1a5c..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/report.go +++ /dev/null @@ -1,31 +0,0 @@ -package api - -// ProcessReportEntry represents the logs of a run of a restream process -type ProcessReportEntry struct { - CreatedAt int64 `json:"created_at" format:"int64"` - Prelude []string `json:"prelude,omitempty"` - Log [][2]string `json:"log,omitempty"` - Matches []string `json:"matches,omitempty"` - ExitedAt int64 `json:"exited_at,omitempty" format:"int64"` - ExitState string `json:"exit_state,omitempty"` - Progress *Progress `json:"progress,omitempty"` - Resources *ProcessUsage `json:"resources,omitempty"` -} - -type ProcessReportHistoryEntry struct { - ProcessReportEntry -} - -// ProcessReport represents the current log and the logs of previous runs of a restream process -type ProcessReport struct { - ProcessReportEntry - History []ProcessReportEntry `json:"history"` -} - -type ProcessReportSearchResult struct { - ProcessID string `json:"id"` - Reference string `json:"reference"` - ExitState string `json:"exit_state"` - CreatedAt int64 `json:"created_at" format:"int64"` - ExitedAt int64 `json:"exited_at" format:"int64"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/rtmp.go b/vendor/github.com/datarhei/core-client-go/v16/api/rtmp.go deleted file mode 100644 index ea3f1e08..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/rtmp.go +++ /dev/null @@ -1,6 +0,0 @@ -package api - -// RTMPChannel represents details about a currently connected RTMP publisher -type RTMPChannel struct { - Name string `json:"name" jsonschema:"minLength=1"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/session.go b/vendor/github.com/datarhei/core-client-go/v16/api/session.go deleted file mode 100644 index 558fb4ce..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/session.go +++ /dev/null @@ -1,68 +0,0 @@ -package api - -// SessionStats are the accumulated numbers for the session summary -type SessionStats struct { - TotalSessions uint64 `json:"sessions" format:"uint64"` - TotalRxBytes uint64 `json:"traffic_rx_mb" format:"uint64"` // megabytes - TotalTxBytes uint64 `json:"traffic_tx_mb" format:"uint64"` // megabytes -} - -// SessionPeers is for the grouping by peers in the summary -type SessionPeers struct { - SessionStats - - Locations map[string]SessionStats `json:"local"` -} - -// Session represents an active session -type Session struct { - ID string `json:"id"` - Reference string `json:"reference"` - CreatedAt int64 `json:"created_at" format:"int64"` - Location string `json:"local"` - Peer string `json:"remote"` - Extra map[string]interface{} `json:"extra"` - RxBytes uint64 `json:"bytes_rx" format:"uint64"` - TxBytes uint64 `json:"bytes_tx" format:"uint64"` - RxBitrate float64 `json:"bandwidth_rx_kbit" swaggertype:"number" jsonschema:"type=number"` // kbit/s - TxBitrate float64 `json:"bandwidth_tx_kbit" swaggertype:"number" jsonschema:"type=number"` // kbit/s -} - -// SessionSummaryActive represents the currently active sessions -type SessionSummaryActive struct { - SessionList []Session `json:"list"` - Sessions uint64 `json:"sessions" format:"uint64"` - RxBitrate float64 `json:"bandwidth_rx_mbit" swaggertype:"number" jsonschema:"type=number"` // mbit/s - TxBitrate float64 `json:"bandwidth_tx_mbit" swaggertype:"number" jsonschema:"type=number"` // mbit/s - MaxSessions uint64 `json:"max_sessions" format:"uint64"` - MaxRxBitrate float64 `json:"max_bandwidth_rx_mbit" swaggertype:"number" jsonschema:"type=number"` // mbit/s - MaxTxBitrate float64 `json:"max_bandwidth_tx_mbit" swaggertype:"number" jsonschema:"type=number"` // mbit/s -} - -// SessionSummarySummary represents the summary (history) of all finished sessions -type SessionSummarySummary struct { - Peers map[string]SessionPeers `json:"remote"` - Locations map[string]SessionStats `json:"local"` - References map[string]SessionStats `json:"reference"` - SessionStats -} - -// SessionSummary is the API representation of a session.Summary -type SessionSummary struct { - Active SessionSummaryActive `json:"active"` - - Summary SessionSummarySummary `json:"summary"` -} - -type SessionsSummary map[string]SessionSummary - -// SessionsActive is the API representation of all active sessions -type SessionsActive map[string][]Session - -type SessionTokenRequest struct { - Match string `json:"match"` - Remote []string `json:"remote"` - Extra map[string]interface{} `json:"extra"` - TTL int64 `json:"ttl_sec"` // seconds - Token string `json:"token,omitempty"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/skills.go b/vendor/github.com/datarhei/core-client-go/v16/api/skills.go deleted file mode 100644 index 6e78c14c..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/skills.go +++ /dev/null @@ -1,89 +0,0 @@ -package api - -// SkillsFilter represents an ffmpeg filter -type SkillsFilter struct { - ID string `json:"id"` - Name string `json:"name"` -} - -// SkillsHWAccel represents an ffmpeg HW accelerator -type SkillsHWAccel struct { - ID string `json:"id"` - Name string `json:"name"` -} - -// SkillsCodec represents an ffmpeg codec -type SkillsCodec struct { - ID string `json:"id"` - Name string `json:"name"` - Encoders []string `json:"encoders"` - Decoders []string `json:"decoders"` -} - -// SkillsHWDevice represents an ffmpeg hardware device -type SkillsHWDevice struct { - ID string `json:"id"` - Name string `json:"name"` - Extra string `json:"extra"` - Media string `json:"media"` -} - -// SkillsDevice represents a group of ffmpeg hardware devices -type SkillsDevice struct { - ID string `json:"id"` - Name string `json:"name"` - Devices []SkillsHWDevice `json:"devices"` -} - -// SkillsFormat represents an ffmpeg format -type SkillsFormat struct { - ID string `json:"id"` - Name string `json:"name"` -} - -// SkillsProtocol represents an ffmpeg protocol -type SkillsProtocol struct { - ID string `json:"id"` - Name string `json:"name"` -} - -// SkillsLibrary represents an avlib ffmpeg is compiled and linked with -type SkillsLibrary struct { - Name string `json:"name"` - Compiled string `json:"compiled"` - Linked string `json:"linked"` -} - -// Skills represents a set of ffmpeg capabilities -type Skills struct { - FFmpeg struct { - Version string `json:"version"` - Compiler string `json:"compiler"` - Configuration string `json:"configuration"` - Libraries []SkillsLibrary `json:"libraries"` - } `json:"ffmpeg"` - - Filters []SkillsFilter `json:"filter"` - HWAccels []SkillsHWAccel `json:"hwaccels"` - - Codecs struct { - Audio []SkillsCodec `json:"audio"` - Video []SkillsCodec `json:"video"` - Subtitle []SkillsCodec `json:"subtitle"` - } `json:"codecs"` - - Devices struct { - Demuxers []SkillsDevice `json:"demuxers"` - Muxers []SkillsDevice `json:"muxers"` - } `json:"devices"` - - Formats struct { - Demuxers []SkillsFormat `json:"demuxers"` - Muxers []SkillsFormat `json:"muxers"` - } `json:"formats"` - - Protocols struct { - Input []SkillsProtocol `json:"input"` - Output []SkillsProtocol `json:"output"` - } `json:"protocols"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/srt.go b/vendor/github.com/datarhei/core-client-go/v16/api/srt.go deleted file mode 100644 index 3198cfc2..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/srt.go +++ /dev/null @@ -1,79 +0,0 @@ -package api - -// SRTStatistics represents the statistics of a SRT connection -type SRTStatistics struct { - MsTimeStamp uint64 `json:"timestamp_ms" format:"uint64"` // The time elapsed, in milliseconds, since the SRT socket has been created - - // Accumulated - - PktSent uint64 `json:"sent_pkt" format:"uint64"` // The total number of sent DATA packets, including retransmitted packets - PktRecv uint64 `json:"recv_pkt" format:"uint64"` // The total number of received DATA packets, including retransmitted packets - PktSentUnique uint64 `json:"sent_unique_pkt" format:"uint64"` // The total number of unique DATA packets sent by the SRT sender - PktRecvUnique uint64 `json:"recv_unique_pkt" format:"uint64"` // The total number of unique original, retransmitted or recovered by the packet filter DATA packets received in time, decrypted without errors and, as a result, scheduled for delivery to the upstream application by the SRT receiver. - PktSndLoss uint64 `json:"send_loss_pkt" format:"uint64"` // The total number of data packets considered or reported as lost at the sender side. Does not correspond to the packets detected as lost at the receiver side. - PktRcvLoss uint64 `json:"recv_loss_pkt" format:"uint64"` // The total number of SRT DATA packets detected as presently missing (either reordered or lost) at the receiver side - PktRetrans uint64 `json:"sent_retrans_pkt" format:"uint64"` // The total number of retransmitted packets sent by the SRT sender - PktRcvRetrans uint64 `json:"recv_retran_pkts" format:"uint64"` // The total number of retransmitted packets registered at the receiver side - PktSentACK uint64 `json:"sent_ack_pkt" format:"uint64"` // The total number of sent ACK (Acknowledgement) control packets - PktRecvACK uint64 `json:"recv_ack_pkt" format:"uint64"` // The total number of received ACK (Acknowledgement) control packets - PktSentNAK uint64 `json:"sent_nak_pkt" format:"uint64"` // The total number of sent NAK (Negative Acknowledgement) control packets - PktRecvNAK uint64 `json:"recv_nak_pkt" format:"uint64"` // The total number of received NAK (Negative Acknowledgement) control packets - PktSentKM uint64 `json:"send_km_pkt" format:"uint64"` // The total number of sent KM (Key Material) control packets - PktRecvKM uint64 `json:"recv_km_pkt" format:"uint64"` // The total number of received KM (Key Material) control packets - UsSndDuration uint64 `json:"send_duration_us" format:"uint64"` // The total accumulated time in microseconds, during which the SRT sender has some data to transmit, including packets that have been sent, but not yet acknowledged - PktSndDrop uint64 `json:"send_drop_pkt" format:"uint64"` // The total number of dropped by the SRT sender DATA packets that have no chance to be delivered in time - PktRcvDrop uint64 `json:"recv_drop_pkt" format:"uint64"` // The total number of dropped by the SRT receiver and, as a result, not delivered to the upstream application DATA packets - PktRcvUndecrypt uint64 `json:"recv_undecrypt_pkt" format:"uint64"` // The total number of packets that failed to be decrypted at the receiver side - - ByteSent uint64 `json:"sent_bytes" format:"uint64"` // Same as pktSent, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteRecv uint64 `json:"recv_bytes" format:"uint64"` // Same as pktRecv, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteSentUnique uint64 `json:"sent_unique_bytes" format:"uint64"` // Same as pktSentUnique, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteRecvUnique uint64 `json:"recv_unique_bytes" format:"uint64"` // Same as pktRecvUnique, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteRcvLoss uint64 `json:"recv_loss_bytes" format:"uint64"` // Same as pktRcvLoss, but expressed in bytes, including payload and all the headers (IP, TCP, SRT), bytes for the presently missing (either reordered or lost) packets' payloads are estimated based on the average packet size - ByteRetrans uint64 `json:"sent_retrans_bytes" format:"uint64"` // Same as pktRetrans, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteSndDrop uint64 `json:"send_drop_bytes" format:"uint64"` // Same as pktSndDrop, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteRcvDrop uint64 `json:"recv_drop_bytes" format:"uint64"` // Same as pktRcvDrop, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - ByteRcvUndecrypt uint64 `json:"recv_undecrypt_bytes" format:"uint64"` // Same as pktRcvUndecrypt, but expressed in bytes, including payload and all the headers (IP, TCP, SRT) - - // Instantaneous - - UsPktSndPeriod float64 `json:"pkt_send_period_us"` // Current minimum time interval between which consecutive packets are sent, in microseconds - PktFlowWindow uint64 `json:"flow_window_pkt" format:"uint64"` // The maximum number of packets that can be "in flight" - PktFlightSize uint64 `json:"flight_size_pkt" format:"uint64"` // The number of packets in flight - MsRTT float64 `json:"rtt_ms"` // Smoothed round-trip time (SRTT), an exponentially-weighted moving average (EWMA) of an endpoint's RTT samples, in milliseconds - MbpsBandwidth float64 `json:"bandwidth_mbit"` // Estimated bandwidth of the network link, in Mbps - ByteAvailSndBuf uint64 `json:"avail_send_buf_bytes" format:"uint64"` // The available space in the sender's buffer, in bytes - ByteAvailRcvBuf uint64 `json:"avail_recv_buf_bytes" format:"uint64"` // The available space in the receiver's buffer, in bytes - MbpsMaxBW float64 `json:"max_bandwidth_mbit"` // Transmission bandwidth limit, in Mbps - ByteMSS uint64 `json:"mss_bytes" format:"uint64"` // Maximum Segment Size (MSS), in bytes - PktSndBuf uint64 `json:"send_buf_pkt" format:"uint64"` // The number of packets in the sender's buffer that are already scheduled for sending or even possibly sent, but not yet acknowledged - ByteSndBuf uint64 `json:"send_buf_bytes" format:"uint64"` // Instantaneous (current) value of pktSndBuf, but expressed in bytes, including payload and all headers (IP, TCP, SRT) - MsSndBuf uint64 `json:"send_buf_ms" format:"uint64"` // The timespan (msec) of packets in the sender's buffer (unacknowledged packets) - MsSndTsbPdDelay uint64 `json:"send_tsbpd_delay_ms" format:"uint64"` // Timestamp-based Packet Delivery Delay value of the peer - PktRcvBuf uint64 `json:"recv_buf_pkt" format:"uint64"` // The number of acknowledged packets in receiver's buffer - ByteRcvBuf uint64 `json:"recv_buf_bytes" format:"uint64"` // Instantaneous (current) value of pktRcvBuf, expressed in bytes, including payload and all headers (IP, TCP, SRT) - MsRcvBuf uint64 `json:"recv_buf_ms" format:"uint64"` // The timespan (msec) of acknowledged packets in the receiver's buffer - MsRcvTsbPdDelay uint64 `json:"recv_tsbpd_delay_ms" format:"uint64"` // Timestamp-based Packet Delivery Delay value set on the socket via SRTO_RCVLATENCY or SRTO_LATENCY - PktReorderTolerance uint64 `json:"reorder_tolerance_pkt" format:"uint64"` // Instant value of the packet reorder tolerance - PktRcvAvgBelatedTime uint64 `json:"pkt_recv_avg_belated_time_ms" format:"uint64"` // Accumulated difference between the current time and the time-to-play of a packet that is received late -} - -type SRTLog struct { - Timestamp int64 `json:"ts" format:"int64"` - Message []string `json:"msg"` -} - -// SRTConnection represents a SRT connection with statistics and logs -type SRTConnection struct { - Log map[string][]SRTLog `json:"log"` - Stats SRTStatistics `json:"stats"` -} - -// SRTChannel represents a SRT publishing connection with its subscribers -type SRTChannel struct { - Name string `json:"name"` - SocketId uint32 `json:"socketid"` - Subscriber []uint32 `json:"subscriber"` - Connections map[uint32]SRTConnection `json:"connections"` - Log map[string][]SRTLog `json:"log"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/api/widget.go b/vendor/github.com/datarhei/core-client-go/v16/api/widget.go deleted file mode 100644 index 5d91bda6..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/api/widget.go +++ /dev/null @@ -1,7 +0,0 @@ -package api - -type WidgetProcess struct { - CurrentSessions uint64 `json:"current_sessions" format:"uint64"` - TotalSessions uint64 `json:"total_sessions" format:"uint64"` - Uptime int64 `json:"uptime"` -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster.go b/vendor/github.com/datarhei/core-client-go/v16/cluster.go deleted file mode 100644 index c53e4e1b..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster.go +++ /dev/null @@ -1,83 +0,0 @@ -package coreclient - -import ( - "context" - "fmt" - "io" - "net/url" - "strings" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) Cluster() (*api.ClusterAboutV1, *api.ClusterAboutV2, error) { - data, err := r.call("GET", "/v3/cluster", nil, nil, "", nil) - if err != nil { - return nil, nil, err - } - - var aboutV1 *api.ClusterAboutV1 - var aboutV2 *api.ClusterAboutV2 - - type version struct { - Version string `json:"version"` - } - - v := version{} - - err = json.Unmarshal(data, &v) - if err != nil { - return nil, nil, err - } - - if strings.HasPrefix(v.Version, "1.") { - aboutV1 = &api.ClusterAboutV1{} - - err = json.Unmarshal(data, aboutV1) - if err != nil { - return nil, nil, err - } - } else if strings.HasPrefix(v.Version, "2.") { - aboutV2 = &api.ClusterAboutV2{} - - err = json.Unmarshal(data, aboutV2) - if err != nil { - return nil, nil, err - } - } else { - err = fmt.Errorf("unsupported version (%s)", v.Version) - } - - return aboutV1, aboutV2, err -} - -func (r *restclient) ClusterHealthy() (bool, error) { - var healthy bool - - data, err := r.call("GET", "/v3/cluster/healthy", nil, nil, "", nil) - if err != nil { - return healthy, err - } - - err = json.Unmarshal(data, &healthy) - - return healthy, err -} - -func (r *restclient) ClusterSnapshot() (io.ReadCloser, error) { - return r.stream(context.Background(), "GET", "/v3/cluster/snapshot", nil, nil, "", nil) -} - -func (r *restclient) ClusterLeave() error { - _, err := r.call("PUT", "/v3/cluster/leave", nil, nil, "", nil) - - return err -} - -func (r *restclient) ClusterTransferLeadership(id string) error { - _, err := r.call("PUT", "/v3/cluster/transfer/"+url.PathEscape(id), nil, nil, "", nil) - - return err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster_db.go b/vendor/github.com/datarhei/core-client-go/v16/cluster_db.go deleted file mode 100644 index 3cb6a097..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster_db.go +++ /dev/null @@ -1,116 +0,0 @@ -package coreclient - -import ( - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) ClusterDBProcessList() ([]api.Process, error) { - var processes []api.Process - - data, err := r.call("GET", "/v3/cluster/db/process", nil, nil, "", nil) - if err != nil { - return processes, err - } - - err = json.Unmarshal(data, &processes) - - return processes, err -} - -func (r *restclient) ClusterDBProcess(id ProcessID) (api.Process, error) { - var info api.Process - - values := &url.Values{} - values.Set("domain", id.Domain) - - data, err := r.call("GET", "/v3/cluster/db/process/"+url.PathEscape(id.ID), values, nil, "", nil) - if err != nil { - return info, err - } - - err = json.Unmarshal(data, &info) - - return info, err -} - -func (r *restclient) ClusterDBUserList() ([]api.IAMUser, error) { - var users []api.IAMUser - - data, err := r.call("GET", "/v3/cluster/db/user", nil, nil, "", nil) - if err != nil { - return users, err - } - - err = json.Unmarshal(data, &users) - - return users, err -} - -func (r *restclient) ClusterDBUser(name string) (api.IAMUser, error) { - var user api.IAMUser - - data, err := r.call("GET", "/v3/cluster/db/user/"+url.PathEscape(name), nil, nil, "", nil) - if err != nil { - return user, err - } - - err = json.Unmarshal(data, &user) - - return user, err -} - -func (r *restclient) ClusterDBPolicies() ([]api.IAMPolicy, error) { - var policies []api.IAMPolicy - - data, err := r.call("GET", "/v3/cluster/db/policies", nil, nil, "", nil) - if err != nil { - return policies, err - } - - err = json.Unmarshal(data, &policies) - - return policies, err -} - -func (r *restclient) ClusterDBLocks() ([]api.ClusterLock, error) { - var locks []api.ClusterLock - - data, err := r.call("GET", "/v3/cluster/db/locks", nil, nil, "", nil) - if err != nil { - return locks, err - } - - err = json.Unmarshal(data, &locks) - - return locks, err -} - -func (r *restclient) ClusterDBKeyValues() (api.ClusterKVS, error) { - var kvs api.ClusterKVS - - data, err := r.call("GET", "/v3/cluster/db/kv", nil, nil, "", nil) - if err != nil { - return kvs, err - } - - err = json.Unmarshal(data, &kvs) - - return kvs, err -} - -func (r *restclient) ClusterDBProcessMap() (api.ClusterProcessMap, error) { - var m api.ClusterProcessMap - - data, err := r.call("GET", "/v3/cluster/db/map/process", nil, nil, "", nil) - if err != nil { - return m, err - } - - err = json.Unmarshal(data, &m) - - return m, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster_fs.go b/vendor/github.com/datarhei/core-client-go/v16/cluster_fs.go deleted file mode 100644 index b6905d41..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster_fs.go +++ /dev/null @@ -1,27 +0,0 @@ -package coreclient - -import ( - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) ClusterFilesystemList(storage, pattern, sort, order string) ([]api.FileInfo, error) { - var files []api.FileInfo - - query := &url.Values{} - query.Set("glob", pattern) - query.Set("sort", sort) - query.Set("order", order) - - data, err := r.call("GET", "/v3/cluster/fs/"+url.PathEscape(storage), query, nil, "", nil) - if err != nil { - return files, err - } - - err = json.Unmarshal(data, &files) - - return files, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster_iam.go b/vendor/github.com/datarhei/core-client-go/v16/cluster_iam.go deleted file mode 100644 index 9ad42131..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster_iam.go +++ /dev/null @@ -1,33 +0,0 @@ -package coreclient - -import "github.com/datarhei/core-client-go/v16/api" - -func (r *restclient) ClusterIdentitiesList() ([]api.IAMUser, error) { - return r.identitiesList("cluster") -} - -func (r *restclient) ClusterIdentity(name string) (api.IAMUser, error) { - return r.identity("cluster", name) -} - -func (r *restclient) ClusterIdentityAdd(u api.IAMUser) error { - return r.identityAdd("cluster", u) -} - -func (r *restclient) ClusterIdentityUpdate(name string, u api.IAMUser) error { - return r.identityUpdate("cluster", name, u) -} - -func (r *restclient) ClusterIdentitySetPolicies(name string, p []api.IAMPolicy) error { - return r.identitySetPolicies("cluster", name, p) -} - -func (r *restclient) ClusterIdentityDelete(name string) error { - return r.identityDelete("cluster", name) -} - -func (r *restclient) ClusterIAMReload() error { - _, err := r.call("PUT", "/v3/cluster/iam/reload", nil, nil, "", nil) - - return err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster_node.go b/vendor/github.com/datarhei/core-client-go/v16/cluster_node.go deleted file mode 100644 index e5708c53..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster_node.go +++ /dev/null @@ -1,123 +0,0 @@ -package coreclient - -import ( - "context" - "io" - "net/url" - "path/filepath" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) ClusterNodeList() ([]api.ClusterNode, error) { - var nodes []api.ClusterNode - - data, err := r.call("GET", "/v3/cluster/node", nil, nil, "", nil) - if err != nil { - return nodes, err - } - - err = json.Unmarshal(data, &nodes) - - return nodes, err -} - -func (r *restclient) ClusterNode(id string) (api.ClusterNode, error) { - var node api.ClusterNode - - data, err := r.call("GET", "/v3/cluster/node/"+url.PathEscape(id), nil, nil, "", nil) - if err != nil { - return node, err - } - - err = json.Unmarshal(data, &node) - - return node, err -} - -func (r *restclient) ClusterNodeFiles(id string) (api.ClusterNodeFiles, error) { - var files api.ClusterNodeFiles - - data, err := r.call("GET", "/v3/cluster/node/"+url.PathEscape(id)+"/files", nil, nil, "", nil) - if err != nil { - return files, err - } - - err = json.Unmarshal(data, &files) - - return files, err -} - -func (r *restclient) ClusterNodeFilesystemList(id, storage, pattern, sort, order string) ([]api.FileInfo, error) { - var files []api.FileInfo - - query := &url.Values{} - query.Set("glob", pattern) - query.Set("sort", sort) - query.Set("order", order) - - data, err := r.call("GET", "/v3/cluster/node/"+url.PathEscape(id)+"/fs/"+url.PathEscape(storage), query, nil, "", nil) - if err != nil { - return files, err - } - - err = json.Unmarshal(data, &files) - - return files, err -} - -func (r *restclient) ClusterNodeFilesystemPutFile(id, storage, path string, data io.Reader) error { - if !filepath.IsAbs(path) { - path = "/" + path - } - - _, err := r.call("PUT", "/v3/cluster/node/"+url.PathEscape(id)+"/fs/"+url.PathEscape(storage)+path, nil, nil, "", data) - - return err -} - -func (r *restclient) ClusterNodeFilesystemGetFile(id, storage, path string) (io.ReadCloser, error) { - if !filepath.IsAbs(path) { - path = "/" + path - } - - return r.stream(context.Background(), "GET", "/v3/cluster/node/"+url.PathEscape(id)+"/fs/"+url.PathEscape(storage)+path, nil, nil, "", nil) -} - -func (r *restclient) ClusterNodeFilesystemDeleteFile(id, storage, path string) error { - if !filepath.IsAbs(path) { - path = "/" + path - } - - _, err := r.call("DELETE", "/v3/cluster/node/"+url.PathEscape(id)+"/fs/"+url.PathEscape(storage)+path, nil, nil, "", nil) - - return err -} - -func (r *restclient) ClusterNodeProcessList(id string, opts ProcessListOptions) ([]api.Process, error) { - var processes []api.Process - - data, err := r.call("GET", "/v3/cluster/node/"+url.PathEscape(id)+"/process", opts.Query(), nil, "", nil) - if err != nil { - return processes, err - } - - err = json.Unmarshal(data, &processes) - - return processes, err -} - -func (r *restclient) ClusterNodeVersion(id string) (api.Version, error) { - var version api.Version - - data, err := r.call("GET", "/v3/cluster/node/"+url.PathEscape(id)+"/version", nil, nil, "", nil) - if err != nil { - return version, err - } - - err = json.Unmarshal(data, &version) - - return version, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/cluster_process.go b/vendor/github.com/datarhei/core-client-go/v16/cluster_process.go deleted file mode 100644 index 0d73d97b..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/cluster_process.go +++ /dev/null @@ -1,43 +0,0 @@ -package coreclient - -import "github.com/datarhei/core-client-go/v16/api" - -func (r *restclient) ClusterProcessList(opts ProcessListOptions) ([]api.Process, error) { - return r.processList("cluster", opts) -} - -func (r *restclient) ClusterProcess(id ProcessID, filter []string) (api.Process, error) { - return r.process("cluster", id, filter) -} - -func (r *restclient) ClusterProcessAdd(p api.ProcessConfig) error { - return r.processAdd("cluster", p) -} - -func (r *restclient) ClusterProcessUpdate(id ProcessID, p api.ProcessConfig) error { - return r.processUpdate("cluster", id, p) -} - -func (r *restclient) ClusterProcessDelete(id ProcessID) error { - return r.processDelete("cluster", id) -} - -func (r *restclient) ClusterProcessCommand(id ProcessID, command string) error { - return r.processCommand("cluster", id, command) -} - -func (r *restclient) ClusterProcessMetadata(id ProcessID, key string) (api.Metadata, error) { - return r.processMetadata("cluster", id, key) -} - -func (r *restclient) ClusterProcessMetadataSet(id ProcessID, key string, metadata api.Metadata) error { - return r.processMetadataSet("cluster", id, key, metadata) -} - -func (r *restclient) ClusterProcessProbe(id ProcessID) (api.Probe, error) { - return r.processProbe("cluster", id) -} - -func (r *restclient) ClusterProcessProbeConfig(config api.ProcessConfig, coreid string) (api.Probe, error) { - return r.processProbeConfig("cluster", config, coreid) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/config.go b/vendor/github.com/datarhei/core-client-go/v16/config.go deleted file mode 100644 index 2e634901..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/config.go +++ /dev/null @@ -1,92 +0,0 @@ -package coreclient - -import ( - "bytes" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -type configVersion struct { - Config struct { - Version int64 `json:"version"` - } `json:"config"` -} - -func (r *restclient) Config() (int64, api.Config, error) { - version := configVersion{} - - data, err := r.call("GET", "/v3/config", nil, nil, "", nil) - if err != nil { - return 0, api.Config{}, err - } - - if err := json.Unmarshal(data, &version); err != nil { - return 0, api.Config{}, err - } - - config := api.Config{} - - if err := json.Unmarshal(data, &config); err != nil { - return 0, api.Config{}, err - } - - configdata, err := json.Marshal(config.Config) - if err != nil { - return 0, api.Config{}, err - } - - switch version.Config.Version { - case 1: - cfg := api.ConfigV1{} - err := json.Unmarshal(configdata, &cfg) - if err != nil { - return 0, api.Config{}, err - } - config.Config = cfg - case 2: - cfg := api.ConfigV2{} - err := json.Unmarshal(configdata, &cfg) - if err != nil { - return 0, api.Config{}, err - } - config.Config = cfg - case 3: - cfg := api.ConfigV3{} - err := json.Unmarshal(configdata, &cfg) - if err != nil { - return 0, api.Config{}, err - } - config.Config = cfg - } - - return version.Config.Version, config, nil -} - -func (r *restclient) ConfigSet(config interface{}) error { - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(config) - - _, err := r.call("PUT", "/v3/config", nil, nil, "application/json", &buf) - - if e, ok := err.(api.Error); ok { - if e.Code == 409 { - ce := api.ConfigError{} - if err := json.Unmarshal(e.Body, &ce); err != nil { - return e - } - return ce - } - } - - return err -} - -func (r *restclient) ConfigReload() error { - _, err := r.call("GET", "/v3/config/reload", nil, nil, "", nil) - - return err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/diskfs.go b/vendor/github.com/datarhei/core-client-go/v16/diskfs.go deleted file mode 100644 index 6ff9e2b6..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/diskfs.go +++ /dev/null @@ -1,27 +0,0 @@ -package coreclient - -import ( - "io" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) DiskFSList(sort, order string) ([]api.FileInfo, error) { - return r.FilesystemList("disk", "", sort, order) -} - -func (r *restclient) DiskFSHasFile(path string) bool { - return r.FilesystemHasFile("disk", path) -} - -func (r *restclient) DiskFSGetFile(path string) (io.ReadCloser, error) { - return r.FilesystemGetFile("disk", path) -} - -func (r *restclient) DiskFSDeleteFile(path string) error { - return r.FilesystemDeleteFile("disk", path) -} - -func (r *restclient) DiskFSAddFile(path string, data io.Reader) error { - return r.FilesystemAddFile("disk", path, data) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/doc.go b/vendor/github.com/datarhei/core-client-go/v16/doc.go deleted file mode 100644 index 6c435379..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/doc.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Package coreclient provides a client for the datarhei Core API (https://github.com/datarhei/core) - -Example for retrieving a list of all processes: - - import "github.com/datarhei/core-client-go/v16" - - client, err := coreclient.New(coreclient.Config{ - Address: "https://example.com:8080", - Username: "foo", - Password: "bar", - }) - if err != nil { - ... - } - - processes, err := client.ProcessList(coreclient.ProcessListOptions{}) - if err != nil { - ... - } -*/ -package coreclient diff --git a/vendor/github.com/datarhei/core-client-go/v16/graph.go b/vendor/github.com/datarhei/core-client-go/v16/graph.go deleted file mode 100644 index 0fc5c2f2..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/graph.go +++ /dev/null @@ -1,26 +0,0 @@ -package coreclient - -import ( - "bytes" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) Graph(query api.GraphQuery) (api.GraphResponse, error) { - var resp api.GraphResponse - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(query) - - data, err := r.call("PUT", "/v3/graph", nil, nil, "application/json", &buf) - if err != nil { - return resp, err - } - - err = json.Unmarshal(data, &resp) - - return resp, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/iam.go b/vendor/github.com/datarhei/core-client-go/v16/iam.go deleted file mode 100644 index c21fc606..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/iam.go +++ /dev/null @@ -1,138 +0,0 @@ -package coreclient - -import ( - "bytes" - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) identitiesList(where string) ([]api.IAMUser, error) { - var users []api.IAMUser - - path := "/v3/iam/user" - if where == "cluster" { - path = "/v3/cluster/iam/user" - } - - data, err := r.call("GET", path, nil, nil, "", nil) - if err != nil { - return users, err - } - - err = json.Unmarshal(data, &users) - - return users, err -} - -func (r *restclient) identity(where, name string) (api.IAMUser, error) { - var user api.IAMUser - - path := "/v3/iam/user/" + url.PathEscape(name) - if where == "cluster" { - path = "/v3/cluster/iam/user/" + url.PathEscape(name) - } - - data, err := r.call("GET", path, nil, nil, "", nil) - if err != nil { - return user, err - } - - err = json.Unmarshal(data, &user) - - return user, err -} - -func (r *restclient) identityAdd(where string, u api.IAMUser) error { - var buf bytes.Buffer - - path := "/v3/iam/user" - if where == "cluster" { - path = "/v3/cluster/iam/user" - } - - e := json.NewEncoder(&buf) - e.Encode(u) - - _, err := r.call("POST", path, nil, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) identityUpdate(where, name string, u api.IAMUser) error { - var buf bytes.Buffer - - path := "/v3/iam/user/" + url.PathEscape(name) - if where == "cluster" { - path = "/v3/cluster/iam/user/" + url.PathEscape(name) - } - - e := json.NewEncoder(&buf) - e.Encode(u) - - _, err := r.call("PUT", path, nil, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) identitySetPolicies(where, name string, p []api.IAMPolicy) error { - var buf bytes.Buffer - - path := "/v3/iam/user/" + url.PathEscape(name) + "/policy" - if where == "cluster" { - path = "/v3/cluster/iam/user/" + url.PathEscape(name) + "/policy" - } - - e := json.NewEncoder(&buf) - e.Encode(p) - - _, err := r.call("PUT", path, nil, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) identityDelete(where, name string) error { - path := "/v3/iam/user/" + url.PathEscape(name) - if where == "cluster" { - path = "/v3/cluster/iam/user/" + url.PathEscape(name) - } - - _, err := r.call("DELETE", path, nil, nil, "", nil) - - return err -} - -func (r *restclient) IdentitiesList() ([]api.IAMUser, error) { - return r.identitiesList("") -} - -func (r *restclient) Identity(name string) (api.IAMUser, error) { - return r.identity("", name) -} - -func (r *restclient) IdentityAdd(u api.IAMUser) error { - return r.identityAdd("", u) -} - -func (r *restclient) IdentityUpdate(name string, u api.IAMUser) error { - return r.identityUpdate("", name, u) -} - -func (r *restclient) IdentitySetPolicies(name string, p []api.IAMPolicy) error { - return r.identitySetPolicies("", name, p) -} - -func (r *restclient) IdentityDelete(name string) error { - return r.identityDelete("", name) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/log.go b/vendor/github.com/datarhei/core-client-go/v16/log.go deleted file mode 100644 index 6b1ff952..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/log.go +++ /dev/null @@ -1,25 +0,0 @@ -package coreclient - -import ( - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) Log() ([]api.LogEvent, error) { - var log []api.LogEvent - - query := &url.Values{} - query.Set("format", "raw") - - data, err := r.call("GET", "/v3/log", query, nil, "", nil) - if err != nil { - return log, err - } - - err = json.Unmarshal(data, &log) - - return log, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/memfs.go b/vendor/github.com/datarhei/core-client-go/v16/memfs.go deleted file mode 100644 index 5bdd4e0c..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/memfs.go +++ /dev/null @@ -1,27 +0,0 @@ -package coreclient - -import ( - "io" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) MemFSList(sort, order string) ([]api.FileInfo, error) { - return r.FilesystemList("mem", "", sort, order) -} - -func (r *restclient) MemFSHasFile(path string) bool { - return r.FilesystemHasFile("mem", path) -} - -func (r *restclient) MemFSGetFile(path string) (io.ReadCloser, error) { - return r.FilesystemGetFile("mem", path) -} - -func (r *restclient) MemFSDeleteFile(path string) error { - return r.FilesystemDeleteFile("mem", path) -} - -func (r *restclient) MemFSAddFile(path string, data io.Reader) error { - return r.FilesystemAddFile("mem", path, data) -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/metadata.go b/vendor/github.com/datarhei/core-client-go/v16/metadata.go deleted file mode 100644 index 05d73f30..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/metadata.go +++ /dev/null @@ -1,47 +0,0 @@ -package coreclient - -import ( - "bytes" - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) Metadata(key string) (api.Metadata, error) { - var m api.Metadata - - path := "/v3/metadata" - if len(key) != 0 { - path += "/" + url.PathEscape(key) - } - - data, err := r.call("GET", path, nil, nil, "", nil) - if err != nil { - return m, err - } - - err = json.Unmarshal(data, &m) - - return m, err -} - -func (r *restclient) MetadataSet(key string, metadata api.Metadata) error { - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(metadata) - - path := "/v3/metadata" - if len(key) != 0 { - path += "/" + url.PathEscape(key) - } - - _, err := r.call("PUT", path, nil, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/metrics.go b/vendor/github.com/datarhei/core-client-go/v16/metrics.go deleted file mode 100644 index 6a52cb00..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/metrics.go +++ /dev/null @@ -1,39 +0,0 @@ -package coreclient - -import ( - "bytes" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) MetricsList() ([]api.MetricsDescription, error) { - descriptions := []api.MetricsDescription{} - - data, err := r.call("GET", "/v3/metrics", nil, nil, "application/json", nil) - if err != nil { - return descriptions, err - } - - err = json.Unmarshal(data, &descriptions) - - return descriptions, err -} - -func (r *restclient) Metrics(query api.MetricsQuery) (api.MetricsResponse, error) { - var m api.MetricsResponse - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(query) - - data, err := r.call("POST", "/v3/metrics", nil, nil, "application/json", &buf) - if err != nil { - return m, err - } - - err = json.Unmarshal(data, &m) - - return m, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/playout.go b/vendor/github.com/datarhei/core-client-go/v16/playout.go deleted file mode 100644 index 61870b70..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/playout.go +++ /dev/null @@ -1,27 +0,0 @@ -package coreclient - -import ( - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) PlayoutStatus(id ProcessID, inputID string) (api.PlayoutStatus, error) { - var status api.PlayoutStatus - - path := "/v3/process/" + url.PathEscape(id.ID) + "/playout/" + url.PathEscape(inputID) + "/status" - - values := &url.Values{} - values.Set("domain", id.Domain) - - data, err := r.call("GET", path, values, nil, "", nil) - if err != nil { - return status, err - } - - err = json.Unmarshal(data, &status) - - return status, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/process.go b/vendor/github.com/datarhei/core-client-go/v16/process.go deleted file mode 100644 index 020f9727..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/process.go +++ /dev/null @@ -1,369 +0,0 @@ -package coreclient - -import ( - "bytes" - "net/url" - "strings" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -type ProcessID struct { - ID string - Domain string -} - -func NewProcessID(id, domain string) ProcessID { - return ProcessID{ - ID: id, - Domain: domain, - } -} - -func ParseProcessID(pid string) ProcessID { - i := strings.LastIndex(pid, "@") - if i == -1 { - return NewProcessID(pid, "") - } - - return NewProcessID(pid[:i], pid[i+1:]) -} - -func ProcessIDFromProcess(p api.Process) ProcessID { - return NewProcessID(p.ID, p.Config.Domain) -} - -func (p ProcessID) String() string { - return p.ID + "@" + p.Domain -} - -type ProcessListOptions struct { - ID []string - Filter []string - Domain string - Reference string - IDPattern string - RefPattern string - OwnerPattern string - DomainPattern string -} - -func (p *ProcessListOptions) Query() *url.Values { - values := &url.Values{} - values.Set("id", strings.Join(p.ID, ",")) - values.Set("filter", strings.Join(p.Filter, ",")) - values.Set("domain", p.Domain) - values.Set("reference", p.Reference) - values.Set("idpattern", p.IDPattern) - values.Set("refpattern", p.RefPattern) - values.Set("ownerpattern", p.OwnerPattern) - values.Set("domainpattern", p.DomainPattern) - - return values -} - -func (r *restclient) processList(where string, opts ProcessListOptions) ([]api.Process, error) { - var processes []api.Process - - path := "/v3/process" - if where == "cluster" { - path = "/v3/cluster/process" - } - - data, err := r.call("GET", path, opts.Query(), nil, "", nil) - if err != nil { - return processes, err - } - - err = json.Unmarshal(data, &processes) - - return processes, err -} - -func (r *restclient) process(where string, id ProcessID, filter []string) (api.Process, error) { - var info api.Process - - path := "/v3/process/" + url.PathEscape(id.ID) - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) - } - - values := &url.Values{} - values.Set("filter", strings.Join(filter, ",")) - values.Set("domain", id.Domain) - - data, err := r.call("GET", path, values, nil, "", nil) - if err != nil { - return info, err - } - - err = json.Unmarshal(data, &info) - - return info, err -} - -func (r *restclient) processAdd(where string, p api.ProcessConfig) error { - var buf bytes.Buffer - - path := "/v3/process" - if where == "cluster" { - path = "/v3/cluster/process" - } - - e := json.NewEncoder(&buf) - e.Encode(p) - - _, err := r.call("POST", path, nil, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) processUpdate(where string, id ProcessID, p api.ProcessConfig) error { - var buf bytes.Buffer - - path := "/v3/process/" + url.PathEscape(id.ID) - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) - } - - e := json.NewEncoder(&buf) - e.Encode(p) - - query := &url.Values{} - query.Set("domain", id.Domain) - - _, err := r.call("PUT", path, query, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) processDelete(where string, id ProcessID) error { - path := "/v3/process/" + url.PathEscape(id.ID) - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) - } - - query := &url.Values{} - query.Set("domain", id.Domain) - - r.call("DELETE", path, query, nil, "", nil) - - return nil -} - -func (r *restclient) processCommand(where string, id ProcessID, command string) error { - var buf bytes.Buffer - - path := "/v3/process/" + url.PathEscape(id.ID) + "/command" - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) + "/command" - } - - e := json.NewEncoder(&buf) - e.Encode(api.Command{ - Command: command, - }) - - query := &url.Values{} - query.Set("domain", id.Domain) - - _, err := r.call("PUT", path, query, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) processMetadata(where string, id ProcessID, key string) (api.Metadata, error) { - var m api.Metadata - - path := "/v3/process/" + url.PathEscape(id.ID) + "/metadata" - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) + "/metadata" - } - - if len(key) != 0 { - path += "/" + url.PathEscape(key) - } - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", path, query, nil, "", nil) - if err != nil { - return m, err - } - - err = json.Unmarshal(data, &m) - - return m, err -} - -func (r *restclient) processMetadataSet(where string, id ProcessID, key string, metadata api.Metadata) error { - var buf bytes.Buffer - - path := "/v3/process/" + url.PathEscape(id.ID) + "/metadata/" + url.PathEscape(key) - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) + "/metadata/" + url.PathEscape(key) - } - - e := json.NewEncoder(&buf) - e.Encode(metadata) - - query := &url.Values{} - query.Set("domain", id.Domain) - - _, err := r.call("PUT", path, query, nil, "application/json", &buf) - if err != nil { - return err - } - - return nil -} - -func (r *restclient) processProbe(where string, id ProcessID) (api.Probe, error) { - var p api.Probe - - path := "/v3/process/" + url.PathEscape(id.ID) + "/probe" - if where == "cluster" { - path = "/v3/cluster/process/" + url.PathEscape(id.ID) + "/probe" - } - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", path, query, nil, "", nil) - if err != nil { - return p, err - } - - err = json.Unmarshal(data, &p) - - return p, err -} - -func (r *restclient) processProbeConfig(where string, config api.ProcessConfig, coreid string) (api.Probe, error) { - var p api.Probe - - query := &url.Values{} - - path := "/v3/process/probe" - if where == "cluster" { - path = "/v3/cluster/process/probe" - - if len(coreid) != 0 { - query.Set("coreid", coreid) - } - } - - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(config) - - data, err := r.call("POST", path, query, nil, "application/json", &buf) - if err != nil { - return p, err - } - - err = json.Unmarshal(data, &p) - - return p, err -} - -func (r *restclient) ProcessList(opts ProcessListOptions) ([]api.Process, error) { - return r.processList("", opts) -} - -func (r *restclient) Process(id ProcessID, filter []string) (api.Process, error) { - return r.process("", id, filter) -} - -func (r *restclient) ProcessAdd(p api.ProcessConfig) error { - return r.processAdd("", p) -} - -func (r *restclient) ProcessUpdate(id ProcessID, p api.ProcessConfig) error { - return r.processUpdate("", id, p) -} - -func (r *restclient) ProcessDelete(id ProcessID) error { - return r.processDelete("", id) -} - -func (r *restclient) ProcessCommand(id ProcessID, command string) error { - return r.processCommand("", id, command) -} - -func (r *restclient) ProcessMetadata(id ProcessID, key string) (api.Metadata, error) { - return r.processMetadata("", id, key) -} - -func (r *restclient) ProcessMetadataSet(id ProcessID, key string, metadata api.Metadata) error { - return r.processMetadataSet("", id, key, metadata) -} - -func (r *restclient) ProcessProbe(id ProcessID) (api.Probe, error) { - return r.processProbe("", id) -} - -func (r *restclient) ProcessProbeConfig(config api.ProcessConfig) (api.Probe, error) { - return r.processProbeConfig("", config, "") -} - -func (r *restclient) ProcessConfig(id ProcessID) (api.ProcessConfig, error) { - var p api.ProcessConfig - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/config", query, nil, "", nil) - if err != nil { - return p, err - } - - err = json.Unmarshal(data, &p) - - return p, err -} - -func (r *restclient) ProcessReport(id ProcessID) (api.ProcessReport, error) { - var p api.ProcessReport - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/report", query, nil, "", nil) - if err != nil { - return p, err - } - - err = json.Unmarshal(data, &p) - - return p, err -} - -func (r *restclient) ProcessState(id ProcessID) (api.ProcessState, error) { - var p api.ProcessState - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", "/v3/process/"+url.PathEscape(id.ID)+"/state", query, nil, "", nil) - if err != nil { - return p, err - } - - err = json.Unmarshal(data, &p) - - return p, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/session.go b/vendor/github.com/datarhei/core-client-go/v16/session.go deleted file mode 100644 index efdaa8b3..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/session.go +++ /dev/null @@ -1,60 +0,0 @@ -package coreclient - -import ( - "bytes" - "net/url" - "strings" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) Sessions(collectors []string) (api.SessionsSummary, error) { - var sessions api.SessionsSummary - - query := &url.Values{} - query.Set("collectors", strings.Join(collectors, ",")) - - data, err := r.call("GET", "/v3/session", query, nil, "", nil) - if err != nil { - return sessions, err - } - - err = json.Unmarshal(data, &sessions) - - return sessions, err -} - -func (r *restclient) SessionsActive(collectors []string) (api.SessionsActive, error) { - var sessions api.SessionsActive - - query := &url.Values{} - query.Set("collectors", strings.Join(collectors, ",")) - - data, err := r.call("GET", "/v3/session/active", query, nil, "", nil) - if err != nil { - return sessions, err - } - - err = json.Unmarshal(data, &sessions) - - return sessions, err -} - -func (r *restclient) SessionToken(name string, req []api.SessionTokenRequest) ([]api.SessionTokenRequest, error) { - var tokens []api.SessionTokenRequest - var buf bytes.Buffer - - e := json.NewEncoder(&buf) - e.Encode(req) - - data, err := r.call("PUT", "/v3/session/token/"+url.PathEscape(name), nil, nil, "application/json", &buf) - if err != nil { - return tokens, err - } - - err = json.Unmarshal(data, &tokens) - - return tokens, err -} diff --git a/vendor/github.com/datarhei/core-client-go/v16/widget.go b/vendor/github.com/datarhei/core-client-go/v16/widget.go deleted file mode 100644 index 1a583013..00000000 --- a/vendor/github.com/datarhei/core-client-go/v16/widget.go +++ /dev/null @@ -1,25 +0,0 @@ -package coreclient - -import ( - "net/url" - - "github.com/goccy/go-json" - - "github.com/datarhei/core-client-go/v16/api" -) - -func (r *restclient) WidgetProcess(id ProcessID) (api.WidgetProcess, error) { - var w api.WidgetProcess - - query := &url.Values{} - query.Set("domain", id.Domain) - - data, err := r.call("GET", "/v3/widget/process"+url.PathEscape(id.ID), query, nil, "", nil) - if err != nil { - return w, err - } - - err = json.Unmarshal(data, &w) - - return w, err -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3fb08275..337bc776 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -85,10 +85,6 @@ github.com/cespare/xxhash/v2 # github.com/cpuguy83/go-md2man/v2 v2.0.4 ## explicit; go 1.11 github.com/cpuguy83/go-md2man/v2/md2man -# github.com/datarhei/core-client-go/v16 v16.11.1-0.20240429143858-23ad5985b894 -## explicit; go 1.18 -github.com/datarhei/core-client-go/v16 -github.com/datarhei/core-client-go/v16/api # github.com/datarhei/gosrt v0.6.0 ## explicit; go 1.20 github.com/datarhei/gosrt