From d391e274d747c3f24ad8b9ffe9d7694743f7250c Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Thu, 25 Jul 2024 21:13:49 +0200 Subject: [PATCH] Fix wrong memory limit, add total memory, add cpu and memory consumed by core itself to node resources --- cluster/about.go | 6 ++++++ cluster/api.go | 5 ++++- cluster/client/client.go | 3 +++ cluster/node/node.go | 6 ++++++ http/api/cluster.go | 3 +++ http/handler/api/cluster.go | 3 +++ resources/resources.go | 15 +++++++++++++++ 7 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cluster/about.go b/cluster/about.go index d5216bdc..3356faec 100644 --- a/cluster/about.go +++ b/cluster/about.go @@ -22,8 +22,11 @@ type ClusterNodeResources struct { NCPU float64 // Number of CPU on this node CPU float64 // Current CPU load, 0-100*ncpu CPULimit float64 // Defined CPU load limit, 0-100*ncpu + CPUCore float64 // Current CPU load of the core itself, 0-100*ncpu Mem uint64 // Currently used memory in bytes MemLimit uint64 // Defined memory limit in bytes + MemTotal uint64 // Total available memory in bytes + MemCore uint64 // Current used memory of the core itself in bytes Error error } @@ -145,8 +148,11 @@ func (c *cluster) About() (ClusterAbout, error) { NCPU: nodeAbout.Resources.NCPU, CPU: nodeAbout.Resources.CPU, CPULimit: nodeAbout.Resources.CPULimit, + CPUCore: nodeAbout.Resources.CPUCore, Mem: nodeAbout.Resources.Mem, MemLimit: nodeAbout.Resources.MemLimit, + MemTotal: nodeAbout.Resources.MemTotal, + MemCore: nodeAbout.Resources.MemCore, Error: nodeAbout.Resources.Error, }, } diff --git a/cluster/api.go b/cluster/api.go index bd3c5ca6..de2f865b 100644 --- a/cluster/api.go +++ b/cluster/api.go @@ -187,8 +187,11 @@ func (a *api) About(c echo.Context) error { NCPU: resources.CPU.NCPU, CPU: (100 - resources.CPU.Idle) * resources.CPU.NCPU, CPULimit: resources.CPU.Limit * resources.CPU.NCPU, + CPUCore: resources.CPU.Core * resources.CPU.NCPU, Mem: resources.Mem.Total - resources.Mem.Available, - MemLimit: resources.Mem.Total, + MemLimit: resources.Mem.Limit, + MemTotal: resources.Mem.Total, + MemCore: resources.Mem.Core, }, } diff --git a/cluster/client/client.go b/cluster/client/client.go index 3040c036..84ab0230 100644 --- a/cluster/client/client.go +++ b/cluster/client/client.go @@ -88,8 +88,11 @@ type AboutResponseResources struct { NCPU float64 `json:"ncpu"` // Number of CPU on this node CPU float64 `json:"cpu"` // Current CPU load, 0-100*ncpu CPULimit float64 `json:"cpu_limit"` // Defined CPU load limit, 0-100*ncpu + CPUCore float64 `json:"cpu_core"` // Current CPU load of the core itself, 0-100*ncpu Mem uint64 `json:"memory_bytes"` // Currently used memory in bytes MemLimit uint64 `json:"memory_limit_bytes"` // Defined memory limit in bytes + MemTotal uint64 `json:"memory_total_bytes"` // Total available memory in bytes + MemCore uint64 `json:"memory_core_bytes"` // Current used memory of the core itself in bytes Error string `json:"error"` // Last error } diff --git a/cluster/node/node.go b/cluster/node/node.go index 5d429463..078da13a 100644 --- a/cluster/node/node.go +++ b/cluster/node/node.go @@ -143,8 +143,11 @@ type Resources struct { NCPU float64 // Number of CPU on this node CPU float64 // Current CPU load, 0-100*ncpu CPULimit float64 // Defined CPU load limit, 0-100*ncpu + CPUCore float64 // Current CPU load of the core itself, 0-100*ncpu Mem uint64 // Currently used memory in bytes MemLimit uint64 // Defined memory limit in bytes + MemTotal uint64 // Total available memory in bytes + MemCore uint64 // Current used memory of the core itself in bytes Error error // Last error } @@ -503,8 +506,11 @@ func (n *Node) ping(ctx context.Context, interval time.Duration) { NCPU: about.Resources.NCPU, CPU: about.Resources.CPU, CPULimit: about.Resources.CPULimit, + CPUCore: about.Resources.CPUCore, Mem: about.Resources.Mem, MemLimit: about.Resources.MemLimit, + MemTotal: about.Resources.MemTotal, + MemCore: about.Resources.MemCore, Error: nil, }, } diff --git a/http/api/cluster.go b/http/api/cluster.go index 29a896b3..1bfb45bc 100644 --- a/http/api/cluster.go +++ b/http/api/cluster.go @@ -43,8 +43,11 @@ type ClusterNodeResources struct { NCPU float64 `json:"ncpu"` CPU float64 `json:"cpu_used"` // percent 0-100*npcu CPULimit float64 `json:"cpu_limit"` // percent 0-100*npcu + CPUCore float64 `json:"cpu_core"` // percent 0-100*ncpu Mem uint64 `json:"memory_used_bytes"` // bytes MemLimit uint64 `json:"memory_limit_bytes"` // bytes + MemTotal uint64 `json:"memory_total_bytes"` // bytes + MemCore uint64 `json:"memory_core_bytes"` // bytes Error string `json:"error"` } diff --git a/http/handler/api/cluster.go b/http/handler/api/cluster.go index a4e00639..10841647 100644 --- a/http/handler/api/cluster.go +++ b/http/handler/api/cluster.go @@ -118,8 +118,11 @@ func (h *ClusterHandler) marshalClusterNode(node cluster.ClusterNode) api.Cluste NCPU: node.Resources.NCPU, CPU: node.Resources.CPU, CPULimit: node.Resources.CPULimit, + CPUCore: node.Resources.CPUCore, Mem: node.Resources.Mem, MemLimit: node.Resources.MemLimit, + MemTotal: node.Resources.MemTotal, + MemCore: node.Resources.MemCore, }, } diff --git a/resources/resources.go b/resources/resources.go index 5d419824..d7255f05 100644 --- a/resources/resources.go +++ b/resources/resources.go @@ -3,6 +3,7 @@ package resources import ( "context" "fmt" + "os" "sync" "time" @@ -20,6 +21,7 @@ type MemoryInfo struct { Available uint64 // bytes Used uint64 // bytes Limit uint64 // bytes + Core uint64 // bytes Throttling bool Error error } @@ -31,6 +33,7 @@ type CPUInfo struct { Idle float64 // percent 0-100 Other float64 // percent 0-100 Limit float64 // percent 0-100 + Core float64 // percent 0-100 Throttling bool Error error } @@ -46,6 +49,8 @@ type resources struct { isCPULimiting bool isMemoryLimiting bool + self psutil.Process + cancelObserver context.CancelFunc lock sync.RWMutex @@ -137,6 +142,11 @@ func New(config Config) (Resources, error) { "max_memory": r.maxMemory, }) + r.self, err = psutil.NewProcess(int32(os.Getpid()), false) + if err != nil { + return nil, fmt.Errorf("unable to create process observer for self: %w", err) + } + r.logger.Debug().Log("Created") r.stopOnce.Do(func() {}) @@ -160,6 +170,7 @@ func (r *resources) Start() { func (r *resources) Stop() { r.stopOnce.Do(func() { r.cancelObserver() + r.self.Stop() r.startOnce = sync.Once{} @@ -326,6 +337,8 @@ func (r *resources) Info() Info { cpustat, cpuerr := r.psutil.CPUPercent() memstat, memerr := r.psutil.VirtualMemory() + selfcpu, _ := r.self.CPUPercent() + selfmem, _ := r.self.VirtualMemory() cpuinfo := CPUInfo{ NCPU: r.ncpu, @@ -334,6 +347,7 @@ func (r *resources) Info() Info { Idle: cpustat.Idle, Other: cpustat.Other, Limit: cpulimit, + Core: selfcpu.System + selfcpu.User + selfcpu.Other, Throttling: cputhrottling, Error: cpuerr, } @@ -343,6 +357,7 @@ func (r *resources) Info() Info { Available: memstat.Available, Used: memstat.Used, Limit: memlimit, + Core: selfmem, Throttling: memthrottling, Error: memerr, }