mirror of
https://github.com/datarhei/core.git
synced 2025-10-31 11:26:52 +08:00
553 lines
12 KiB
Go
553 lines
12 KiB
Go
package process
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/datarhei/core/v16/log"
|
|
"github.com/datarhei/core/v16/psutil"
|
|
)
|
|
|
|
type Usage struct {
|
|
CPU struct {
|
|
NCPU float64 // number of logical processors
|
|
Current float64 // percent 0-100*ncpu
|
|
Average float64 // percent 0-100*ncpu
|
|
Max float64 // percent 0-100*ncpu
|
|
Limit float64 // percent 0-100*ncpu
|
|
IsThrottling bool
|
|
}
|
|
Memory struct {
|
|
Current uint64 // bytes
|
|
Average float64 // bytes
|
|
Max uint64 // bytes
|
|
Limit uint64 // bytes
|
|
}
|
|
}
|
|
|
|
type LimitFunc func(cpu float64, memory uint64)
|
|
|
|
type LimitMode int
|
|
|
|
func (m LimitMode) String() string {
|
|
if m == LimitModeHard {
|
|
return "hard"
|
|
}
|
|
|
|
if m == LimitModeSoft {
|
|
return "soft"
|
|
}
|
|
|
|
return "undefined"
|
|
}
|
|
|
|
const (
|
|
LimitModeHard LimitMode = 0 // Killing the process if either CPU or memory is above the limit for a certain time
|
|
LimitModeSoft LimitMode = 1 // Throttling the CPU if activated, killing the process if memory is above the limit for a certain time
|
|
)
|
|
|
|
type LimiterConfig struct {
|
|
CPU float64 // Max. CPU usage in percent 0-100 in hard mode, 0-100*ncpu in softmode
|
|
Memory uint64 // Max. memory usage in bytes
|
|
WaitFor time.Duration // Duration for one of the limits has to be above the limit until OnLimit gets triggered
|
|
OnLimit LimitFunc // Function to be triggered if limits are exceeded
|
|
Mode LimitMode // How to limit CPU usage
|
|
PSUtil psutil.Util
|
|
Logger log.Logger
|
|
}
|
|
|
|
type Limiter interface {
|
|
// Start starts the limiter with a psutil.Process.
|
|
Start(process psutil.Process) error
|
|
|
|
// Stop stops the limiter. The limiter can be reused by calling Start() again
|
|
Stop()
|
|
|
|
// Current returns the current CPU and memory values
|
|
// Deprecated: use Usage()
|
|
Current() (cpu float64, memory uint64)
|
|
|
|
// Limits returns the defined CPU and memory limits. Values <= 0 means no limit
|
|
// Deprecated: use Usage()
|
|
Limits() (cpu float64, memory uint64)
|
|
|
|
// Usage returns the current state of the limiter, such as current, average, max, and
|
|
// limit values for CPU and memory.
|
|
Usage() Usage
|
|
|
|
// Limit enables or disables the throttling of the CPU or killing because of to much
|
|
// memory consumption.
|
|
Limit(cpu, memory bool) error
|
|
|
|
// Mode returns in which mode the limiter is running in.
|
|
Mode() LimitMode
|
|
}
|
|
|
|
type limiter struct {
|
|
psutil psutil.Util
|
|
|
|
ncpu float64
|
|
ncpuFactor float64
|
|
proc psutil.Process
|
|
lock sync.RWMutex
|
|
cancel context.CancelFunc
|
|
onLimit LimitFunc
|
|
|
|
lastUsage Usage
|
|
lastUsageLock sync.RWMutex
|
|
|
|
cpu float64 // CPU limit
|
|
cpuCurrent float64 // Current CPU load of this process
|
|
cpuLast float64 // Last CPU load of this process
|
|
cpuMax float64 // Max. CPU load of this process
|
|
cpuTop float64 // Decaying max. CPU load of this process
|
|
cpuAvg float64 // Average CPU load of this process
|
|
cpuAvgCounter uint64 // Counter for average calculation
|
|
cpuLimitSince time.Time // Time when the CPU limit has been reached (hard limiter mode)
|
|
cpuLimitEnable bool // Whether CPU throttling is enabled (soft limiter mode)
|
|
cpuThrottling bool // Whether CPU throttling is currently active (soft limiter mode)
|
|
|
|
memory uint64 // Memory limit (bytes)
|
|
memoryCurrent uint64 // Current memory usage
|
|
memoryLast uint64 // Last memory usage
|
|
memoryMax uint64 // Max. memory usage
|
|
memoryTop uint64 // Decaying max. memory usage
|
|
memoryAvg float64 // Average memory usage
|
|
memoryAvgCounter uint64 // Counter for average memory calculation
|
|
memoryLimitSince time.Time // Time when the memory limit has been reached (hard limiter mode)
|
|
memoryLimitEnable bool // Whether memory limiting is enabled (soft limiter mode)
|
|
|
|
waitFor time.Duration
|
|
mode LimitMode
|
|
|
|
cancelLimit context.CancelFunc
|
|
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewLimiter returns a new Limiter
|
|
func NewLimiter(config LimiterConfig) Limiter {
|
|
l := &limiter{
|
|
cpu: config.CPU,
|
|
memory: config.Memory,
|
|
waitFor: config.WaitFor,
|
|
onLimit: config.OnLimit,
|
|
mode: config.Mode,
|
|
psutil: config.PSUtil,
|
|
logger: config.Logger,
|
|
}
|
|
|
|
if l.logger == nil {
|
|
l.logger = log.New("")
|
|
}
|
|
|
|
if l.psutil == nil {
|
|
l.psutil = psutil.DefaultUtil
|
|
}
|
|
|
|
if ncpu, err := l.psutil.CPUCounts(true); err != nil {
|
|
l.ncpu = 1
|
|
} else {
|
|
l.ncpu = ncpu
|
|
}
|
|
|
|
l.lastUsage.CPU.NCPU = l.ncpu
|
|
l.lastUsage.CPU.Limit = l.cpu * l.ncpu
|
|
l.lastUsage.Memory.Limit = l.memory
|
|
|
|
l.ncpuFactor = 1
|
|
|
|
mode := "hard"
|
|
if l.mode == LimitModeSoft {
|
|
mode = "soft"
|
|
l.cpu /= l.ncpu
|
|
l.ncpuFactor = l.ncpu
|
|
}
|
|
|
|
l.cpu /= 100
|
|
|
|
if l.onLimit == nil {
|
|
l.onLimit = func(float64, uint64) {}
|
|
}
|
|
|
|
l.logger = l.logger.WithFields(log.Fields{
|
|
"cpu": l.cpu * l.ncpuFactor,
|
|
"memory": l.memory,
|
|
"mode": mode,
|
|
})
|
|
|
|
return l
|
|
}
|
|
|
|
func (l *limiter) reset() {
|
|
l.cpuCurrent = 0
|
|
l.cpuLast = 0
|
|
l.cpuAvg = 0
|
|
l.cpuAvgCounter = 0
|
|
l.cpuMax = 0
|
|
l.cpuTop = 0
|
|
l.cpuLimitEnable = false
|
|
l.cpuThrottling = false
|
|
|
|
l.memoryCurrent = 0
|
|
l.memoryLast = 0
|
|
l.memoryAvg = 0
|
|
l.memoryAvgCounter = 0
|
|
l.memoryMax = 0
|
|
l.memoryTop = 0
|
|
l.memoryLimitEnable = false
|
|
}
|
|
|
|
func (l *limiter) Start(process psutil.Process) error {
|
|
l.lock.Lock()
|
|
defer l.lock.Unlock()
|
|
|
|
if l.proc != nil {
|
|
return fmt.Errorf("limiter is already running")
|
|
}
|
|
|
|
l.reset()
|
|
|
|
l.proc = process
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
l.cancel = cancel
|
|
|
|
go l.ticker(ctx, time.Second)
|
|
|
|
if l.mode == LimitModeSoft {
|
|
ctx, cancel = context.WithCancel(context.Background())
|
|
l.cancelLimit = cancel
|
|
|
|
go l.limitCPU(ctx, l.cpu, time.Second)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (l *limiter) Stop() {
|
|
l.lock.Lock()
|
|
defer l.lock.Unlock()
|
|
|
|
if l.proc == nil {
|
|
return
|
|
}
|
|
|
|
l.cancel()
|
|
|
|
if l.cancelLimit != nil {
|
|
l.cancelLimit()
|
|
l.cancelLimit = nil
|
|
}
|
|
|
|
l.proc.Stop()
|
|
l.proc = nil
|
|
|
|
l.reset()
|
|
}
|
|
|
|
func (l *limiter) ticker(ctx context.Context, interval time.Duration) {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case t := <-ticker.C:
|
|
l.collect(t)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *limiter) collect(_ time.Time) {
|
|
l.lock.Lock()
|
|
proc := l.proc
|
|
l.lock.Unlock()
|
|
|
|
if proc == nil {
|
|
return
|
|
}
|
|
|
|
mstat, merr := proc.VirtualMemory()
|
|
cpustat, cerr := proc.CPUPercent()
|
|
|
|
l.lock.Lock()
|
|
|
|
if merr == nil {
|
|
l.memoryLast, l.memoryCurrent = l.memoryCurrent, mstat
|
|
|
|
if l.memoryCurrent > l.memoryMax {
|
|
l.memoryMax = l.memoryCurrent
|
|
}
|
|
|
|
if l.memoryCurrent > l.memoryTop {
|
|
l.memoryTop = l.memoryCurrent
|
|
} else {
|
|
l.memoryTop = uint64(float64(l.memoryTop) * 0.95)
|
|
}
|
|
|
|
l.memoryAvgCounter++
|
|
|
|
l.memoryAvg = ((l.memoryAvg * float64(l.memoryAvgCounter-1)) + float64(l.memoryCurrent)) / float64(l.memoryAvgCounter)
|
|
}
|
|
|
|
if cerr == nil {
|
|
l.cpuLast, l.cpuCurrent = l.cpuCurrent, (cpustat.System+cpustat.User+cpustat.Other)/100
|
|
|
|
if l.cpuCurrent > l.cpuMax {
|
|
l.cpuMax = l.cpuCurrent
|
|
}
|
|
|
|
if l.cpuCurrent > l.cpuTop {
|
|
l.cpuTop = l.cpuCurrent
|
|
} else {
|
|
l.cpuTop = l.cpuTop * 0.95
|
|
}
|
|
|
|
l.cpuAvgCounter++
|
|
|
|
l.cpuAvg = ((l.cpuAvg * float64(l.cpuAvgCounter-1)) + l.cpuCurrent) / float64(l.cpuAvgCounter)
|
|
}
|
|
|
|
isLimitExceeded := false
|
|
|
|
if l.mode == LimitModeHard {
|
|
if l.cpu > 0 {
|
|
if l.cpuCurrent > l.cpu {
|
|
// Current value is higher than the limit
|
|
if l.cpuLast <= l.cpu {
|
|
// If the previous value is below the limit, then we reached the
|
|
// limit as of now
|
|
l.cpuLimitSince = time.Now()
|
|
}
|
|
|
|
if time.Since(l.cpuLimitSince) >= l.waitFor {
|
|
l.logger.Warn().Log("CPU limit exceeded")
|
|
isLimitExceeded = true
|
|
}
|
|
}
|
|
}
|
|
|
|
if l.memory > 0 {
|
|
if l.memoryCurrent > l.memory {
|
|
// Current value is higher than the limit
|
|
if l.memoryLast <= l.memory {
|
|
// If the previous value is below the limit, then we reached the
|
|
// limit as of now
|
|
l.memoryLimitSince = time.Now()
|
|
}
|
|
|
|
if time.Since(l.memoryLimitSince) >= l.waitFor {
|
|
l.logger.Warn().Log("Memory limit exceeded")
|
|
isLimitExceeded = true
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
if l.memory > 0 && l.memoryLimitEnable {
|
|
if l.memoryCurrent > l.memory {
|
|
// Current value is higher than the limit
|
|
l.logger.Warn().Log("Memory limit exceeded")
|
|
isLimitExceeded = true
|
|
}
|
|
}
|
|
}
|
|
|
|
l.logger.Debug().WithFields(log.Fields{
|
|
"cur_cpu": l.cpuCurrent * l.ncpuFactor,
|
|
"top_cpu": l.cpuTop * l.ncpuFactor,
|
|
"cur_mem": l.memoryCurrent,
|
|
"top_mem": l.memoryTop,
|
|
"exceeded": isLimitExceeded,
|
|
}).Log("Observation")
|
|
|
|
if isLimitExceeded {
|
|
go l.onLimit(l.cpuCurrent*l.ncpuFactor*100, l.memoryCurrent)
|
|
}
|
|
|
|
l.lastUsageLock.Lock()
|
|
l.lastUsage.CPU.Current = l.cpuCurrent * l.ncpu * 100
|
|
l.lastUsage.CPU.Average = l.cpuAvg * l.ncpu * 100
|
|
l.lastUsage.CPU.Max = l.cpuMax * l.ncpu * 100
|
|
l.lastUsage.CPU.IsThrottling = l.cpuThrottling
|
|
|
|
l.lastUsage.Memory.Current = l.memoryCurrent
|
|
l.lastUsage.Memory.Average = l.memoryAvg
|
|
l.lastUsage.Memory.Max = l.memoryMax
|
|
l.lastUsageLock.Unlock()
|
|
|
|
l.lock.Unlock()
|
|
}
|
|
|
|
func (l *limiter) Limit(cpu, memory bool) error {
|
|
l.lock.Lock()
|
|
defer l.lock.Unlock()
|
|
|
|
if l.mode == LimitModeHard {
|
|
return nil
|
|
}
|
|
|
|
if memory {
|
|
if !l.memoryLimitEnable {
|
|
l.memoryLimitEnable = true
|
|
|
|
l.logger.Debug().Log("Memory limiter enabled")
|
|
}
|
|
} else {
|
|
if l.memoryLimitEnable {
|
|
l.memoryLimitEnable = false
|
|
|
|
l.logger.Debug().Log("Memory limiter disabled")
|
|
}
|
|
}
|
|
|
|
if cpu {
|
|
if !l.cpuLimitEnable {
|
|
l.cpuLimitEnable = true
|
|
|
|
l.logger.Debug().Log("CPU limiter enabled")
|
|
}
|
|
} else {
|
|
if l.cpuLimitEnable {
|
|
l.cpuLimitEnable = false
|
|
|
|
l.logger.Debug().Log("CPU limiter disabled")
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// limitCPU will limit the CPU usage of this process. The limit is the max. CPU usage
|
|
// normed to 0-1. The interval defines how long a time slot is that will be splitted
|
|
// into sleeping and working.
|
|
// Inspired by https://github.com/opsengine/cpulimit
|
|
func (l *limiter) limitCPU(ctx context.Context, limit float64, interval time.Duration) {
|
|
defer func() {
|
|
l.lock.Lock()
|
|
if l.proc != nil {
|
|
l.proc.Resume()
|
|
}
|
|
l.cpuThrottling = false
|
|
l.lock.Unlock()
|
|
|
|
l.logger.Debug().Log("CPU throttler disabled")
|
|
}()
|
|
|
|
var workingrate float64 = -1
|
|
var factorTopLimit float64 = 0
|
|
var topLimit float64 = 0
|
|
|
|
l.logger.Debug().WithField("limit", limit*l.ncpu).Log("CPU throttler enabled")
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
|
|
l.lock.Lock()
|
|
|
|
if !l.cpuLimitEnable {
|
|
if factorTopLimit > 0 {
|
|
factorTopLimit -= 10
|
|
} else {
|
|
if l.cpuThrottling {
|
|
if l.proc != nil {
|
|
l.proc.Resume()
|
|
}
|
|
l.cpuThrottling = false
|
|
}
|
|
l.lock.Unlock()
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue
|
|
}
|
|
} else {
|
|
factorTopLimit = 100
|
|
topLimit = l.cpuTop - limit
|
|
l.cpuThrottling = true
|
|
}
|
|
|
|
lim := limit
|
|
|
|
if topLimit > 0 {
|
|
// After releasing the limiter, the process will not get the full CPU capacity back.
|
|
// Instead the limit will be gradually lifted by increments until it reaches the
|
|
// CPU top value. The CPU top value has to be larger than the actual limit.
|
|
lim += (100 - factorTopLimit) / 100 * topLimit
|
|
}
|
|
|
|
pcpu := l.cpuCurrent
|
|
|
|
l.lock.Unlock()
|
|
|
|
if workingrate < 0 {
|
|
workingrate = limit
|
|
}
|
|
// else {
|
|
// workingrate = math.Min(workingrate/pcpu*limit, 1)
|
|
//}
|
|
|
|
workingrate = lim
|
|
|
|
worktime := float64(interval.Nanoseconds()) * workingrate
|
|
sleeptime := float64(interval.Nanoseconds()) - worktime
|
|
|
|
l.logger.Debug().WithFields(log.Fields{
|
|
"limit": lim * l.ncpu,
|
|
"pcpu": pcpu,
|
|
"factor": factorTopLimit,
|
|
"worktime": (time.Duration(worktime) * time.Nanosecond).String(),
|
|
"sleeptime": (time.Duration(sleeptime) * time.Nanosecond).String(),
|
|
}).Log("Throttler")
|
|
|
|
l.lock.Lock()
|
|
if l.proc != nil {
|
|
l.proc.Resume()
|
|
}
|
|
l.lock.Unlock()
|
|
|
|
time.Sleep(time.Duration(worktime) * time.Nanosecond)
|
|
|
|
if sleeptime > 0 {
|
|
l.lock.Lock()
|
|
if l.proc != nil {
|
|
l.proc.Suspend()
|
|
}
|
|
l.lock.Unlock()
|
|
|
|
time.Sleep(time.Duration(sleeptime) * time.Nanosecond)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *limiter) Current() (cpu float64, memory uint64) {
|
|
l.lastUsageLock.RLock()
|
|
defer l.lastUsageLock.RUnlock()
|
|
|
|
cpu = l.lastUsage.CPU.Current / l.ncpu
|
|
memory = l.lastUsage.Memory.Current
|
|
|
|
return
|
|
}
|
|
|
|
func (l *limiter) Usage() Usage {
|
|
l.lastUsageLock.RLock()
|
|
defer l.lastUsageLock.RUnlock()
|
|
|
|
return l.lastUsage
|
|
}
|
|
|
|
func (l *limiter) Limits() (cpu float64, memory uint64) {
|
|
return l.cpu * 100, l.memory
|
|
}
|
|
|
|
func (l *limiter) Mode() LimitMode {
|
|
return l.mode
|
|
}
|