mirror of
https://github.com/kontera-technologies/go-supervisor
synced 2025-10-04 14:22:43 +08:00
@@ -110,6 +110,11 @@ type ProcessOptions struct {
|
|||||||
// EventTimeFormat is the time format used when events are marshaled to string.
|
// EventTimeFormat is the time format used when events are marshaled to string.
|
||||||
// Will use defaultEventTimeFormat when set to "".
|
// Will use defaultEventTimeFormat when set to "".
|
||||||
EventTimeFormat string
|
EventTimeFormat string
|
||||||
|
|
||||||
|
// RunTimeout is the duration that the process can run before we terminate the process.
|
||||||
|
// Set to <= 0, for an unlimited run timeout
|
||||||
|
// Will use defaultRunTimeout when set to 0.
|
||||||
|
RunTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// init initializes the opts structure with default and required options.
|
// init initializes the opts structure with default and required options.
|
||||||
@@ -154,12 +159,18 @@ func initProcessOptions(opts ProcessOptions) *ProcessOptions {
|
|||||||
if opts.IdleTimeout == 0 {
|
if opts.IdleTimeout == 0 {
|
||||||
opts.IdleTimeout = defaultIdleTimeout
|
opts.IdleTimeout = defaultIdleTimeout
|
||||||
}
|
}
|
||||||
|
if opts.IdleTimeout < 0 {
|
||||||
|
opts.IdleTimeout = time.Duration(maxDuration)
|
||||||
|
}
|
||||||
if opts.TerminationGraceTimeout == 0 {
|
if opts.TerminationGraceTimeout == 0 {
|
||||||
opts.TerminationGraceTimeout = defaultTerminationGraceTimeout
|
opts.TerminationGraceTimeout = defaultTerminationGraceTimeout
|
||||||
}
|
}
|
||||||
if opts.EventTimeFormat == "" {
|
if opts.EventTimeFormat == "" {
|
||||||
opts.EventTimeFormat = defaultEventTimeFormat
|
opts.EventTimeFormat = defaultEventTimeFormat
|
||||||
}
|
}
|
||||||
|
if opts.RunTimeout <= 0 {
|
||||||
|
opts.RunTimeout = defaultRunTimeout
|
||||||
|
}
|
||||||
if opts.In == nil {
|
if opts.In == nil {
|
||||||
opts.In = make(chan []byte)
|
opts.In = make(chan []byte)
|
||||||
}
|
}
|
||||||
|
@@ -15,6 +15,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const maxDuration = 1<<63 - 1
|
||||||
const (
|
const (
|
||||||
defaultMaxSpawns = 1
|
defaultMaxSpawns = 1
|
||||||
defaultMaxSpawnAttempts = 10
|
defaultMaxSpawnAttempts = 10
|
||||||
@@ -25,6 +26,7 @@ const (
|
|||||||
defaultNotifyEventTimeout = time.Millisecond
|
defaultNotifyEventTimeout = time.Millisecond
|
||||||
defaultParserBufferSize = 4096
|
defaultParserBufferSize = 4096
|
||||||
defaultIdleTimeout = 10 * time.Second
|
defaultIdleTimeout = 10 * time.Second
|
||||||
|
defaultRunTimeout = time.Duration(maxDuration)
|
||||||
defaultTerminationGraceTimeout = time.Second
|
defaultTerminationGraceTimeout = time.Second
|
||||||
defaultEventTimeFormat = time.RFC3339Nano
|
defaultEventTimeFormat = time.RFC3339Nano
|
||||||
)
|
)
|
||||||
@@ -208,7 +210,7 @@ func (p *Process) unprotectedStart() error {
|
|||||||
go readerToChan(p.opts.OutputParser(outPipe, p.opts.ParserBufferSize), p.opts.Out, isOutClosed, p.stopC, heartbeat)
|
go readerToChan(p.opts.OutputParser(outPipe, p.opts.ParserBufferSize), p.opts.Out, isOutClosed, p.stopC, heartbeat)
|
||||||
go readerToChan(p.opts.ErrorParser(errPipe, p.opts.ParserBufferSize), p.opts.Err, isErrClosed, p.stopC, nil)
|
go readerToChan(p.opts.ErrorParser(errPipe, p.opts.ParserBufferSize), p.opts.Err, isErrClosed, p.stopC, nil)
|
||||||
|
|
||||||
go monitorHeartBeat(p.opts.IdleTimeout, heartbeat, isMonitorClosed, p.stopC, p.Stop, p.notifyEvent)
|
go MonitorHeartBeat(p.opts.IdleTimeout, p.opts.RunTimeout, heartbeat, isMonitorClosed, p.stopC, p.Stop, p.notifyEvent)
|
||||||
|
|
||||||
var ensureOnce sync.Once
|
var ensureOnce sync.Once
|
||||||
p.ensureAllClosed = func() {
|
p.ensureAllClosed = func() {
|
||||||
@@ -218,7 +220,9 @@ func (p *Process) unprotectedStart() error {
|
|||||||
default:
|
default:
|
||||||
log.Printf("[%s] ensureAllClosed was called before stopC channel was closed.", p.opts.Id)
|
log.Printf("[%s] ensureAllClosed was called before stopC channel was closed.", p.opts.Id)
|
||||||
}
|
}
|
||||||
if p.opts.Debug { log.Printf("[%s] Starting to ensure all pipes have closed.", p.opts.Id) }
|
if p.opts.Debug {
|
||||||
|
log.Printf("[%s] Starting to ensure all pipes have closed.", p.opts.Id)
|
||||||
|
}
|
||||||
if cErr := ensureClosed("stdin", isInClosed, inPipe.Close); cErr != nil {
|
if cErr := ensureClosed("stdin", isInClosed, inPipe.Close); cErr != nil {
|
||||||
log.Printf("[%s] Possible memory leak, stdin go-routine not closed. Error: %s", p.opts.Id, cErr)
|
log.Printf("[%s] Possible memory leak, stdin go-routine not closed. Error: %s", p.opts.Id, cErr)
|
||||||
}
|
}
|
||||||
@@ -304,16 +308,17 @@ func readerToChan(producer ProduceFn, out chan<- *interface{}, closeWhenDone, st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// monitorHeartBeat monitors the heartbeat channel and stops the process if idleTimeout time is passed without a
|
// MonitorHeartBeat monitors the heartbeat channel and stops the process if idleTimeout time is passed without a
|
||||||
// positive heartbeat, or if a negative heartbeat is passed.
|
// positive heartbeat, or if a negative heartbeat is passed, or if the run timeout passed.
|
||||||
//
|
//
|
||||||
// isMonitorClosed will be closed when this function exists.
|
// isMonitorClosed will be closed when this function exists.
|
||||||
//
|
//
|
||||||
// When stopC closes, this function will exit immediately.
|
// When stopC closes, this function will exit immediately.
|
||||||
func monitorHeartBeat(idleTimeout time.Duration, heartbeat, isMonitorClosed, stopC chan bool, stop func() error, notifyEvent func(string, ...interface{})) {
|
func MonitorHeartBeat(idleTimeout time.Duration, runTimeout time.Duration, heartbeat, isMonitorClosed, stopC chan bool, stop func() error, notifyEvent func(string, ...interface{})) {
|
||||||
t := time.NewTimer(idleTimeout)
|
t := time.NewTimer(idleTimeout)
|
||||||
|
r := time.NewTimer(runTimeout)
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
|
defer r.Stop()
|
||||||
for alive := true; alive; {
|
for alive := true; alive; {
|
||||||
select {
|
select {
|
||||||
case <-stopC:
|
case <-stopC:
|
||||||
@@ -334,6 +339,9 @@ func monitorHeartBeat(idleTimeout time.Duration, heartbeat, isMonitorClosed, sto
|
|||||||
case <-t.C:
|
case <-t.C:
|
||||||
alive = false
|
alive = false
|
||||||
notifyEvent("MissingHeartbeat", "Stopping process.")
|
notifyEvent("MissingHeartbeat", "Stopping process.")
|
||||||
|
case <-r.C:
|
||||||
|
alive = false
|
||||||
|
notifyEvent("RunTimePassed", "Stopping process.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@@ -717,3 +717,21 @@ func test_timings(t *testing.T) {
|
|||||||
|
|
||||||
log.Println(prodInNum, prodOutNum, incOutNum)
|
log.Println(prodInNum, prodOutNum, incOutNum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMonitorRunTimeout(t *testing.T) {
|
||||||
|
heartbeat, isMonitorClosed, stopC := make(chan bool), make(chan bool), make(chan bool)
|
||||||
|
result := make(chan string)
|
||||||
|
resEvent := make(chan string)
|
||||||
|
|
||||||
|
stopF := func() error {
|
||||||
|
result <- "Stopped"
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
eventNotify := func(event string, message ...interface{}) {
|
||||||
|
resEvent <- event
|
||||||
|
}
|
||||||
|
go su.MonitorHeartBeat(20*time.Millisecond, 10*time.Millisecond, heartbeat, isMonitorClosed, stopC, stopF, eventNotify)
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
assertExpectedEqualsActual(t, <-resEvent, "RunTimePassed")
|
||||||
|
assertExpectedEqualsActual(t, <-result, "Stopped")
|
||||||
|
}
|
||||||
|
Reference in New Issue
Block a user