feat(worker): make maximum consecutive failures configurable (#1692)

backoff.go

@@ -33,7 +33,7 @@ func (e *exponentialBackoff) recordFailure() bool {
 	e.backoff = min(e.backoff*2, e.maxBackoff)
 
 	e.mu.Unlock()
-	return e.failureCount >= e.maxConsecutiveFailures
+	return e.maxConsecutiveFailures != -1 && e.failureCount >= e.maxConsecutiveFailures
 }
 
 // wait sleeps for the backoff duration if failureCount is non-zero.

caddy/app.go

@@ -114,7 +114,13 @@ func (f *FrankenPHPApp) Start() error {
 		frankenphp.WithMaxWaitTime(f.MaxWaitTime),
 	}
 	for _, w := range append(f.Workers) {
-		opts = append(opts, frankenphp.WithWorkers(w.Name, repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env, w.Watch))
+		workerOpts := []frankenphp.WorkerOption{
+			frankenphp.WithWorkerEnv(w.Env),
+			frankenphp.WithWorkerWatchMode(w.Watch),
+			frankenphp.WithWorkerMaxFailures(w.MaxConsecutiveFailures),
+		}
+
+		opts = append(opts, frankenphp.WithWorkers(w.Name, repl.ReplaceKnown(w.FileName, ""), w.Num, workerOpts...))
 	}
 
 	frankenphp.Shutdown()

caddy/workerconfig.go

@@ -29,6 +29,8 @@ type workerConfig struct {
 	Env map[string]string `json:"env,omitempty"`
 	// Directories to watch for file changes
 	Watch []string `json:"watch,omitempty"`
+	// MaxConsecutiveFailures sets the maximum number of consecutive failures before panicking (defaults to 6, set to -1 to never panick)
+	MaxConsecutiveFailures int `json:"max_consecutive_failures,omitempty"`
 }
 
 func parseWorkerConfig(d *caddyfile.Dispenser) (workerConfig, error) {
@@ -94,8 +96,22 @@ func parseWorkerConfig(d *caddyfile.Dispenser) (workerConfig, error) {
 			} else {
 				wc.Watch = append(wc.Watch, d.Val())
 			}
+		case "max_consecutive_failures":
+			if !d.NextArg() {
+				return wc, d.ArgErr()
+			}
+
+			v, err := strconv.Atoi(d.Val())
+			if err != nil {
+				return wc, err
+			}
+			if v < -1 {
+				return wc, errors.New("max_consecutive_failures must be >= -1")
+			}
+
+			wc.MaxConsecutiveFailures = int(v)
 		default:
-			allowedDirectives := "name, file, num, env, watch"
+			allowedDirectives := "name, file, num, env, watch, max_consecutive_failures"
 			return wc, wrongSubDirectiveError("worker", allowedDirectives, v)
 		}
 	}

docs/config.md

@@ -71,6 +71,7 @@ The `frankenphp` [global option](https://caddyserver.com/docs/caddyfile/concepts
 			env <key> <value> # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables.
 			watch <path> # Sets the path to watch for file changes. Can be specified more than once for multiple paths.
 			name <name> # Sets the name of the worker, used in logs and metrics. Default: absolute path of worker file
+			max_consecutive_failures <num> # Sets the maximum number of consecutive failures before the worker is considered unhealthy, -1 means the worker will always restart. Default: 6.
 		}
 	}
 }

docs/fr/config.md

@@ -70,6 +70,7 @@ L'[option globale](https://caddyserver.com/docs/caddyfile/concepts#global-option
             env <key> <value> # Définit une variable d'environnement supplémentaire avec la valeur donnée. Peut être spécifié plusieurs fois pour régler plusieurs variables d'environnement.
             watch <path> # Définit le chemin d'accès à surveiller pour les modifications de fichiers. Peut être spécifié plusieurs fois pour plusieurs chemins.
             name <name> # Définit le nom du worker, utilisé dans les journaux et les métriques. Défaut : chemin absolu du fichier du worker
+            max_consecutive_failures <num> # Définit le nombre maximum d'échecs consécutifs avant que le worker ne soit considéré comme défaillant, -1 signifie que le worker redémarre toujours. Par défaut : 6.
         }
     }
 }

docs/fr/worker.md

@@ -150,6 +150,17 @@ Si le script worker reste en place plus longtemps que le dernier backoff \* 2, F
 Toutefois, si le script de worker continue d'échouer avec un code de sortie non nul dans un court laps de temps
 (par exemple, une faute de frappe dans un script), FrankenPHP plantera avec l'erreur : `too many consecutive failures` (trop d'échecs consécutifs).
 
+Le nombre d'échecs consécutifs peut être configuré dans votre [Caddyfile](config.md#configuration-du-caddyfile) avec l'option `max_consecutive_failures` :
+
+```caddyfile
+frankenphp {
+    worker {
+        # ...
+        max_consecutive_failures 10
+    }
+}
+```
+
 ## Comportement des superglobales
 
 [Les superglobales PHP](https://www.php.net/manual/fr/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...)

docs/worker.md

@@ -146,6 +146,19 @@ it will not penalize the worker script and restart it again.
 However, if the worker script continues to fail with a non-zero exit code in a short period of time
 (for example, having a typo in a script), FrankenPHP will crash with the error: `too many consecutive failures`.
 
+The number of consecutive failures can be configured in your [Caddyfile](config.md#caddyfile-config) with the `max_consecutive_failures` option:
+
+```caddyfile
+frankenphp {
+    worker {
+        # ...
+        max_consecutive_failures 10
+    }
+}
+```
+
+```caddyfile
+
 ## Superglobals Behavior
 
 [PHP superglobals](https://www.php.net/manual/en/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...)

frankenphp_test.go

@@ -66,7 +66,11 @@ func runTest(t *testing.T, test func(func(http.ResponseWriter, *http.Request), *
 
 	initOpts := []frankenphp.Option{frankenphp.WithLogger(opts.logger)}
 	if opts.workerScript != "" {
-		initOpts = append(initOpts, frankenphp.WithWorkers("workerName", testDataDir+opts.workerScript, opts.nbWorkers, opts.env, opts.watch))
+		workerOpts := []frankenphp.WorkerOption{
+			frankenphp.WithWorkerEnv(opts.env),
+			frankenphp.WithWorkerWatchMode(opts.watch),
+		}
+		initOpts = append(initOpts, frankenphp.WithWorkers("workerName", testDataDir+opts.workerScript, opts.nbWorkers, workerOpts...))
 	}
 	initOpts = append(initOpts, opts.initOpts...)
 	if opts.phpIni != nil {

options.go

@@ -1,13 +1,20 @@
 package frankenphp
 
 import (
+	"fmt"
 	"log/slog"
 	"time"
 )
 
+// defaultMaxConsecutiveFailures is the default maximum number of consecutive failures before panicking
+const defaultMaxConsecutiveFailures = 6
+
 // Option instances allow to configure FrankenPHP.
 type Option func(h *opt) error
 
+// WorkerOption instances allow configuring FrankenPHP worker.
+type WorkerOption func(*workerOpt) error
+
 // opt contains the available options.
 //
 // If you change this, also update the Caddy module and the documentation.
@@ -22,11 +29,12 @@ type opt struct {
 }
 
 type workerOpt struct {
-	name     string
-	fileName string
-	num      int
-	env      PreparedEnv
-	watch    []string
+	name                   string
+	fileName               string
+	num                    int
+	env                    PreparedEnv
+	watch                  []string
+	maxConsecutiveFailures int
 }
 
 // WithNumThreads configures the number of PHP threads to start.
@@ -55,9 +63,54 @@ func WithMetrics(m Metrics) Option {
 }
 
 // WithWorkers configures the PHP workers to start
-func WithWorkers(name string, fileName string, num int, env map[string]string, watch []string) Option {
+func WithWorkers(name string, fileName string, num int, options ...WorkerOption) Option {
 	return func(o *opt) error {
-		o.workers = append(o.workers, workerOpt{name, fileName, num, PrepareEnv(env), watch})
+		worker := workerOpt{
+			name:                   name,
+			fileName:               fileName,
+			num:                    num,
+			env:                    PrepareEnv(nil),
+			watch:                  []string{},
+			maxConsecutiveFailures: defaultMaxConsecutiveFailures,
+		}
+
+		for _, option := range options {
+			if err := option(&worker); err != nil {
+				return err
+			}
+		}
+
+		o.workers = append(o.workers, worker)
+
+		return nil
+	}
+}
+
+// WithWorkerEnv sets environment variables for the worker
+func WithWorkerEnv(env map[string]string) WorkerOption {
+	return func(w *workerOpt) error {
+		w.env = PrepareEnv(env)
+
+		return nil
+	}
+}
+
+// WithWorkerWatchMode sets directories to watch for file changes
+func WithWorkerWatchMode(watch []string) WorkerOption {
+	return func(w *workerOpt) error {
+		w.watch = watch
+
+		return nil
+	}
+}
+
+// WithWorkerMaxFailures sets the maximum number of consecutive failures before panicking
+func WithWorkerMaxFailures(maxFailures int) WorkerOption {
+	return func(w *workerOpt) error {
+		if maxFailures < -1 {
+			return fmt.Errorf("max consecutive failures must be >= -1, got %d", maxFailures)
+		}
+		w.maxConsecutiveFailures = maxFailures
 
 		return nil
 	}

phpmainthread_test.go

@@ -96,8 +96,16 @@ func TestTransitionThreadsWhileDoingRequests(t *testing.T) {
 
 	assert.NoError(t, Init(
 		WithNumThreads(numThreads),
-		WithWorkers(worker1Name, worker1Path, 1, map[string]string{"ENV1": "foo"}, []string{}),
-		WithWorkers(worker2Name, worker2Path, 1, map[string]string{"ENV1": "foo"}, []string{}),
+		WithWorkers(worker1Name, worker1Path, 1,
+			WithWorkerEnv(map[string]string{"ENV1": "foo"}),
+			WithWorkerWatchMode([]string{}),
+			WithWorkerMaxFailures(0),
+		),
+		WithWorkers(worker2Name, worker2Path, 1,
+			WithWorkerEnv(map[string]string{"ENV1": "foo"}),
+			WithWorkerWatchMode([]string{}),
+			WithWorkerMaxFailures(0),
+		),
 		WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))),
 	))
 
@@ -182,8 +190,8 @@ func TestFinishBootingAWorkerScript(t *testing.T) {
 
 func TestReturnAnErrorIf2WorkersHaveTheSameFileName(t *testing.T) {
 	workers = make(map[string]*worker)
-	_, err1 := newWorker(workerOpt{fileName: "filename.php"})
-	_, err2 := newWorker(workerOpt{fileName: "filename.php"})
+	_, err1 := newWorker(workerOpt{fileName: "filename.php", maxConsecutiveFailures: defaultMaxConsecutiveFailures})
+	_, err2 := newWorker(workerOpt{fileName: "filename.php", maxConsecutiveFailures: defaultMaxConsecutiveFailures})
 
 	assert.NoError(t, err1)
 	assert.Error(t, err2, "two workers cannot have the same filename")
@@ -191,8 +199,8 @@ func TestReturnAnErrorIf2WorkersHaveTheSameFileName(t *testing.T) {
 
 func TestReturnAnErrorIf2ModuleWorkersHaveTheSameName(t *testing.T) {
 	workers = make(map[string]*worker)
-	_, err1 := newWorker(workerOpt{fileName: "filename.php", name: "workername"})
-	_, err2 := newWorker(workerOpt{fileName: "filename2.php", name: "workername"})
+	_, err1 := newWorker(workerOpt{fileName: "filename.php", name: "workername", maxConsecutiveFailures: defaultMaxConsecutiveFailures})
+	_, err2 := newWorker(workerOpt{fileName: "filename2.php", name: "workername", maxConsecutiveFailures: defaultMaxConsecutiveFailures})
 
 	assert.NoError(t, err1)
 	assert.Error(t, err2, "two workers cannot have the same name")
@@ -203,8 +211,9 @@ func getDummyWorker(fileName string) *worker {
 		workers = make(map[string]*worker)
 	}
 	worker, _ := newWorker(workerOpt{
-		fileName: testDataPath + "/" + fileName,
-		num:      1,
+		fileName:               testDataPath + "/" + fileName,
+		num:                    1,
+		maxConsecutiveFailures: defaultMaxConsecutiveFailures,
 	})
 	return worker
 }

scaling_test.go

@@ -37,7 +37,11 @@ func TestScaleAWorkerThreadUpAndDown(t *testing.T) {
 	assert.NoError(t, Init(
 		WithNumThreads(2),
 		WithMaxThreads(3),
-		WithWorkers(workerName, workerPath, 1, map[string]string{}, []string{}),
+		WithWorkers(workerName, workerPath, 1,
+			WithWorkerEnv(map[string]string{}),
+			WithWorkerWatchMode([]string{}),
+			WithWorkerMaxFailures(0),
+		),
 		WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))),
 	))
 

threadworker.go

@@ -30,7 +30,7 @@ func convertToWorkerThread(thread *phpThread, worker *worker) {
 		backoff: &exponentialBackoff{
 			maxBackoff:             1 * time.Second,
 			minBackoff:             100 * time.Millisecond,
-			maxConsecutiveFailures: 6,
+			maxConsecutiveFailures: worker.maxConsecutiveFailures,
 		},
 	})
 	worker.attachThread(thread)
@@ -116,7 +116,6 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) {
 
 	// on exit status 0 we just run the worker script again
 	if exitStatus == 0 && !handler.isBootingScript {
-		// TODO: make the max restart configurable
 		metrics.StopWorker(worker.name, StopReasonRestart)
 		handler.backoff.recordSuccess()
 		logger.LogAttrs(ctx, slog.LevelDebug, "restarting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus))

worker.go

@@ -14,13 +14,14 @@ import (
 
 // represents a worker script and can have many threads assigned to it
 type worker struct {
-	name        string
-	fileName    string
-	num         int
-	env         PreparedEnv
-	requestChan chan *frankenPHPContext
-	threads     []*phpThread
-	threadMutex sync.RWMutex
+	name                   string
+	fileName               string
+	num                    int
+	env                    PreparedEnv
+	requestChan            chan *frankenPHPContext
+	threads                []*phpThread
+	threadMutex            sync.RWMutex
+	maxConsecutiveFailures int
 }
 
 var (
@@ -99,12 +100,13 @@ func newWorker(o workerOpt) (*worker, error) {
 
 	o.env["FRANKENPHP_WORKER\x00"] = "1"
 	w := &worker{
-		name:        o.name,
-		fileName:    absFileName,
-		num:         o.num,
-		env:         o.env,
-		requestChan: make(chan *frankenPHPContext),
-		threads:     make([]*phpThread, 0, o.num),
+		name:                   o.name,
+		fileName:               absFileName,
+		num:                    o.num,
+		env:                    o.env,
+		requestChan:            make(chan *frankenPHPContext),
+		threads:                make([]*phpThread, 0, o.num),
+		maxConsecutiveFailures: o.maxConsecutiveFailures,
 	}
 	workers[key] = w
 

worker_test.go

@@ -121,8 +121,16 @@ func TestWorkerGetOpt(t *testing.T) {
 
 func ExampleServeHTTP_workers() {
 	if err := frankenphp.Init(
-		frankenphp.WithWorkers("worker1", "worker1.php", 4, map[string]string{"ENV1": "foo"}, []string{}),
-		frankenphp.WithWorkers("worker2", "worker2.php", 2, map[string]string{"ENV2": "bar"}, []string{}),
+		frankenphp.WithWorkers("worker1", "worker1.php", 4,
+			frankenphp.WithWorkerEnv(map[string]string{"ENV1": "foo"}),
+			frankenphp.WithWorkerWatchMode([]string{}),
+			frankenphp.WithWorkerMaxFailures(0),
+		),
+		frankenphp.WithWorkers("worker2", "worker2.php", 2,
+			frankenphp.WithWorkerEnv(map[string]string{"ENV2": "bar"}),
+			frankenphp.WithWorkerWatchMode([]string{}),
+			frankenphp.WithWorkerMaxFailures(0),
+		),
 	); err != nil {
 		panic(err)
 	}