feat(worker): make maximum consecutive failures configurable

Author: alexandre-daubois

caddy/app.go

@@ -114,7 +114,11 @@ func (f *FrankenPHPApp) Start() error {
 		frankenphp.WithMaxWaitTime(f.MaxWaitTime),
 	}
 	for _, w := range append(f.Workers) {
-		opts = append(opts, frankenphp.WithWorkers(w.Name, repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env, w.Watch))
+		maxFailures := w.MaxConsecutiveFailures
+		if maxFailures <= 0 {
+			maxFailures = frankenphp.DefaultMaxConsecutiveFailures
+		}
+		opts = append(opts, frankenphp.WithWorkers(w.Name, repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env, w.Watch, maxFailures))
 	}
 
 	frankenphp.Shutdown()

caddy/workerconfig.go

@@ -29,6 +29,8 @@ type workerConfig struct {
 	Env map[string]string `json:"env,omitempty"`
 	// Directories to watch for file changes
 	Watch []string `json:"watch,omitempty"`
+	// MaxConsecutiveFailures sets the maximum number of consecutive failures before panicking (defaults to 6)
+	MaxConsecutiveFailures int `json:"max_consecutive_failures,omitempty"`
 }
 
 func parseWorkerConfig(d *caddyfile.Dispenser) (workerConfig, error) {
@@ -94,8 +96,19 @@ func parseWorkerConfig(d *caddyfile.Dispenser) (workerConfig, error) {
 			} else {
 				wc.Watch = append(wc.Watch, d.Val())
 			}
+		case "max_consecutive_failures":
+			if !d.NextArg() {
+				return wc, d.ArgErr()
+			}
+
+			v, err := strconv.ParseUint(d.Val(), 10, 32)
+			if err != nil {
+				return wc, err
+			}
+
+			wc.MaxConsecutiveFailures = int(v)
 		default:
-			allowedDirectives := "name, file, num, env, watch"
+			allowedDirectives := "name, file, num, env, watch, max_consecutive_failures"
 			return wc, wrongSubDirectiveError("worker", allowedDirectives, v)
 		}
 	}

docs/config.md

@@ -71,6 +71,7 @@ The `frankenphp` [global option](https://caddyserver.com/docs/caddyfile/concepts
 			env <key> <value> # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables.
 			watch <path> # Sets the path to watch for file changes. Can be specified more than once for multiple paths.
 			name <name> # Sets the name of the worker, used in logs and metrics. Default: absolute path of worker file
+			max_consecutive_failures <num> # Sets the maximum number of consecutive failures before the worker is considered unhealthy. Default: 6.
 		}
 	}
 }

docs/fr/worker.md

@@ -150,6 +150,17 @@ Si le script worker reste en place plus longtemps que le dernier backoff \* 2, F
 Toutefois, si le script de worker continue d'échouer avec un code de sortie non nul dans un court laps de temps
 (par exemple, une faute de frappe dans un script), FrankenPHP plantera avec l'erreur : `too many consecutive failures` (trop d'échecs consécutifs).
 
+Le nombre d'échecs consécutifs peut être configuré dans votre [Caddyfile](config.md#configuration-du-caddyfile) avec l'option `max_consecutive_failures` :
+
+```caddyfile
+frankenphp {
+    worker {
+        # ...
+        max_consecutive_failures 10
+    }
+}
+```
+
 ## Comportement des superglobales
 
 [Les superglobales PHP](https://www.php.net/manual/fr/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...)

docs/worker.md

@@ -146,6 +146,19 @@ it will not penalize the worker script and restart it again.
 However, if the worker script continues to fail with a non-zero exit code in a short period of time
 (for example, having a typo in a script), FrankenPHP will crash with the error: `too many consecutive failures`.
 
+The number of consecutive failures can be configured in your [Caddyfile](config.md#caddyfile-config) with the `max_consecutive_failures` option:
+
+```caddyfile
+frankenphp {
+    worker {
+        # ...
+        max_consecutive_failures 10
+    }
+}
+```
+
+```caddyfile
+
 ## Superglobals Behavior
 
 [PHP superglobals](https://www.php.net/manual/en/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...)

options.go

@@ -5,6 +5,9 @@ import (
 	"time"
 )
 
+// DefaultMaxConsecutiveFailures is the default maximum number of consecutive failures before panicking
+const DefaultMaxConsecutiveFailures = 6
+
 // Option instances allow to configure FrankenPHP.
 type Option func(h *opt) error
 
@@ -22,11 +25,12 @@ type opt struct {
 }
 
 type workerOpt struct {
-	name     string
-	fileName string
-	num      int
-	env      PreparedEnv
-	watch    []string
+	name                   string
+	fileName               string
+	num                    int
+	env                    PreparedEnv
+	watch                  []string
+	maxConsecutiveFailures int
 }
 
 // WithNumThreads configures the number of PHP threads to start.
@@ -55,9 +59,12 @@ func WithMetrics(m Metrics) Option {
 }
 
 // WithWorkers configures the PHP workers to start
-func WithWorkers(name string, fileName string, num int, env map[string]string, watch []string) Option {
+func WithWorkers(name string, fileName string, num int, env map[string]string, watch []string, maxConsecutiveFailures int) Option {
 	return func(o *opt) error {
-		o.workers = append(o.workers, workerOpt{name, fileName, num, PrepareEnv(env), watch})
+		if maxConsecutiveFailures <= 0 {
+			maxConsecutiveFailures = DefaultMaxConsecutiveFailures
+		}
+		o.workers = append(o.workers, workerOpt{name, fileName, num, PrepareEnv(env), watch, maxConsecutiveFailures})
 
 		return nil
 	}

phpmainthread_test.go

@@ -96,8 +96,8 @@ func TestTransitionThreadsWhileDoingRequests(t *testing.T) {
 
 	assert.NoError(t, Init(
 		WithNumThreads(numThreads),
-		WithWorkers(worker1Name, worker1Path, 1, map[string]string{"ENV1": "foo"}, []string{}),
-		WithWorkers(worker2Name, worker2Path, 1, map[string]string{"ENV1": "foo"}, []string{}),
+		WithWorkers(worker1Name, worker1Path, 1, map[string]string{"ENV1": "foo"}, []string{}, 0),
+		WithWorkers(worker2Name, worker2Path, 1, map[string]string{"ENV1": "foo"}, []string{}, 0),
 		WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))),
 	))
 
@@ -182,17 +182,17 @@ func TestFinishBootingAWorkerScript(t *testing.T) {
 
 func TestReturnAnErrorIf2WorkersHaveTheSameFileName(t *testing.T) {
 	workers = make(map[string]*worker)
-	_, err1 := newWorker(workerOpt{fileName: "filename.php"})
-	_, err2 := newWorker(workerOpt{fileName: "filename.php"})
+	_, err1 := newWorker(workerOpt{fileName: "filename.php", maxConsecutiveFailures: 0})
+	_, err2 := newWorker(workerOpt{fileName: "filename.php", maxConsecutiveFailures: 0})
 
 	assert.NoError(t, err1)
 	assert.Error(t, err2, "two workers cannot have the same filename")
 }
 
 func TestReturnAnErrorIf2ModuleWorkersHaveTheSameName(t *testing.T) {
 	workers = make(map[string]*worker)
-	_, err1 := newWorker(workerOpt{fileName: "filename.php", name: "workername"})
-	_, err2 := newWorker(workerOpt{fileName: "filename2.php", name: "workername"})
+	_, err1 := newWorker(workerOpt{fileName: "filename.php", name: "workername", maxConsecutiveFailures: 0})
+	_, err2 := newWorker(workerOpt{fileName: "filename2.php", name: "workername", maxConsecutiveFailures: 0})
 
 	assert.NoError(t, err1)
 	assert.Error(t, err2, "two workers cannot have the same name")
@@ -203,8 +203,9 @@ func getDummyWorker(fileName string) *worker {
 		workers = make(map[string]*worker)
 	}
 	worker, _ := newWorker(workerOpt{
-		fileName: testDataPath + "/" + fileName,
-		num:      1,
+		fileName:               testDataPath + "/" + fileName,
+		num:                    1,
+		maxConsecutiveFailures: 0,
 	})
 	return worker
 }

scaling_test.go

@@ -37,7 +37,7 @@ func TestScaleAWorkerThreadUpAndDown(t *testing.T) {
 	assert.NoError(t, Init(
 		WithNumThreads(2),
 		WithMaxThreads(3),
-		WithWorkers(workerName, workerPath, 1, map[string]string{}, []string{}),
+		WithWorkers(workerName, workerPath, 1, map[string]string{}, []string{}, 0),
 		WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))),
 	))
 

threadworker.go

@@ -30,7 +30,7 @@ func convertToWorkerThread(thread *phpThread, worker *worker) {
 		backoff: &exponentialBackoff{
 			maxBackoff:             1 * time.Second,
 			minBackoff:             100 * time.Millisecond,
-			maxConsecutiveFailures: 6,
+			maxConsecutiveFailures: worker.maxConsecutiveFailures,
 		},
 	})
 	worker.attachThread(thread)
@@ -116,7 +116,6 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) {
 
 	// on exit status 0 we just run the worker script again
 	if exitStatus == 0 && !handler.isBootingScript {
-		// TODO: make the max restart configurable
 		metrics.StopWorker(worker.name, StopReasonRestart)
 		handler.backoff.recordSuccess()
 		logger.LogAttrs(ctx, slog.LevelDebug, "restarting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus))

worker.go

@@ -14,13 +14,14 @@ import (
 
 // represents a worker script and can have many threads assigned to it
 type worker struct {
-	name        string
-	fileName    string
-	num         int
-	env         PreparedEnv
-	requestChan chan *frankenPHPContext
-	threads     []*phpThread
-	threadMutex sync.RWMutex
+	name                   string
+	fileName               string
+	num                    int
+	env                    PreparedEnv
+	requestChan            chan *frankenPHPContext
+	threads                []*phpThread
+	threadMutex            sync.RWMutex
+	maxConsecutiveFailures int
 }
 
 var (
@@ -99,12 +100,13 @@ func newWorker(o workerOpt) (*worker, error) {
 
 	o.env["FRANKENPHP_WORKER\x00"] = "1"
 	w := &worker{
-		name:        o.name,
-		fileName:    absFileName,
-		num:         o.num,
-		env:         o.env,
-		requestChan: make(chan *frankenPHPContext),
-		threads:     make([]*phpThread, 0, o.num),
+		name:                   o.name,
+		fileName:               absFileName,
+		num:                    o.num,
+		env:                    o.env,
+		requestChan:            make(chan *frankenPHPContext),
+		threads:                make([]*phpThread, 0, o.num),
+		maxConsecutiveFailures: o.maxConsecutiveFailures,
 	}
 	workers[key] = w