refactor: improve ExtensionWorkers API (#1952)

* refactor: improve ExtensionWorkers API

* Update workerextension.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update workerextension.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update caddy/app.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* review

* fix tests

* docs

* errors

* improved error handling

* fix race

* add missing return

* use %q in Errorf

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

threadregular.go

@@ -84,14 +84,15 @@ func (handler *regularThread) afterRequest() {
 	handler.requestContext = nil
 }
 
-func handleRequestWithRegularPHPThreads(fc *frankenPHPContext) {
+func handleRequestWithRegularPHPThreads(fc *frankenPHPContext) error {
 	metrics.StartRequest()
 	select {
 	case regularRequestChan <- fc:
 		// a thread was available to handle the request immediately
 		<-fc.done
 		metrics.StopRequest()
-		return
+
+		return nil
 	default:
 		// no thread was available
 	}
@@ -104,14 +105,17 @@ func handleRequestWithRegularPHPThreads(fc *frankenPHPContext) {
 			metrics.DequeuedRequest()
 			<-fc.done
 			metrics.StopRequest()
-			return
+
+			return nil
 		case scaleChan <- fc:
 			// the request has triggered scaling, continue to wait for a thread
 		case <-timeoutChan(maxWaitTime):
 			// the request has timed out stalling
 			metrics.DequeuedRequest()
-			fc.reject(504, "Gateway Timeout")
-			return
+
+			fc.reject(ErrMaxWaitTimeExceeded)
+
+			return ErrMaxWaitTimeExceeded
 		}
 	}
 }