From 483eba3699e7bac4832f89f4e04a5159fd276eb9 Mon Sep 17 00:00:00 2001 From: Nicolas Grekas Date: Mon, 16 Nov 2026 12:33:59 +0100 Subject: [PATCH] fix: restart worker scripts when opcache schedules a restart A worker uses opcache's shared memory for as long as its script runs. A restart of that memory reuses memory the worker still references. Once opcache waits for the threads of the process (php/php-src#24276), the restart never runs while workers are alive. --- docs/metrics.md | 2 +- frankenphp.c | 47 +++++++++++ frankenphp.go | 18 +++-- frankenphp.h | 5 ++ metrics.go | 2 +- metrics_test.go | 4 +- opcache_test.go | 78 ++++++++++++++++++ opcacherestart.go | 157 ++++++++++++++++++++++++++++++++++++ opcacherestart_test.go | 41 ++++++++++ testdata/opcache/worker.php | 45 +++++++++++ threadworker.go | 43 +++++++--- 11 files changed, 419 insertions(+), 23 deletions(-) create mode 100644 opcache_test.go create mode 100644 opcacherestart.go create mode 100644 opcacherestart_test.go create mode 100644 testdata/opcache/worker.php diff --git a/docs/metrics.md b/docs/metrics.md index b87d2231f4..81f1594891 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -27,7 +27,7 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. -opcache schedules a restart when its cache is full and more than `opcache.max_wasted_percentage` of it was wasted by invalidations. The restart runs at the next request start; caching is off until then, and running PHP threads may still reference the old memory, which can crash the process. The counter should stay at zero: raise `opcache.memory_consumption`, `opcache.max_accelerated_files` or `opcache.max_wasted_percentage`. `manual` should never appear, FrankenPHP overrides `opcache_reset()`. Each restart is also logged. This metric will be removed once opcache handles restarts safely under ZTS. +opcache schedules a restart when its cache is full and more than `opcache.max_wasted_percentage` of it was wasted by invalidations. The restart runs at the next request start; caching is off until then, and running PHP threads may still reference the old memory, which can crash the process. A worker uses this memory for as long as its script runs, so FrankenPHP restarts the worker scripts when a restart is scheduled, at most once a minute. The counter should stay at zero: raise `opcache.memory_consumption`, `opcache.max_accelerated_files` or `opcache.max_wasted_percentage`. `manual` should never appear, FrankenPHP overrides `opcache_reset()`. Each restart is also logged. ## Threads State Endpoint diff --git a/frankenphp.c b/frankenphp.c index ca7ec27b14..84758896d0 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -1176,6 +1176,53 @@ static void frankenphp_opcache_restart_hook(int reason) { } #endif +/* Tells, through opcache_get_status(), whether opcache caches the scripts of + * the current request (FRANKENPHP_OPCACHE_CACHED) and whether a restart is + * pending or in progress (FRANKENPHP_OPCACHE_RESTART_PENDING), or -1 when + * opcache cannot tell */ +int frankenphp_opcache_state(void) { + zend_function *func = zend_hash_str_find_ptr(CG(function_table), + ZEND_STRL("opcache_get_status")); + if (func == NULL) { + return -1; + } + + zval status, fetch_scripts; + ZVAL_FALSE(&fetch_scripts); + + /* opcache.restrict_api makes opcache_get_status() raise a warning */ + int error_reporting = EG(error_reporting); + EG(error_reporting) = 0; + zend_call_known_function(func, NULL, NULL, &status, 1, &fetch_scripts, NULL); + EG(error_reporting) = error_reporting; + + if (Z_TYPE(status) != IS_ARRAY) { + zval_ptr_dtor(&status); + + return -1; + } + + int state = 0; + zval *value = + zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("opcache_enabled")); + if (value != NULL && zend_is_true(value)) { + state |= FRANKENPHP_OPCACHE_CACHED; + } + value = zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("restart_pending")); + if (value != NULL && zend_is_true(value)) { + state |= FRANKENPHP_OPCACHE_RESTART_PENDING; + } + value = + zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("restart_in_progress")); + if (value != NULL && zend_is_true(value)) { + state |= FRANKENPHP_OPCACHE_RESTART_PENDING; + } + + zval_ptr_dtor(&status); + + return state; +} + /* {{{ thread-safe opcache reset */ PHP_FUNCTION(frankenphp_opcache_reset) { go_schedule_opcache_reset(frankenphp_thread_index()); diff --git a/frankenphp.go b/frankenphp.go index f18ed6c999..063d0774eb 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -929,16 +929,17 @@ func opcacheRestartScheduled(reason int) { m.OpcacheRestart(reasonText) } - if !globalLogger.Enabled(globalCtx, slog.LevelWarn) { - return + if globalLogger.Enabled(globalCtx, slog.LevelWarn) { + // written synchronously, under opcache's lock: a line deferred to a + // goroutine is lost if the restart crashes the process + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, + "opcache restart scheduled, caching stops until the next request start carries it out while other threads may still reference the old memory: raise opcache.memory_consumption, opcache.max_accelerated_files or opcache.max_wasted_percentage", + slog.String("reason", reasonText), + ) } - // written synchronously, under opcache's lock: a line deferred to a - // goroutine is lost if the restart crashes the process - globalLogger.LogAttrs(globalCtx, slog.LevelWarn, - "opcache restart scheduled, caching stops until the next request start carries it out while other threads may still reference the old memory: raise opcache.memory_consumption, opcache.max_accelerated_files or opcache.max_wasted_percentage", - slog.String("reason", reasonText), - ) + // last, as opcache marks the restart as pending only once this returns + restartWorkersForOpcache() } func convertArgs(args []string) (C.int, []*C.char) { @@ -975,4 +976,5 @@ func resetGlobals() { hotReloadEnabled = false maxIdleTime = defaultMaxIdleTime maxRequestsPerThread = 0 + resetOpcacheRestarts() } diff --git a/frankenphp.h b/frankenphp.h index fba2078305..497630006c 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -190,6 +190,11 @@ bool frankenphp_new_php_thread(uintptr_t thread_index); bool frankenphp_shutdown_dummy_request(void); void frankenphp_update_local_thread_context(bool is_worker); +/* flags returned by frankenphp_opcache_state() */ +#define FRANKENPHP_OPCACHE_CACHED 1 +#define FRANKENPHP_OPCACHE_RESTART_PENDING 2 +int frankenphp_opcache_state(void); + int frankenphp_execute_script_cli(char *script, int argc, char **argv, bool eval); diff --git a/metrics.go b/metrics.go index 852305aa35..bb01d0227d 100644 --- a/metrics.go +++ b/metrics.go @@ -407,7 +407,7 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { Name: "frankenphp_queue_depth", Help: "Number of regular queued requests", }), - // experimental: to be removed once opcache handles restarts safely under ZTS + // experimental opcacheRestarts: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "frankenphp_opcache_restarts", Help: "Number of restarts of opcache's shared memory scheduled, by reason (experimental, should stay at zero)", diff --git a/metrics_test.go b/metrics_test.go index 4d5ec1a637..cc724d5f4b 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -257,7 +257,7 @@ func TestOpcacheRestartScheduledLogsAndCounts(t *testing.T) { m := NewPrometheusMetrics(prometheus.NewRegistry()) prevLogger, prevMetrics := globalLogger, metrics globalLogger, metrics = slog.New(slog.NewTextHandler(&buf, nil)), m - t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics }) + t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics; resetOpcacheRestarts() }) opcacheRestartScheduled(1) opcacheRestartScheduled(7) @@ -278,7 +278,7 @@ func TestOpcacheRestartScheduledWithoutOpcacheMetrics(t *testing.T) { var buf bytes.Buffer prevLogger, prevMetrics := globalLogger, metrics globalLogger, metrics = slog.New(slog.NewTextHandler(&buf, nil)), nullMetrics{} - t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics }) + t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics; resetOpcacheRestarts() }) assert.NotPanics(t, func() { opcacheRestartScheduled(0) }) assert.Contains(t, buf.String(), "reason=oom") diff --git a/opcache_test.go b/opcache_test.go new file mode 100644 index 0000000000..d9059ad9e0 --- /dev/null +++ b/opcache_test.go @@ -0,0 +1,78 @@ +package frankenphp_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A worker uses opcache's shared memory for as long as its script runs, so +// its script restarts when opcache schedules a restart of that memory. +func TestOpcacheRestartRestartsWorkers(t *testing.T) { + if frankenphp.Version().VersionID < 80400 { + t.Skip("PHP reports opcache restarts since 8.4") + } + + dir := t.TempDir() + + runTest(t, func(handler func(http.ResponseWriter, *http.Request), _ *httptest.Server, _ int) { + type status struct { + Token string + Cached bool + Pending bool + Restarts int + Corrupt bool + } + get := func(batch int) (s status) { + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "http://example.com/opcache/worker.php?dir="+dir+"&batch="+strconv.Itoa(batch), nil)) + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &s), w.Body.String()) + + return s + } + + before := get(0) + if !before.Cached { + t.Skip("opcache does not cache the worker") + } + + scheduled := before + for batch := 1; batch <= 20 && !scheduled.Pending; batch++ { + scheduled = get(batch) + } + if !scheduled.Pending { + t.Skip("could not make opcache schedule a restart") + } + + var after status + if !assert.Eventually(t, func() bool { + after = get(0) + + return after.Restarts > 0 && after.Cached && after.Token != before.Token + }, 10*time.Second, 10*time.Millisecond) { + t.Logf("before: %+v, after: %+v", before, after) + } + assert.False(t, after.Corrupt) + }, &testOptions{ + workerScript: "opcache/worker.php", + nbWorkers: 1, + nbParallelRequests: 1, + phpIni: map[string]string{ + "opcache.enable": "1", + "opcache.memory_consumption": "8", + "opcache.interned_strings_buffer": "1", + "opcache.max_accelerated_files": "200", + "opcache.max_wasted_percentage": "1", + "opcache.validate_timestamps": "1", + "opcache.revalidate_freq": "0", + "opcache.file_update_protection": "0", + }, + }) +} diff --git a/opcacherestart.go b/opcacherestart.go new file mode 100644 index 0000000000..86d4628e78 --- /dev/null +++ b/opcacherestart.go @@ -0,0 +1,157 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "log/slog" + "sync" + "sync/atomic" + "time" +) + +// A worker script uses opcache's shared memory from its start until it ends. +// A restart of that memory either waits for the worker scripts, or reuses +// memory that they still reference. When opcache schedules a restart, worker +// scripts restart at their next request boundary, and idle ones right away. +// A worker script that starts while the restart is pending isn't cached by +// opcache, so it starts again once the restart is done. + +// opcacheRestartCooldown is the minimum time between two rounds of worker +// restarts, so that a cache too small for the app doesn't restart the workers +// in a loop +var opcacheRestartCooldown = time.Minute + +var opcacheRestarts struct { + // round is incremented each time the worker scripts must restart + round atomic.Uint64 + // wake is closed, then replaced, to wake up the idle worker threads + wake atomic.Pointer[chan struct{}] + + mu sync.Mutex + lastRound time.Time + delayed *time.Timer +} + +func init() { + wake := make(chan struct{}) + opcacheRestarts.wake.Store(&wake) +} + +// restartWorkersForOpcache starts a round of worker restarts, or delays it +// until the cooldown has elapsed +func restartWorkersForOpcache() { + o := &opcacheRestarts + o.mu.Lock() + defer o.mu.Unlock() + + if o.delayed != nil { + return + } + + wait := opcacheRestartCooldown - time.Since(o.lastRound) + if wait <= 0 { + startOpcacheRestartRound() + + return + } + + if globalLogger.Enabled(globalCtx, slog.LevelWarn) { + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "opcache restarts are too frequent, delaying the restart of the workers", slog.Duration("delay", wait)) + } + + o.delayed = time.AfterFunc(wait, func() { + o.mu.Lock() + defer o.mu.Unlock() + + if o.delayed != nil { + o.delayed = nil + startOpcacheRestartRound() + } + }) +} + +// startOpcacheRestartRound must be called with opcacheRestarts.mu held +func startOpcacheRestartRound() { + opcacheRestarts.lastRound = time.Now() + opcacheRestarts.round.Add(1) + wakeWorkersForOpcache() +} + +// wakeWorkersForOpcache wakes up the idle worker threads, so that they check +// whether they must restart their script +func wakeWorkersForOpcache() { + wake := make(chan struct{}) + close(*opcacheRestarts.wake.Swap(&wake)) +} + +func opcacheWakeChan() <-chan struct{} { + return *opcacheRestarts.wake.Load() +} + +func resetOpcacheRestarts() { + o := &opcacheRestarts + o.mu.Lock() + defer o.mu.Unlock() + + if o.delayed != nil { + o.delayed.Stop() + o.delayed = nil + } + o.lastRound = time.Time{} +} + +// opcacheStarted records how opcache treats the worker script that just started +func (handler *workerThread) opcacheStarted() { + handler.opcacheStale = false + handler.opcacheUncached = false + if opcacheRestarts.round.Load() == 0 { + // no restart was scheduled so far + return + } + + state := C.frankenphp_opcache_state() + switch { + case state < 0: + case state&C.FRANKENPHP_OPCACHE_CACHED != 0: + // the script started right before opcache marked the restart as + // pending: it uses the memory the restart reuses + handler.opcacheStale = state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0 + handler.opcacheRetried = false + case state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0: + // start again once the restart is done + handler.opcacheUncached = true + default: + // the script started while the restart was running, or opcache + // skips it for another reason: start again, but only once + handler.opcacheUncached = !handler.opcacheRetried + } +} + +// opcacheRestartNeeded tells whether the worker script must restart because +// of an opcache restart +func (handler *workerThread) opcacheRestartNeeded() bool { + if round := opcacheRestarts.round.Load(); handler.opcacheStale || handler.opcacheRound != round { + // the script uses the memory the restart reuses; ask only once, in + // case the script keeps calling frankenphp_handle_request() + handler.opcacheStale = false + handler.opcacheRound = round + handler.opcacheRetried = false + + return true + } + + if !handler.opcacheUncached { + return false + } + + state := C.frankenphp_opcache_state() + if state >= 0 && state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0 { + // keep running without the cache until the restart is done + return false + } + + handler.opcacheUncached = false + handler.opcacheRetried = true + + return state >= 0 +} diff --git a/opcacherestart_test.go b/opcacherestart_test.go new file mode 100644 index 0000000000..392230b0da --- /dev/null +++ b/opcacherestart_test.go @@ -0,0 +1,41 @@ +package frankenphp + +import ( + "bytes" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestOpcacheRestartsWithinTheCooldownAreCoalesced(t *testing.T) { + var buf bytes.Buffer + prevLogger, prevCooldown := globalLogger, opcacheRestartCooldown + globalLogger, opcacheRestartCooldown = slog.New(slog.NewTextHandler(&buf, nil)), 100*time.Millisecond + resetOpcacheRestarts() + t.Cleanup(func() { + resetOpcacheRestarts() + globalLogger, opcacheRestartCooldown = prevLogger, prevCooldown + }) + + round := opcacheRestarts.round.Load() + wake := opcacheWakeChan() + + restartWorkersForOpcache() + assert.Equal(t, round+1, opcacheRestarts.round.Load(), "the first restart must restart the workers right away") + select { + case <-wake: + default: + t.Error("the idle workers must be woken up") + } + + restartWorkersForOpcache() + restartWorkersForOpcache() + assert.Equal(t, round+1, opcacheRestarts.round.Load(), "restarts within the cooldown must be delayed") + assert.Contains(t, buf.String(), `level=WARN msg="opcache restarts are too frequent`) + + assert.Eventually(t, func() bool { return opcacheRestarts.round.Load() == round+2 }, time.Second, 10*time.Millisecond) + time.Sleep(2 * opcacheRestartCooldown) + assert.Equal(t, round+2, opcacheRestarts.round.Load(), "delayed restarts must make a single round") +} diff --git a/testdata/opcache/worker.php b/testdata/opcache/worker.php new file mode 100644 index 0000000000..94731ffda9 --- /dev/null +++ b/testdata/opcache/worker.php @@ -0,0 +1,45 @@ + 1, 'b' => [2, 3], 'c' => 'literal']; + +$token = bin2hex(random_bytes(4)); +$canary = 'opcache_canary_interned_string'; +$expected = crc32($canary.serialize(OPCACHE_CANARY)); + +$handler = static function () use ($token, $canary, $expected) { + if (!function_exists('opcache_get_status')) { + echo '{}'; + + return; + } + + // compile and invalidate scripts until opcache schedules a restart + if ($batch = (int) ($_GET['batch'] ?? 0)) { + $dir = $_GET['dir']; + for ($i = 0; $i < 120 && !opcache_get_status(false)['restart_pending']; ++$i) { + file_put_contents("$dir/f{$batch}_$i.php", " $token, + 'cached' => $status['opcache_enabled'], + 'pending' => $status['restart_pending'], + 'restarts' => $s['oom_restarts'] + $s['hash_restarts'] + $s['manual_restarts'], + 'corrupt' => crc32($canary.serialize(OPCACHE_CANARY)) !== $expected, + ]); +}; + +while (frankenphp_handle_request($handler)); diff --git a/threadworker.go b/threadworker.go index b542b9aa35..4ca9e37e1b 100644 --- a/threadworker.go +++ b/threadworker.go @@ -20,9 +20,13 @@ type workerThread struct { worker *worker dummyFrankenPHPContext *frankenPHPContext workerFrankenPHPContext *frankenPHPContext - isBootingScript bool // true if the worker has not reached frankenphp_handle_request yet - failureCount int // number of consecutive startup failures - requestCount int // number of requests handled since last restart + isBootingScript bool // true if the worker has not reached frankenphp_handle_request yet + failureCount int // number of consecutive startup failures + requestCount int // number of requests handled since last restart + opcacheRound uint64 // round of opcache restarts when the script started + opcacheStale bool // cached by opcache although a restart is pending + opcacheUncached bool // not cached by opcache because a restart was pending + opcacheRetried bool // started again because opcache did not cache it } func convertToWorkerThread(thread *phpThread, worker *worker) { @@ -101,6 +105,7 @@ func setupWorkerScript(handler *workerThread, worker *worker) { handler.dummyFrankenPHPContext = fc handler.isBootingScript = true handler.requestCount = 0 + handler.opcacheRound = opcacheRestarts.round.Load() if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "starting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex)) @@ -196,6 +201,8 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { // worker is truly ready only after reaching frankenphp_handle_request() metrics.ReadyWorker(handler.worker.name) + + handler.opcacheStarted() } // max_requests reached: signal reboot for full ZTS cleanup @@ -217,18 +224,32 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { handler.state.Set(state.Ready) } + // opcache schedules a restart of its memory: restart the script to let go of it + if handler.opcacheRestartNeeded() { + return false, nil + } + handler.state.MarkAsWaiting(true) var fc *frankenPHPContext - select { - case <-handler.thread.drainChan: - if globalLogger.Enabled(globalCtx, slog.LevelDebug) { - globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "shutting down", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) - } + for wake := opcacheWakeChan(); fc == nil; { + select { + case <-handler.thread.drainChan: + if globalLogger.Enabled(globalCtx, slog.LevelDebug) { + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "shutting down", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) + } - return false, nil - case fc = <-handler.thread.requestChan: - case fc = <-handler.worker.requestChan: + return false, nil + case <-wake: + if handler.opcacheRestartNeeded() { + handler.state.MarkAsWaiting(false) + + return false, nil + } + wake = opcacheWakeChan() + case fc = <-handler.thread.requestChan: + case fc = <-handler.worker.requestChan: + } } handler.requestCount++