Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP

For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used.

opcache schedules a restart when its cache is full and more than `opcache.max_wasted_percentage` of it was wasted by invalidations. The restart runs at the next request start; caching is off until then, and running PHP threads may still reference the old memory, which can crash the process. The counter should stay at zero: raise `opcache.memory_consumption`, `opcache.max_accelerated_files` or `opcache.max_wasted_percentage`. `manual` should never appear, FrankenPHP overrides `opcache_reset()`. Each restart is also logged. This metric will be removed once opcache handles restarts safely under ZTS.
opcache schedules a restart when its cache is full and more than `opcache.max_wasted_percentage` of it was wasted by invalidations. The restart runs at the next request start; caching is off until then, and running PHP threads may still reference the old memory, which can crash the process. A worker uses this memory for as long as its script runs, so FrankenPHP restarts the worker scripts when a restart is scheduled, at most once a minute. The counter should stay at zero: raise `opcache.memory_consumption`, `opcache.max_accelerated_files` or `opcache.max_wasted_percentage`. `manual` should never appear, FrankenPHP overrides `opcache_reset()`. Each restart is also logged.

## Threads State Endpoint

Expand Down
47 changes: 47 additions & 0 deletions frankenphp.c
Original file line number Diff line number Diff line change
Expand Up @@ -1176,6 +1176,53 @@ static void frankenphp_opcache_restart_hook(int reason) {
}
#endif

/* Tells, through opcache_get_status(), whether opcache caches the scripts of
* the current request (FRANKENPHP_OPCACHE_CACHED) and whether a restart is
* pending or in progress (FRANKENPHP_OPCACHE_RESTART_PENDING), or -1 when
* opcache cannot tell */
int frankenphp_opcache_state(void) {
zend_function *func = zend_hash_str_find_ptr(CG(function_table),
ZEND_STRL("opcache_get_status"));
if (func == NULL) {
return -1;
}

zval status, fetch_scripts;
ZVAL_FALSE(&fetch_scripts);

/* opcache.restrict_api makes opcache_get_status() raise a warning */
int error_reporting = EG(error_reporting);
EG(error_reporting) = 0;
zend_call_known_function(func, NULL, NULL, &status, 1, &fetch_scripts, NULL);
EG(error_reporting) = error_reporting;

if (Z_TYPE(status) != IS_ARRAY) {
zval_ptr_dtor(&status);

return -1;
}

int state = 0;
zval *value =
zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("opcache_enabled"));
if (value != NULL && zend_is_true(value)) {
state |= FRANKENPHP_OPCACHE_CACHED;
}
value = zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("restart_pending"));
if (value != NULL && zend_is_true(value)) {
state |= FRANKENPHP_OPCACHE_RESTART_PENDING;
}
value =
zend_hash_str_find(Z_ARRVAL(status), ZEND_STRL("restart_in_progress"));
if (value != NULL && zend_is_true(value)) {
state |= FRANKENPHP_OPCACHE_RESTART_PENDING;
}

zval_ptr_dtor(&status);

return state;
}

/* {{{ thread-safe opcache reset */
PHP_FUNCTION(frankenphp_opcache_reset) {
go_schedule_opcache_reset(frankenphp_thread_index());
Expand Down
18 changes: 10 additions & 8 deletions frankenphp.go
Original file line number Diff line number Diff line change
Expand Up @@ -929,16 +929,17 @@ func opcacheRestartScheduled(reason int) {
m.OpcacheRestart(reasonText)
}

if !globalLogger.Enabled(globalCtx, slog.LevelWarn) {
return
if globalLogger.Enabled(globalCtx, slog.LevelWarn) {
// written synchronously, under opcache's lock: a line deferred to a
// goroutine is lost if the restart crashes the process
globalLogger.LogAttrs(globalCtx, slog.LevelWarn,
"opcache restart scheduled, caching stops until the next request start carries it out while other threads may still reference the old memory: raise opcache.memory_consumption, opcache.max_accelerated_files or opcache.max_wasted_percentage",
slog.String("reason", reasonText),
)
}

// written synchronously, under opcache's lock: a line deferred to a
// goroutine is lost if the restart crashes the process
globalLogger.LogAttrs(globalCtx, slog.LevelWarn,
"opcache restart scheduled, caching stops until the next request start carries it out while other threads may still reference the old memory: raise opcache.memory_consumption, opcache.max_accelerated_files or opcache.max_wasted_percentage",
slog.String("reason", reasonText),
)
// last, as opcache marks the restart as pending only once this returns
restartWorkersForOpcache()
}

func convertArgs(args []string) (C.int, []*C.char) {
Expand Down Expand Up @@ -975,4 +976,5 @@ func resetGlobals() {
hotReloadEnabled = false
maxIdleTime = defaultMaxIdleTime
maxRequestsPerThread = 0
resetOpcacheRestarts()
}
5 changes: 5 additions & 0 deletions frankenphp.h
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,11 @@ bool frankenphp_new_php_thread(uintptr_t thread_index);
bool frankenphp_shutdown_dummy_request(void);
void frankenphp_update_local_thread_context(bool is_worker);

/* flags returned by frankenphp_opcache_state() */
#define FRANKENPHP_OPCACHE_CACHED 1
#define FRANKENPHP_OPCACHE_RESTART_PENDING 2
int frankenphp_opcache_state(void);

int frankenphp_execute_script_cli(char *script, int argc, char **argv,
bool eval);

Expand Down
2 changes: 1 addition & 1 deletion metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics {
Name: "frankenphp_queue_depth",
Help: "Number of regular queued requests",
}),
// experimental: to be removed once opcache handles restarts safely under ZTS
// experimental
opcacheRestarts: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "frankenphp_opcache_restarts",
Help: "Number of restarts of opcache's shared memory scheduled, by reason (experimental, should stay at zero)",
Expand Down
4 changes: 2 additions & 2 deletions metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func TestOpcacheRestartScheduledLogsAndCounts(t *testing.T) {
m := NewPrometheusMetrics(prometheus.NewRegistry())
prevLogger, prevMetrics := globalLogger, metrics
globalLogger, metrics = slog.New(slog.NewTextHandler(&buf, nil)), m
t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics })
t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics; resetOpcacheRestarts() })

opcacheRestartScheduled(1)
opcacheRestartScheduled(7)
Expand All @@ -278,7 +278,7 @@ func TestOpcacheRestartScheduledWithoutOpcacheMetrics(t *testing.T) {
var buf bytes.Buffer
prevLogger, prevMetrics := globalLogger, metrics
globalLogger, metrics = slog.New(slog.NewTextHandler(&buf, nil)), nullMetrics{}
t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics })
t.Cleanup(func() { globalLogger, metrics = prevLogger, prevMetrics; resetOpcacheRestarts() })

assert.NotPanics(t, func() { opcacheRestartScheduled(0) })
assert.Contains(t, buf.String(), "reason=oom")
Expand Down
78 changes: 78 additions & 0 deletions opcache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package frankenphp_test

import (
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"

"github.com/dunglas/frankenphp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// A worker uses opcache's shared memory for as long as its script runs, so
// its script restarts when opcache schedules a restart of that memory.
func TestOpcacheRestartRestartsWorkers(t *testing.T) {
if frankenphp.Version().VersionID < 80400 {
t.Skip("PHP reports opcache restarts since 8.4")
}

dir := t.TempDir()

runTest(t, func(handler func(http.ResponseWriter, *http.Request), _ *httptest.Server, _ int) {
type status struct {
Token string
Cached bool
Pending bool
Restarts int
Corrupt bool
}
get := func(batch int) (s status) {
w := httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "http://example.com/opcache/worker.php?dir="+dir+"&batch="+strconv.Itoa(batch), nil))
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &s), w.Body.String())

return s
}

before := get(0)
if !before.Cached {
t.Skip("opcache does not cache the worker")
}

scheduled := before
for batch := 1; batch <= 20 && !scheduled.Pending; batch++ {
scheduled = get(batch)
}
if !scheduled.Pending {
t.Skip("could not make opcache schedule a restart")
}

var after status
if !assert.Eventually(t, func() bool {
after = get(0)

return after.Restarts > 0 && after.Cached && after.Token != before.Token
}, 10*time.Second, 10*time.Millisecond) {
t.Logf("before: %+v, after: %+v", before, after)
}
assert.False(t, after.Corrupt)
}, &testOptions{
workerScript: "opcache/worker.php",
nbWorkers: 1,
nbParallelRequests: 1,
phpIni: map[string]string{
"opcache.enable": "1",
"opcache.memory_consumption": "8",
"opcache.interned_strings_buffer": "1",
"opcache.max_accelerated_files": "200",
"opcache.max_wasted_percentage": "1",
"opcache.validate_timestamps": "1",
"opcache.revalidate_freq": "0",
"opcache.file_update_protection": "0",
},
})
}
157 changes: 157 additions & 0 deletions opcacherestart.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package frankenphp

// #include "frankenphp.h"
import "C"
import (
"log/slog"
"sync"
"sync/atomic"
"time"
)

// A worker script uses opcache's shared memory from its start until it ends.
// A restart of that memory either waits for the worker scripts, or reuses
// memory that they still reference. When opcache schedules a restart, worker
// scripts restart at their next request boundary, and idle ones right away.
// A worker script that starts while the restart is pending isn't cached by
// opcache, so it starts again once the restart is done.

// opcacheRestartCooldown is the minimum time between two rounds of worker
// restarts, so that a cache too small for the app doesn't restart the workers
// in a loop
var opcacheRestartCooldown = time.Minute

var opcacheRestarts struct {
// round is incremented each time the worker scripts must restart
round atomic.Uint64
// wake is closed, then replaced, to wake up the idle worker threads
wake atomic.Pointer[chan struct{}]

mu sync.Mutex
lastRound time.Time
delayed *time.Timer
}

func init() {
wake := make(chan struct{})
opcacheRestarts.wake.Store(&wake)
}

// restartWorkersForOpcache starts a round of worker restarts, or delays it
// until the cooldown has elapsed
func restartWorkersForOpcache() {
o := &opcacheRestarts
o.mu.Lock()
defer o.mu.Unlock()

if o.delayed != nil {
return
}

wait := opcacheRestartCooldown - time.Since(o.lastRound)
if wait <= 0 {
startOpcacheRestartRound()

return
}

if globalLogger.Enabled(globalCtx, slog.LevelWarn) {
globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "opcache restarts are too frequent, delaying the restart of the workers", slog.Duration("delay", wait))
}

o.delayed = time.AfterFunc(wait, func() {
o.mu.Lock()
defer o.mu.Unlock()

if o.delayed != nil {
o.delayed = nil
startOpcacheRestartRound()
}
})
}

// startOpcacheRestartRound must be called with opcacheRestarts.mu held
func startOpcacheRestartRound() {
opcacheRestarts.lastRound = time.Now()
opcacheRestarts.round.Add(1)
wakeWorkersForOpcache()
}

// wakeWorkersForOpcache wakes up the idle worker threads, so that they check
// whether they must restart their script
func wakeWorkersForOpcache() {
wake := make(chan struct{})
close(*opcacheRestarts.wake.Swap(&wake))
}

func opcacheWakeChan() <-chan struct{} {
return *opcacheRestarts.wake.Load()
}

func resetOpcacheRestarts() {
o := &opcacheRestarts
o.mu.Lock()
defer o.mu.Unlock()

if o.delayed != nil {
o.delayed.Stop()
o.delayed = nil
}
o.lastRound = time.Time{}
}

// opcacheStarted records how opcache treats the worker script that just started
func (handler *workerThread) opcacheStarted() {
handler.opcacheStale = false
handler.opcacheUncached = false
if opcacheRestarts.round.Load() == 0 {
// no restart was scheduled so far
return
}

state := C.frankenphp_opcache_state()
switch {
case state < 0:
case state&C.FRANKENPHP_OPCACHE_CACHED != 0:
// the script started right before opcache marked the restart as
// pending: it uses the memory the restart reuses
handler.opcacheStale = state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0
handler.opcacheRetried = false
case state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0:
// start again once the restart is done
handler.opcacheUncached = true
default:
// the script started while the restart was running, or opcache
// skips it for another reason: start again, but only once
handler.opcacheUncached = !handler.opcacheRetried
}
}

// opcacheRestartNeeded tells whether the worker script must restart because
// of an opcache restart
func (handler *workerThread) opcacheRestartNeeded() bool {
if round := opcacheRestarts.round.Load(); handler.opcacheStale || handler.opcacheRound != round {
// the script uses the memory the restart reuses; ask only once, in
// case the script keeps calling frankenphp_handle_request()
handler.opcacheStale = false
handler.opcacheRound = round
handler.opcacheRetried = false

return true
}

if !handler.opcacheUncached {
return false
}

state := C.frankenphp_opcache_state()
if state >= 0 && state&C.FRANKENPHP_OPCACHE_RESTART_PENDING != 0 {
// keep running without the cache until the restart is done
return false
}

handler.opcacheUncached = false
handler.opcacheRetried = true

return state >= 0
}
Loading
Loading