diff --git a/docker-compose.yml b/docker-compose.yml index b273dfdfc59..7317c64ba5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -570,7 +570,7 @@ services: args: BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" - command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' + command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS" "0 0 * * * gosu sentry find /data/replays -mindepth 1 -mtime +$SENTRY_EVENT_RETENTION_DAYS -delete"' nginx: <<: *restart_policy ports: diff --git a/install.sh b/install.sh index f6515c70455..9c634195597 100755 --- a/install.sh +++ b/install.sh @@ -38,6 +38,7 @@ source install/ensure-relay-credentials.sh source install/generate-secret-key.sh source install/build-docker-images.sh source install/ensure-sentry-data-ownership.sh +source install/migrate-replay-storage.sh source install/migrate-seaweedfs-kek.sh source install/upgrade-postgres.sh source install/bootstrap-s3-nodestore.sh diff --git a/install/migrate-replay-storage.sh b/install/migrate-replay-storage.sh new file mode 100644 index 00000000000..b13440a73c5 --- /dev/null +++ b/install/migrate-replay-storage.sh @@ -0,0 +1,100 @@ +echo "${_group}Migrating Session Replay recordings storage ..." + +# Session Replay recordings used to go to the default filestore (`/data/files`) because +# `replay.storage.backend` was not set. Sentry writes them straight to storage, without +# `File` rows, so `sentry cleanup` never deletes them and they piled up forever. +# +# They now live in `/data/replays`, which the `sentry-cleanup` container prunes after +# `SENTRY_EVENT_RETENTION_DAYS`. On upgrade, this points `replay.storage.*` there and moves +# existing recordings over so they keep playing back and expire like new ones. + +move_replay_recordings=0 +if ! grep -q "replay.storage.backend" $SENTRY_CONFIG_YML; then + apply_config_changes_replays=0 + if [[ -z "$APPLY_AUTOMATIC_CONFIG_UPDATES" ]]; then + echo + echo "Session Replay recordings are moving from '/data/files' to '/data/replays'," + echo "so they are deleted automatically once they pass the retention period." + echo "To complete this, your sentry/config.yml file needs to be modified and existing" + echo "recordings need to be moved to the new directory." + echo "Would you like us to perform this modification automatically?" + echo + + yn="" + until [ ! -z "$yn" ]; do + read -p "y or n? " yn + case $yn in + y | yes | 1) + export apply_config_changes_replays=1 + echo + echo -n "Thank you." + ;; + n | no | 0) + export apply_config_changes_replays=0 + echo + echo -n "Alright, replay recordings will keep going to '/data/files' and will not be deleted automatically." + ;; + *) yn="" ;; + esac + done + + echo + echo "To avoid this prompt in the future, use one of these flags:" + echo + echo " --apply-automatic-config-updates" + echo " --no-apply-automatic-config-updates" + echo + echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" + echo + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" + echo + sleep 5 + fi + + if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_replays" == 1 ]]; then + replays_config=$(sed -n "/^replay.storage.backend/,/location: '\/data\/replays'/{p}" sentry/config.example.yml) + echo "$replays_config" >>$SENTRY_CONFIG_YML + move_replay_recordings=1 + fi +fi + +# Recordings in `/data/files` are stored as ///. +# Regular file blobs are stored as <2 hex>/<4 hex>/<26 hex> files, so they never match a +# directory at the third level and are left alone. +$dcr --no-deps -e MOVE_REPLAY_RECORDINGS=$move_replay_recordings --entrypoint python3 web -c ' +import os +import re +import shutil + +SOURCE = "/data/files" +TARGET = "/data/replays" +NUMBER = re.compile(r"[0-9]+") +REPLAY_ID = re.compile(r"[0-9a-f]{32}") + + +def subdirs(path, pattern): + return [e for e in os.scandir(path) if e.is_dir() and pattern.fullmatch(e.name)] + + +os.makedirs(TARGET, exist_ok=True) + +if os.environ["MOVE_REPLAY_RECORDINGS"] == "1" and os.path.isdir(SOURCE): + moved = 0 + for retention in subdirs(SOURCE, NUMBER): + for project in subdirs(retention.path, NUMBER): + for replay in subdirs(project.path, REPLAY_ID): + destination = os.path.join(TARGET, retention.name, project.name, replay.name) + if os.path.exists(destination): + continue + os.makedirs(os.path.dirname(destination), exist_ok=True) + shutil.move(replay.path, destination) + moved += 1 + try: + os.rmdir(project.path) + except OSError: + pass + print(f"Moved {moved} replay recordings from {SOURCE} to {TARGET}.") +' + +echo "${_endgroup}" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 07135809a3f..d55f10f9b30 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -113,6 +113,13 @@ filestore.profiles-options: addressing_style: "path" signature_version: "s3v4" +# Session Replay recordings are kept in their own directory so the +# `sentry-cleanup` cron job can delete them after the retention period. +# Sentry does not delete them on its own. +replay.storage.backend: 'filesystem' +replay.storage.options: + location: '/data/replays' + symbolicator.enabled: true symbolicator.options: url: "http://symbolicator:3021"