From 361dd091237a889d598f6f8c0e5acfb74c0e2166 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:33:42 +0000 Subject: [PATCH 1/2] fix(replays): Store replay recordings in SeaweedFS with a lifecycle policy Replay recording segments were written to the default filestore on the sentry-data volume. Sentry writes them without File rows and relies on a bucket lifecycle policy to expire them, so nothing ever deleted them. Store them in a 'replays' SeaweedFS bucket that expires objects after SENTRY_EVENT_RETENTION_DAYS, mirroring the profiles bucket setup. Refs getsentry/sentry#121278 Co-Authored-By: Dominik Buszowiecki <44422760+DominikB2014@users.noreply.github.com> --- _unit-test/bootstrap-s3-replays-test.sh | 26 ++++++ install.sh | 1 + install/bootstrap-s3-replays.sh | 101 ++++++++++++++++++++++++ sentry/config.example.yml | 15 ++++ 4 files changed, 143 insertions(+) create mode 100755 _unit-test/bootstrap-s3-replays-test.sh create mode 100644 install/bootstrap-s3-replays.sh diff --git a/_unit-test/bootstrap-s3-replays-test.sh b/_unit-test/bootstrap-s3-replays-test.sh new file mode 100755 index 00000000000..77936fbb280 --- /dev/null +++ b/_unit-test/bootstrap-s3-replays-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source _unit-test/_test_setup.sh +source install/dc-detect-version.sh +source install/create-docker-volumes.sh +source install/ensure-files-from-examples.sh +export COMPOSE_PROFILES="feature-complete" +# Set the flag to apply automatic updates +export APPLY_AUTOMATIC_CONFIG_UPDATES=1 + +# Here we're just gonna test to run it multiple times +# Only to make sure it doesn't break +for i in $(seq 1 5); do + source install/bootstrap-s3-replays.sh +done + +# Ensure that the bucket has been created +if ! $dc exec seaweedfs s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" ls | grep -q "s3://replays"; then + echo "Error: Expected the 'replays' bucket to exist" + exit 1 +fi + +# Manual cleanup, otherwise `create-docker-volumes.sh` will fail +$dc down -v --remove-orphans + +report_success diff --git a/install.sh b/install.sh index f6515c70455..53f93dde115 100755 --- a/install.sh +++ b/install.sh @@ -43,6 +43,7 @@ source install/upgrade-postgres.sh source install/bootstrap-s3-nodestore.sh source install/bootstrap-snuba.sh source install/bootstrap-s3-profiles.sh +source install/bootstrap-s3-replays.sh source install/set-up-and-migrate-database.sh source install/migrate-pgbouncer.sh source install/geoip.sh diff --git a/install/bootstrap-s3-replays.sh b/install/bootstrap-s3-replays.sh new file mode 100644 index 00000000000..745672e6d26 --- /dev/null +++ b/install/bootstrap-s3-replays.sh @@ -0,0 +1,101 @@ +# Session Replay recording segments used to go to the default filestore (`/data/files` on the +# `sentry-data` volume) because `replay.storage.backend` was not set. Sentry writes these blobs +# straight to storage, without `File` rows, and expects a bucket lifecycle policy to expire them. +# `sentry cleanup` never deletes them, so they piled up forever. +# +# This script creates a "replays" bucket on SeaweedFS with a lifecycle policy matching +# `SENTRY_EVENT_RETENTION_DAYS`, and points `replay.storage.*` at it. + +# Should only run when `$COMPOSE_PROFILES` is set to `feature-complete` +if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then + echo "${_group}Bootstrapping seaweedfs (replays)..." + + start_service_and_wait_ready seaweedfs + $dcx seaweedfs apk add --no-cache s3cmd + s3cmd="$dc exec seaweedfs s3cmd" + + bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) + + if ! echo "$bucket_list" | grep -q "s3://replays"; then + apply_config_changes_replays=0 + # Only touch if no existing replay storage config is found + if ! grep -q "replay.storage.backend" $SENTRY_CONFIG_YML; then + if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then + echo + echo "We are moving Session Replay recordings from the 'sentry-data' volume to SeaweedFS," + echo "so they are deleted automatically once they pass the retention period." + echo "To complete this, your sentry/config.yml file needs to be modified." + echo + echo "Replays recorded before this change stay on the 'sentry-data' volume and will no" + echo "longer play back. They are not deleted automatically." + echo "Would you like us to perform this modification automatically?" + echo + + yn="" + until [ ! -z "$yn" ]; do + read -p "y or n? " yn + case $yn in + y | yes | 1) + export apply_config_changes_replays=1 + echo + echo -n "Thank you." + ;; + n | no | 0) + export apply_config_changes_replays=0 + echo + echo -n "Alright, you will need to update your sentry/config.yml file manually before running 'docker compose up'." + ;; + *) yn="" ;; + esac + done + + echo + echo "To avoid this prompt in the future, use one of these flags:" + echo + echo " --apply-automatic-config-updates" + echo " --no-apply-automatic-config-updates" + echo + echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" + echo + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" + echo + sleep 5 + fi + + if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_replays" == 1 ]]; then + replays_config=$(sed -n '/^replay.storage.backend/,/s3v4"/{p}' sentry/config.example.yml) + echo "$replays_config" >>$SENTRY_CONFIG_YML + fi + fi + + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://replays + else + echo "'replays' bucket already exists on SeaweedFS. Skipping creation." + fi + + if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" || "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 ]]; then + lifecycle_policy=$( + cat < + + + Sentry-Replays-Rule + Enabled + + + $SENTRY_EVENT_RETENTION_DAYS + + + +EOF + ) + + $dc exec seaweedfs sh -c "printf '%s' '$lifecycle_policy' > /tmp/replays-lifecycle-policy.xml" + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' setlifecycle /tmp/replays-lifecycle-policy.xml s3://replays + + echo "Making sure the bucket lifecycle policy is all set up correctly..." + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' getlifecycle s3://replays + fi + echo "${_endgroup}" +fi diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 07135809a3f..14578bd19cd 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -113,6 +113,21 @@ filestore.profiles-options: addressing_style: "path" signature_version: "s3v4" +# Session Replay recordings are stored in their own SeaweedFS bucket. The +# bucket's lifecycle policy deletes recordings once they pass the retention +# period, because Sentry does not delete them on its own. +replay.storage.backend: 's3' +replay.storage.options: + bucket_acl: "private" + default_acl: "private" + access_key: "sentry" + secret_key: "sentry" + bucket_name: "replays" + region_name: "us-east-1" + endpoint_url: "http://seaweedfs:8333" + addressing_style: "path" + signature_version: "s3v4" + symbolicator.enabled: true symbolicator.options: url: "http://symbolicator:3021" From 1b14bef067624017d9ea9ca166b4e44a6ee9229c Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:40:11 +0000 Subject: [PATCH 2/2] fix(replays): Keep replay recordings on the filesystem and prune them Drop the SeaweedFS bucket in favor of the filesystem backend with a dedicated /data/replays directory. A daily sentry-cleanup cron job deletes recordings older than SENTRY_EVENT_RETENTION_DAYS. On upgrade, install points replay.storage.* at /data/replays and moves existing recordings out of /data/files so they keep playing back and expire like new ones. Co-Authored-By: Dominik Buszowiecki <44422760+DominikB2014@users.noreply.github.com> --- _unit-test/bootstrap-s3-replays-test.sh | 26 ------ docker-compose.yml | 2 +- install.sh | 2 +- install/bootstrap-s3-replays.sh | 101 ------------------------ install/migrate-replay-storage.sh | 100 +++++++++++++++++++++++ sentry/config.example.yml | 18 ++--- 6 files changed, 107 insertions(+), 142 deletions(-) delete mode 100755 _unit-test/bootstrap-s3-replays-test.sh delete mode 100644 install/bootstrap-s3-replays.sh create mode 100644 install/migrate-replay-storage.sh diff --git a/_unit-test/bootstrap-s3-replays-test.sh b/_unit-test/bootstrap-s3-replays-test.sh deleted file mode 100755 index 77936fbb280..00000000000 --- a/_unit-test/bootstrap-s3-replays-test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -source _unit-test/_test_setup.sh -source install/dc-detect-version.sh -source install/create-docker-volumes.sh -source install/ensure-files-from-examples.sh -export COMPOSE_PROFILES="feature-complete" -# Set the flag to apply automatic updates -export APPLY_AUTOMATIC_CONFIG_UPDATES=1 - -# Here we're just gonna test to run it multiple times -# Only to make sure it doesn't break -for i in $(seq 1 5); do - source install/bootstrap-s3-replays.sh -done - -# Ensure that the bucket has been created -if ! $dc exec seaweedfs s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" ls | grep -q "s3://replays"; then - echo "Error: Expected the 'replays' bucket to exist" - exit 1 -fi - -# Manual cleanup, otherwise `create-docker-volumes.sh` will fail -$dc down -v --remove-orphans - -report_success diff --git a/docker-compose.yml b/docker-compose.yml index b273dfdfc59..7317c64ba5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -570,7 +570,7 @@ services: args: BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" - command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' + command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS" "0 0 * * * gosu sentry find /data/replays -mindepth 1 -mtime +$SENTRY_EVENT_RETENTION_DAYS -delete"' nginx: <<: *restart_policy ports: diff --git a/install.sh b/install.sh index 53f93dde115..9c634195597 100755 --- a/install.sh +++ b/install.sh @@ -38,12 +38,12 @@ source install/ensure-relay-credentials.sh source install/generate-secret-key.sh source install/build-docker-images.sh source install/ensure-sentry-data-ownership.sh +source install/migrate-replay-storage.sh source install/migrate-seaweedfs-kek.sh source install/upgrade-postgres.sh source install/bootstrap-s3-nodestore.sh source install/bootstrap-snuba.sh source install/bootstrap-s3-profiles.sh -source install/bootstrap-s3-replays.sh source install/set-up-and-migrate-database.sh source install/migrate-pgbouncer.sh source install/geoip.sh diff --git a/install/bootstrap-s3-replays.sh b/install/bootstrap-s3-replays.sh deleted file mode 100644 index 745672e6d26..00000000000 --- a/install/bootstrap-s3-replays.sh +++ /dev/null @@ -1,101 +0,0 @@ -# Session Replay recording segments used to go to the default filestore (`/data/files` on the -# `sentry-data` volume) because `replay.storage.backend` was not set. Sentry writes these blobs -# straight to storage, without `File` rows, and expects a bucket lifecycle policy to expire them. -# `sentry cleanup` never deletes them, so they piled up forever. -# -# This script creates a "replays" bucket on SeaweedFS with a lifecycle policy matching -# `SENTRY_EVENT_RETENTION_DAYS`, and points `replay.storage.*` at it. - -# Should only run when `$COMPOSE_PROFILES` is set to `feature-complete` -if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then - echo "${_group}Bootstrapping seaweedfs (replays)..." - - start_service_and_wait_ready seaweedfs - $dcx seaweedfs apk add --no-cache s3cmd - s3cmd="$dc exec seaweedfs s3cmd" - - bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) - - if ! echo "$bucket_list" | grep -q "s3://replays"; then - apply_config_changes_replays=0 - # Only touch if no existing replay storage config is found - if ! grep -q "replay.storage.backend" $SENTRY_CONFIG_YML; then - if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then - echo - echo "We are moving Session Replay recordings from the 'sentry-data' volume to SeaweedFS," - echo "so they are deleted automatically once they pass the retention period." - echo "To complete this, your sentry/config.yml file needs to be modified." - echo - echo "Replays recorded before this change stay on the 'sentry-data' volume and will no" - echo "longer play back. They are not deleted automatically." - echo "Would you like us to perform this modification automatically?" - echo - - yn="" - until [ ! -z "$yn" ]; do - read -p "y or n? " yn - case $yn in - y | yes | 1) - export apply_config_changes_replays=1 - echo - echo -n "Thank you." - ;; - n | no | 0) - export apply_config_changes_replays=0 - echo - echo -n "Alright, you will need to update your sentry/config.yml file manually before running 'docker compose up'." - ;; - *) yn="" ;; - esac - done - - echo - echo "To avoid this prompt in the future, use one of these flags:" - echo - echo " --apply-automatic-config-updates" - echo " --no-apply-automatic-config-updates" - echo - echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" - echo - echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" - echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" - echo - sleep 5 - fi - - if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_replays" == 1 ]]; then - replays_config=$(sed -n '/^replay.storage.backend/,/s3v4"/{p}' sentry/config.example.yml) - echo "$replays_config" >>$SENTRY_CONFIG_YML - fi - fi - - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://replays - else - echo "'replays' bucket already exists on SeaweedFS. Skipping creation." - fi - - if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" || "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 ]]; then - lifecycle_policy=$( - cat < - - - Sentry-Replays-Rule - Enabled - - - $SENTRY_EVENT_RETENTION_DAYS - - - -EOF - ) - - $dc exec seaweedfs sh -c "printf '%s' '$lifecycle_policy' > /tmp/replays-lifecycle-policy.xml" - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' setlifecycle /tmp/replays-lifecycle-policy.xml s3://replays - - echo "Making sure the bucket lifecycle policy is all set up correctly..." - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' getlifecycle s3://replays - fi - echo "${_endgroup}" -fi diff --git a/install/migrate-replay-storage.sh b/install/migrate-replay-storage.sh new file mode 100644 index 00000000000..b13440a73c5 --- /dev/null +++ b/install/migrate-replay-storage.sh @@ -0,0 +1,100 @@ +echo "${_group}Migrating Session Replay recordings storage ..." + +# Session Replay recordings used to go to the default filestore (`/data/files`) because +# `replay.storage.backend` was not set. Sentry writes them straight to storage, without +# `File` rows, so `sentry cleanup` never deletes them and they piled up forever. +# +# They now live in `/data/replays`, which the `sentry-cleanup` container prunes after +# `SENTRY_EVENT_RETENTION_DAYS`. On upgrade, this points `replay.storage.*` there and moves +# existing recordings over so they keep playing back and expire like new ones. + +move_replay_recordings=0 +if ! grep -q "replay.storage.backend" $SENTRY_CONFIG_YML; then + apply_config_changes_replays=0 + if [[ -z "$APPLY_AUTOMATIC_CONFIG_UPDATES" ]]; then + echo + echo "Session Replay recordings are moving from '/data/files' to '/data/replays'," + echo "so they are deleted automatically once they pass the retention period." + echo "To complete this, your sentry/config.yml file needs to be modified and existing" + echo "recordings need to be moved to the new directory." + echo "Would you like us to perform this modification automatically?" + echo + + yn="" + until [ ! -z "$yn" ]; do + read -p "y or n? " yn + case $yn in + y | yes | 1) + export apply_config_changes_replays=1 + echo + echo -n "Thank you." + ;; + n | no | 0) + export apply_config_changes_replays=0 + echo + echo -n "Alright, replay recordings will keep going to '/data/files' and will not be deleted automatically." + ;; + *) yn="" ;; + esac + done + + echo + echo "To avoid this prompt in the future, use one of these flags:" + echo + echo " --apply-automatic-config-updates" + echo " --no-apply-automatic-config-updates" + echo + echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" + echo + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" + echo + sleep 5 + fi + + if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_replays" == 1 ]]; then + replays_config=$(sed -n "/^replay.storage.backend/,/location: '\/data\/replays'/{p}" sentry/config.example.yml) + echo "$replays_config" >>$SENTRY_CONFIG_YML + move_replay_recordings=1 + fi +fi + +# Recordings in `/data/files` are stored as ///. +# Regular file blobs are stored as <2 hex>/<4 hex>/<26 hex> files, so they never match a +# directory at the third level and are left alone. +$dcr --no-deps -e MOVE_REPLAY_RECORDINGS=$move_replay_recordings --entrypoint python3 web -c ' +import os +import re +import shutil + +SOURCE = "/data/files" +TARGET = "/data/replays" +NUMBER = re.compile(r"[0-9]+") +REPLAY_ID = re.compile(r"[0-9a-f]{32}") + + +def subdirs(path, pattern): + return [e for e in os.scandir(path) if e.is_dir() and pattern.fullmatch(e.name)] + + +os.makedirs(TARGET, exist_ok=True) + +if os.environ["MOVE_REPLAY_RECORDINGS"] == "1" and os.path.isdir(SOURCE): + moved = 0 + for retention in subdirs(SOURCE, NUMBER): + for project in subdirs(retention.path, NUMBER): + for replay in subdirs(project.path, REPLAY_ID): + destination = os.path.join(TARGET, retention.name, project.name, replay.name) + if os.path.exists(destination): + continue + os.makedirs(os.path.dirname(destination), exist_ok=True) + shutil.move(replay.path, destination) + moved += 1 + try: + os.rmdir(project.path) + except OSError: + pass + print(f"Moved {moved} replay recordings from {SOURCE} to {TARGET}.") +' + +echo "${_endgroup}" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 14578bd19cd..d55f10f9b30 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -113,20 +113,12 @@ filestore.profiles-options: addressing_style: "path" signature_version: "s3v4" -# Session Replay recordings are stored in their own SeaweedFS bucket. The -# bucket's lifecycle policy deletes recordings once they pass the retention -# period, because Sentry does not delete them on its own. -replay.storage.backend: 's3' +# Session Replay recordings are kept in their own directory so the +# `sentry-cleanup` cron job can delete them after the retention period. +# Sentry does not delete them on its own. +replay.storage.backend: 'filesystem' replay.storage.options: - bucket_acl: "private" - default_acl: "private" - access_key: "sentry" - secret_key: "sentry" - bucket_name: "replays" - region_name: "us-east-1" - endpoint_url: "http://seaweedfs:8333" - addressing_style: "path" - signature_version: "s3v4" + location: '/data/replays' symbolicator.enabled: true symbolicator.options: