From bfecb9b5b83fe57c224ce36469fe40379487886c Mon Sep 17 00:00:00 2001 From: RobHooper Date: Tue, 7 Apr 2026 11:36:47 +0100 Subject: [PATCH 1/8] feat: add sync script --- pillar/registry.sls | 10 +++++++++ salt/aws/files/aws-settings.local | 5 +++++ salt/sync/files/sync-to-s3.sh | 34 +++++++++++++++++++++++++++++++ salt/sync/init.sls | 31 ++++++++++++++++++++++++++++ salt/top.sls | 3 +++ 5 files changed, 83 insertions(+) create mode 100644 salt/sync/files/sync-to-s3.sh create mode 100644 salt/sync/init.sls diff --git a/pillar/registry.sls b/pillar/registry.sls index 973869f8..729e1309 100644 --- a/pillar/registry.sls +++ b/pillar/registry.sls @@ -46,6 +46,16 @@ prometheus: node_exporter: smartmon: True +sync: + location: ocp-registry-backup/file-sync + directories: + /data/storage/exporter: + /data/storage/spoonbill: + /home/collect/scrapyd/dbs: + /home/collect/scrapyd/eggs: + /home/collect/scrapyd/jobs: + /home/collect/scrapyd/logs: + apache: public_access: True modules: diff --git a/salt/aws/files/aws-settings.local b/salt/aws/files/aws-settings.local index dfd29b0b..46d33302 100644 --- a/salt/aws/files/aws-settings.local +++ b/salt/aws/files/aws-settings.local @@ -14,3 +14,8 @@ S3_SITE_BACKUP_BUCKET="" BACKUP_DIRECTORIES=() # BACKUP_EXCLUDE="--exclude=/home/example/public_html/unwanted/dir" BACKUP_EXCLUDE="" + +# File sync +S3_SYNC_BUCKET="" +# Space deliminated bash array e.x. ("/home/siteone", "/home/sitetwo") +SYNC_DIRECTORIES=() diff --git a/salt/sync/files/sync-to-s3.sh b/salt/sync/files/sync-to-s3.sh new file mode 100644 index 00000000..fc0a7378 --- /dev/null +++ b/salt/sync/files/sync-to-s3.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Backup multiple directories and upload to AWS S3 + +set -euo pipefail + +# shellcheck disable=SC1091 +. /home/sysadmin-tools/aws-settings.local + +export AWS_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY +export AWS_DEFAULT_REGION + +if [ "$LOGNAME" != "root" ]; then + echo "ERROR: Execution of $0 stopped as not run by user root!" + exit 2 +fi + +if [ ! -x "$AWS_CLI" ]; then + echo "Error: The aws executable is not installed" + exit 3 +fi + +if [ -z "$SYNC_DIRECTORIES" ]; then + echo "Error: SYNC_DIRECTORIES isn't set or is empty" + exit 4 +fi + +for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do + SAFENAME="${DIRECTORY/#\//}" + SAFENAME="${SAFENAME/%\//}" + + $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$BASENAME/" --only-show-errors --delete +done diff --git a/salt/sync/init.sls b/salt/sync/init.sls new file mode 100644 index 00000000..8cfe5aa6 --- /dev/null +++ b/salt/sync/init.sls @@ -0,0 +1,31 @@ +{% from 'lib.sls' import set_config %} + +include: + - aws + +{{ set_config('aws-settings.local', 'S3_SYNC_BUCKET', pillar.sync.location) }} + +/home/sysadmin-tools/bin/sync-to-s3.sh: + file.managed: + - source: salt://sync/files/sync-to-s3.sh + - mode: 750 + - require: + - file: /home/sysadmin-tools/bin + +/etc/cron.d/sync_to_s3: + file.managed: + - contents: | + MAILTO=root + 15 04 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh + - require: + - file: /home/sysadmin-tools/bin/sync-to-s3.sh + +set SYNC_DIRECTORIES setting: + file.keyvalue: + - name: /home/sysadmin-tools/aws-settings.local + - key: SYNC_DIRECTORIES + - value: '( "{{ pillar.sync.directories|join('" "') }}" )' + - append_if_not_found: True + - require: + - file: /home/sysadmin-tools/bin + - sls: aws diff --git a/salt/top.sls b/salt/top.sls index 3c1c51b6..423f4923 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -87,6 +87,9 @@ base: 'I@backup:*': - backup + 'I@sync:*': + - sync + 'I@cron:*': - cron From 8deb2fdd5731b92bb3fa16adae13f486f6eb51c7 Mon Sep 17 00:00:00 2001 From: RobHooper Date: Tue, 7 Apr 2026 11:45:34 +0100 Subject: [PATCH 2/8] docs: note directory requirement --- pillar/registry.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/pillar/registry.sls b/pillar/registry.sls index 729e1309..9736888e 100644 --- a/pillar/registry.sls +++ b/pillar/registry.sls @@ -49,6 +49,7 @@ prometheus: sync: location: ocp-registry-backup/file-sync directories: + # Should match: https://ocdsdeploy.readthedocs.io/en/latest/deploy/servers/data-registry.html#filesystem /data/storage/exporter: /data/storage/spoonbill: /home/collect/scrapyd/dbs: From 35615338323691b4c328a96becfbaacf6e442366 Mon Sep 17 00:00:00 2001 From: RobHooper Date: Tue, 7 Apr 2026 11:47:08 +0100 Subject: [PATCH 3/8] fix: correct variable --- salt/sync/files/sync-to-s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sync/files/sync-to-s3.sh b/salt/sync/files/sync-to-s3.sh index fc0a7378..80f5573b 100644 --- a/salt/sync/files/sync-to-s3.sh +++ b/salt/sync/files/sync-to-s3.sh @@ -30,5 +30,5 @@ for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do SAFENAME="${DIRECTORY/#\//}" SAFENAME="${SAFENAME/%\//}" - $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$BASENAME/" --only-show-errors --delete + $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete done From 12d10ccbdb037b781f0c07382cf145728ed95c4b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 7 Apr 2026 12:58:54 -0400 Subject: [PATCH 4/8] chore: Move backup and sync into aws/ --- docs/develop/style/pillar.rst | 2 +- salt/{backup/init.sls => aws/backup.sls} | 2 +- salt/{backup => aws}/files/site-backup-to-s3.sh | 0 salt/{sync => aws}/files/sync-to-s3.sh | 0 salt/{sync/init.sls => aws/sync.sls} | 2 +- salt/top.sls | 4 ++-- 6 files changed, 5 insertions(+), 5 deletions(-) rename salt/{backup/init.sls => aws/backup.sls} (93%) rename salt/{backup => aws}/files/site-backup-to-s3.sh (100%) rename salt/{sync => aws}/files/sync-to-s3.sh (100%) rename salt/{sync/init.sls => aws/sync.sls} (93%) diff --git a/docs/develop/style/pillar.rst b/docs/develop/style/pillar.rst index d4f2a664..bb816ef3 100644 --- a/docs/develop/style/pillar.rst +++ b/docs/develop/style/pillar.rst @@ -16,7 +16,7 @@ Order the top-level keys in these layers: - Time: ``ntp`` - Monitoring: ``netdata``, ``prometheus`` - Logging: ``rsyslog``, ``logrotate`` - - Scheduling: ``cron``, ``backup`` + - Scheduling: ``cron``, ``backup``, ``sync`` - Application services diff --git a/salt/backup/init.sls b/salt/aws/backup.sls similarity index 93% rename from salt/backup/init.sls rename to salt/aws/backup.sls index ca31cbfd..79051b58 100644 --- a/salt/backup/init.sls +++ b/salt/aws/backup.sls @@ -7,7 +7,7 @@ include: /home/sysadmin-tools/bin/site-backup-to-s3.sh: file.managed: - - source: salt://backup/files/site-backup-to-s3.sh + - source: salt://aws/backup/files/site-backup-to-s3.sh - mode: 750 - require: - file: /home/sysadmin-tools/bin diff --git a/salt/backup/files/site-backup-to-s3.sh b/salt/aws/files/site-backup-to-s3.sh similarity index 100% rename from salt/backup/files/site-backup-to-s3.sh rename to salt/aws/files/site-backup-to-s3.sh diff --git a/salt/sync/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh similarity index 100% rename from salt/sync/files/sync-to-s3.sh rename to salt/aws/files/sync-to-s3.sh diff --git a/salt/sync/init.sls b/salt/aws/sync.sls similarity index 93% rename from salt/sync/init.sls rename to salt/aws/sync.sls index 8cfe5aa6..a31e19ac 100644 --- a/salt/sync/init.sls +++ b/salt/aws/sync.sls @@ -7,7 +7,7 @@ include: /home/sysadmin-tools/bin/sync-to-s3.sh: file.managed: - - source: salt://sync/files/sync-to-s3.sh + - source: salt://aws/sync/files/sync-to-s3.sh - mode: 750 - require: - file: /home/sysadmin-tools/bin diff --git a/salt/top.sls b/salt/top.sls index 423f4923..bcd1f424 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -85,10 +85,10 @@ base: - nginx.cloudflare 'I@backup:*': - - backup + - aws.backup 'I@sync:*': - - sync + - aws.sync 'I@cron:*': - cron From e466271b4be2bbab57a366f4e50f1ccb203a1355 Mon Sep 17 00:00:00 2001 From: RobHooper Date: Wed, 8 Apr 2026 16:13:30 +0100 Subject: [PATCH 5/8] fix: new script paths --- salt/aws/backup.sls | 2 +- salt/aws/sync.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/aws/backup.sls b/salt/aws/backup.sls index 79051b58..e9fc102b 100644 --- a/salt/aws/backup.sls +++ b/salt/aws/backup.sls @@ -7,7 +7,7 @@ include: /home/sysadmin-tools/bin/site-backup-to-s3.sh: file.managed: - - source: salt://aws/backup/files/site-backup-to-s3.sh + - source: salt://aws/files/site-backup-to-s3.sh - mode: 750 - require: - file: /home/sysadmin-tools/bin diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls index a31e19ac..757121f9 100644 --- a/salt/aws/sync.sls +++ b/salt/aws/sync.sls @@ -7,7 +7,7 @@ include: /home/sysadmin-tools/bin/sync-to-s3.sh: file.managed: - - source: salt://aws/sync/files/sync-to-s3.sh + - source: salt://aws/files/sync-to-s3.sh - mode: 750 - require: - file: /home/sysadmin-tools/bin From ebec02b4ff74f83e031b9302872860efccc2bbd9 Mon Sep 17 00:00:00 2001 From: RobHooper Date: Wed, 8 Apr 2026 17:20:24 +0100 Subject: [PATCH 6/8] feat: run sync twice daily --- salt/aws/sync.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls index 757121f9..be5a4330 100644 --- a/salt/aws/sync.sls +++ b/salt/aws/sync.sls @@ -16,7 +16,7 @@ include: file.managed: - contents: | MAILTO=root - 15 04 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh + 15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh - require: - file: /home/sysadmin-tools/bin/sync-to-s3.sh From 527bef31b82cc83f5d6b0a632493579a2db75149 Mon Sep 17 00:00:00 2001 From: RobHooper Date: Wed, 8 Apr 2026 17:20:48 +0100 Subject: [PATCH 7/8] docs: documenting new sync feature --- docs/develop/update/backup.rst | 20 ++++++++++++++++++++ docs/maintain/backup.rst | 12 ++++++++++++ 2 files changed, 32 insertions(+) diff --git a/docs/develop/update/backup.rst b/docs/develop/update/backup.rst index cb8b5192..60450ca3 100644 --- a/docs/develop/update/backup.rst +++ b/docs/develop/update/backup.rst @@ -20,3 +20,23 @@ Configure backups /home/coalition/public_html/: #. :doc:`Deploy the server<../../deploy/deploy>` + +Sync Directory +-------------- + +.. note:: + + Sync is used for Disaster Recovery, this is not a true, immutable backup. + +#. Create and configure an :ref:`S3 backup bucket` +#. Configure the :doc:`AWS CLI` +#. In the server's Pillar file, set ``sync.location`` to a bucket and prefix, and ``sync.directories`` to a dict of paths without values. You can annotate what a path must match, for example: + + .. code-block:: yaml + + backup: + location: ocp-registry-backup/file-sync + directories: + /data/storage/exporter: + +#. :doc:`Deploy the server<../../deploy/deploy>` diff --git a/docs/maintain/backup.rst b/docs/maintain/backup.rst index 607c18d2..ff4ec5df 100644 --- a/docs/maintain/backup.rst +++ b/docs/maintain/backup.rst @@ -153,6 +153,18 @@ Test tar -xzvf example.tar.gz # Review the backup files and compare the names, sizes and timestamps to the actual files. +Sync directory +-------------- + +Backup script + `sync-to-s3.sh `__ copies directories directly into S3. +Servers + - ``registry`` +Test + .. code-block:: bash + + aws s3 sync s3://S3_SYNC_BUCKET/ $DIRECTORY --dryrun --checksum-mode enabled + .. _backups-snapshots: Disk snapshots From a460ac906b1c217a576d4174b623ce37b95c72c0 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 8 Apr 2026 15:55:52 -0400 Subject: [PATCH 8/8] docs(backup): Add attention admonition about lifecycle rules --- docs/develop/update/backup.rst | 10 +++++++--- docs/maintain/backup.rst | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/develop/update/backup.rst b/docs/develop/update/backup.rst index 60450ca3..b10ec35b 100644 --- a/docs/develop/update/backup.rst +++ b/docs/develop/update/backup.rst @@ -21,12 +21,16 @@ Configure backups #. :doc:`Deploy the server<../../deploy/deploy>` -Sync Directory --------------- +Sync directories +---------------- .. note:: - Sync is used for Disaster Recovery, this is not a true, immutable backup. + This is used only for disaster recovery. This is not a true, immutable backup. + +.. attention:: + + If this uses the same bucket as backups, ensure the :ref:`IAM backup policy` sets *Prefix* and is not scoped to the entire bucket. #. Create and configure an :ref:`S3 backup bucket` #. Configure the :doc:`AWS CLI` diff --git a/docs/maintain/backup.rst b/docs/maintain/backup.rst index ff4ec5df..7d94e537 100644 --- a/docs/maintain/backup.rst +++ b/docs/maintain/backup.rst @@ -153,8 +153,8 @@ Test tar -xzvf example.tar.gz # Review the backup files and compare the names, sizes and timestamps to the actual files. -Sync directory --------------- +Sync directories +---------------- Backup script `sync-to-s3.sh `__ copies directories directly into S3.