diff --git a/docs/develop/style/pillar.rst b/docs/develop/style/pillar.rst index d4f2a664..bb816ef3 100644 --- a/docs/develop/style/pillar.rst +++ b/docs/develop/style/pillar.rst @@ -16,7 +16,7 @@ Order the top-level keys in these layers: - Time: ``ntp`` - Monitoring: ``netdata``, ``prometheus`` - Logging: ``rsyslog``, ``logrotate`` - - Scheduling: ``cron``, ``backup`` + - Scheduling: ``cron``, ``backup``, ``sync`` - Application services diff --git a/docs/develop/update/backup.rst b/docs/develop/update/backup.rst index cb8b5192..b10ec35b 100644 --- a/docs/develop/update/backup.rst +++ b/docs/develop/update/backup.rst @@ -20,3 +20,27 @@ Configure backups /home/coalition/public_html/: #. :doc:`Deploy the server<../../deploy/deploy>` + +Sync directories +---------------- + +.. note:: + + This is used only for disaster recovery. This is not a true, immutable backup. + +.. attention:: + + If this uses the same bucket as backups, ensure the :ref:`IAM backup policy` sets *Prefix* and is not scoped to the entire bucket. + +#. Create and configure an :ref:`S3 backup bucket` +#. Configure the :doc:`AWS CLI` +#. In the server's Pillar file, set ``sync.location`` to a bucket and prefix, and ``sync.directories`` to a dict of paths without values. You can annotate what a path must match, for example: + + .. code-block:: yaml + + backup: + location: ocp-registry-backup/file-sync + directories: + /data/storage/exporter: + +#. :doc:`Deploy the server<../../deploy/deploy>` diff --git a/docs/maintain/backup.rst b/docs/maintain/backup.rst index 607c18d2..7d94e537 100644 --- a/docs/maintain/backup.rst +++ b/docs/maintain/backup.rst @@ -153,6 +153,18 @@ Test tar -xzvf example.tar.gz # Review the backup files and compare the names, sizes and timestamps to the actual files. +Sync directories +---------------- + +Backup script + `sync-to-s3.sh `__ copies directories directly into S3. +Servers + - ``registry`` +Test + .. code-block:: bash + + aws s3 sync s3://S3_SYNC_BUCKET/ $DIRECTORY --dryrun --checksum-mode enabled + .. _backups-snapshots: Disk snapshots diff --git a/pillar/registry.sls b/pillar/registry.sls index 973869f8..9736888e 100644 --- a/pillar/registry.sls +++ b/pillar/registry.sls @@ -46,6 +46,17 @@ prometheus: node_exporter: smartmon: True +sync: + location: ocp-registry-backup/file-sync + directories: + # Should match: https://ocdsdeploy.readthedocs.io/en/latest/deploy/servers/data-registry.html#filesystem + /data/storage/exporter: + /data/storage/spoonbill: + /home/collect/scrapyd/dbs: + /home/collect/scrapyd/eggs: + /home/collect/scrapyd/jobs: + /home/collect/scrapyd/logs: + apache: public_access: True modules: diff --git a/salt/backup/init.sls b/salt/aws/backup.sls similarity index 93% rename from salt/backup/init.sls rename to salt/aws/backup.sls index ca31cbfd..e9fc102b 100644 --- a/salt/backup/init.sls +++ b/salt/aws/backup.sls @@ -7,7 +7,7 @@ include: /home/sysadmin-tools/bin/site-backup-to-s3.sh: file.managed: - - source: salt://backup/files/site-backup-to-s3.sh + - source: salt://aws/files/site-backup-to-s3.sh - mode: 750 - require: - file: /home/sysadmin-tools/bin diff --git a/salt/aws/files/aws-settings.local b/salt/aws/files/aws-settings.local index dfd29b0b..46d33302 100644 --- a/salt/aws/files/aws-settings.local +++ b/salt/aws/files/aws-settings.local @@ -14,3 +14,8 @@ S3_SITE_BACKUP_BUCKET="" BACKUP_DIRECTORIES=() # BACKUP_EXCLUDE="--exclude=/home/example/public_html/unwanted/dir" BACKUP_EXCLUDE="" + +# File sync +S3_SYNC_BUCKET="" +# Space deliminated bash array e.x. ("/home/siteone", "/home/sitetwo") +SYNC_DIRECTORIES=() diff --git a/salt/backup/files/site-backup-to-s3.sh b/salt/aws/files/site-backup-to-s3.sh similarity index 100% rename from salt/backup/files/site-backup-to-s3.sh rename to salt/aws/files/site-backup-to-s3.sh diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh new file mode 100644 index 00000000..80f5573b --- /dev/null +++ b/salt/aws/files/sync-to-s3.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Backup multiple directories and upload to AWS S3 + +set -euo pipefail + +# shellcheck disable=SC1091 +. /home/sysadmin-tools/aws-settings.local + +export AWS_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY +export AWS_DEFAULT_REGION + +if [ "$LOGNAME" != "root" ]; then + echo "ERROR: Execution of $0 stopped as not run by user root!" + exit 2 +fi + +if [ ! -x "$AWS_CLI" ]; then + echo "Error: The aws executable is not installed" + exit 3 +fi + +if [ -z "$SYNC_DIRECTORIES" ]; then + echo "Error: SYNC_DIRECTORIES isn't set or is empty" + exit 4 +fi + +for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do + SAFENAME="${DIRECTORY/#\//}" + SAFENAME="${SAFENAME/%\//}" + + $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete +done diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls new file mode 100644 index 00000000..be5a4330 --- /dev/null +++ b/salt/aws/sync.sls @@ -0,0 +1,31 @@ +{% from 'lib.sls' import set_config %} + +include: + - aws + +{{ set_config('aws-settings.local', 'S3_SYNC_BUCKET', pillar.sync.location) }} + +/home/sysadmin-tools/bin/sync-to-s3.sh: + file.managed: + - source: salt://aws/files/sync-to-s3.sh + - mode: 750 + - require: + - file: /home/sysadmin-tools/bin + +/etc/cron.d/sync_to_s3: + file.managed: + - contents: | + MAILTO=root + 15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh + - require: + - file: /home/sysadmin-tools/bin/sync-to-s3.sh + +set SYNC_DIRECTORIES setting: + file.keyvalue: + - name: /home/sysadmin-tools/aws-settings.local + - key: SYNC_DIRECTORIES + - value: '( "{{ pillar.sync.directories|join('" "') }}" )' + - append_if_not_found: True + - require: + - file: /home/sysadmin-tools/bin + - sls: aws diff --git a/salt/top.sls b/salt/top.sls index 3c1c51b6..bcd1f424 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -85,7 +85,10 @@ base: - nginx.cloudflare 'I@backup:*': - - backup + - aws.backup + + 'I@sync:*': + - aws.sync 'I@cron:*': - cron