feat: add sync script

docs/develop/style/pillar.rst

@@ -16,7 +16,7 @@ Order the top-level keys in these layers:
    -  Time: ``ntp``
    -  Monitoring: ``netdata``, ``prometheus``
    -  Logging: ``rsyslog``, ``logrotate``
-   -  Scheduling: ``cron``, ``backup``
+   -  Scheduling: ``cron``, ``backup``, ``sync``
 
 -  Application services
 

docs/develop/update/backup.rst

@@ -20,3 +20,27 @@ Configure backups
           /home/coalition/public_html/:
 
 #. :doc:`Deploy the server<../../deploy/deploy>`
+
+Sync directories
+----------------
+
+.. note::
+
+   This is used only for disaster recovery. This is not a true, immutable backup.
+
+.. attention::
+
+   If this uses the same bucket as backups, ensure the :ref:`IAM backup policy<aws-iam-backup-policy>` sets *Prefix* and is not scoped to the entire bucket.
+
+#. Create and configure an :ref:`S3 backup bucket<aws-s3-bucket>`
+#. Configure the :doc:`AWS CLI<awscli>`
+#. In the server's Pillar file, set ``sync.location`` to a bucket and prefix, and ``sync.directories`` to a dict of paths without values. You can annotate what a path must match, for example:
+
+   .. code-block:: yaml
+
+      backup:
+        location: ocp-registry-backup/file-sync
+        directories:
+          /data/storage/exporter:
+
+#. :doc:`Deploy the server<../../deploy/deploy>`

docs/maintain/backup.rst

@@ -153,6 +153,18 @@ Test
      tar -xzvf example.tar.gz
      # Review the backup files and compare the names, sizes and timestamps to the actual files.
 
+Sync directories
+----------------
+
+Backup script
+  `sync-to-s3.sh <https://github.com/open-contracting/deploy/blob/main/salt/aws/files/sync-to-s3.sh>`__ copies directories directly into S3.
+Servers
+  -  ``registry``
+Test
+  .. code-block:: bash
+
+     aws s3 sync s3://S3_SYNC_BUCKET/ $DIRECTORY --dryrun --checksum-mode enabled
+
 .. _backups-snapshots:
 
 Disk snapshots

pillar/registry.sls

@@ -46,6 +46,17 @@ prometheus:
   node_exporter:
     smartmon: True
 
+sync:
+  location: ocp-registry-backup/file-sync
+  directories:
+    # Should match: https://ocdsdeploy.readthedocs.io/en/latest/deploy/servers/data-registry.html#filesystem
+    /data/storage/exporter:
+    /data/storage/spoonbill:
+    /home/collect/scrapyd/dbs:
+    /home/collect/scrapyd/eggs:
+    /home/collect/scrapyd/jobs:
+    /home/collect/scrapyd/logs:
+
 apache:
   public_access: True
   modules:

salt/backup/init.sls → salt/aws/backup.sls

@@ -7,7 +7,7 @@ include:
 
 /home/sysadmin-tools/bin/site-backup-to-s3.sh:
   file.managed:
-    - source: salt://backup/files/site-backup-to-s3.sh
+    - source: salt://aws/files/site-backup-to-s3.sh
     - mode: 750
     - require:
       - file: /home/sysadmin-tools/bin

salt/aws/files/aws-settings.local

@@ -14,3 +14,8 @@ S3_SITE_BACKUP_BUCKET=""
 BACKUP_DIRECTORIES=()
 # BACKUP_EXCLUDE="--exclude=/home/example/public_html/unwanted/dir"
 BACKUP_EXCLUDE=""
+
+# File sync
+S3_SYNC_BUCKET=""
+# Space deliminated bash array e.x. ("/home/siteone", "/home/sitetwo")
+SYNC_DIRECTORIES=()

salt/aws/files/sync-to-s3.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Backup multiple directories and upload to AWS S3
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+. /home/sysadmin-tools/aws-settings.local
+
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+export AWS_DEFAULT_REGION
+
+if [ "$LOGNAME" != "root" ]; then
+    echo "ERROR: Execution of $0 stopped as not run by user root!"
+    exit 2
+fi
+
+if [ ! -x "$AWS_CLI" ]; then
+    echo "Error: The aws executable is not installed"
+    exit 3
+fi
+
+if [ -z "$SYNC_DIRECTORIES" ]; then
+    echo "Error: SYNC_DIRECTORIES isn't set or is empty"
+    exit 4
+fi
+
+for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do
+    SAFENAME="${DIRECTORY/#\//}"
+    SAFENAME="${SAFENAME/%\//}"
+
+    $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete
+done

salt/aws/sync.sls

@@ -0,0 +1,31 @@
+{% from 'lib.sls' import set_config %}
+
+include:
+  - aws
+
+{{ set_config('aws-settings.local', 'S3_SYNC_BUCKET', pillar.sync.location) }}
+
+/home/sysadmin-tools/bin/sync-to-s3.sh:
+  file.managed:
+    - source: salt://aws/files/sync-to-s3.sh
+    - mode: 750
+    - require:
+      - file: /home/sysadmin-tools/bin
+
+/etc/cron.d/sync_to_s3:
+  file.managed:
+    - contents: |
+        MAILTO=root
+        15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh
+    - require:
+      - file: /home/sysadmin-tools/bin/sync-to-s3.sh
+
+set SYNC_DIRECTORIES setting:
+  file.keyvalue:
+    - name: /home/sysadmin-tools/aws-settings.local
+    - key: SYNC_DIRECTORIES
+    - value: '( "{{ pillar.sync.directories|join('" "') }}" )'
+    - append_if_not_found: True
+    - require:
+      - file: /home/sysadmin-tools/bin
+      - sls: aws

salt/top.sls

@@ -85,7 +85,10 @@ base:
     - nginx.cloudflare
 
   'I@backup:*':
-    - backup
+    - aws.backup
+
+  'I@sync:*':
+    - aws.sync
 
   'I@cron:*':
     - cron