diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index 31efa39..b9718f0 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -106,6 +106,8 @@ jobs: healthcheck: no-healthcheck - database: postgresql healthcheck: no-healthcheck + - database: postgresql + healthcheck: failing-healthcheck steps: - name: Checkout diff --git a/docs/backup-and-restore.fa.md b/docs/backup-and-restore.fa.md index 4538218..4e0201c 100644 --- a/docs/backup-and-restore.fa.md +++ b/docs/backup-and-restore.fa.md @@ -142,7 +142,10 @@ Xray داخل پوشه داده دست‌نخورده می‌مانند و کپ اگر `--fresh` پس از آماده‌سازی نصب شکست خورد، پنل اجرا نمی‌شود و داده‌ها برای بررسی باقی می‌مانند. پس از اصلاح علت در لاگ، اگر Compose و تنظیمات مقصد ساخته -شده‌اند، با restore معمولی روی همان نصب دوباره امتحان کنید. +شده‌اند، همان فرمان `--fresh` را دوباره اجرا کنید: تلاش قبلی به‌صورت restore معمولی +روی همان نصب ادامه پیدا می‌کند (بکاپ دیگری پذیرفته نمی‌شود) و از پوشه‌های برنامه و داده +نیمه‌کاره آن کپی پیش از جایگزینی نمی‌گیرد. restore معمولی با همان بکاپ هم کار می‌کند، +اما پیش از آن از این پوشه‌ها کپی می‌گیرد. ## بکاپ قدیمی بدون شناسنامه diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 567cef8..7f2df68 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -165,8 +165,11 @@ restores can finish earlier databases before a later one fails. Take a separate current backup before replacing an existing installation. Fresh recovery leaves application services stopped on failure; it keeps provisioned storage for diagnosis rather than destroying it or claiming automatic rollback. Once a failed fresh -recovery has provisioned Compose/configuration, diagnose the log and retry with -ordinary restore against that installation. +recovery has provisioned Compose/configuration, diagnose the log, fix the cause +and run the same `--fresh` command again: it continues the earlier attempt as an +ordinary restore of that installation (it refuses a different backup) and makes no +safety copies of its half-provisioned app and data directories. An ordinary restore +of the same backup also works, but copies those directories first. ## Old backups without recovery metadata diff --git a/lib/pasarguard-backup.sh b/lib/pasarguard-backup.sh index 6ba5850..e61fbdd 100644 --- a/lib/pasarguard-backup.sh +++ b/lib/pasarguard-backup.sh @@ -1877,6 +1877,7 @@ backup_command() { --exclude 'backup-files.sha256' --exclude '.pasarguard-recovery-compose.json' --exclude '.pasarguard-destination-compose.yml' + --exclude '.pasarguard-fresh-restore' --exclude 'pasarguard_ts_compat.*' --exclude '*_combined.zip' --exclude 'pasarguard_env_cleaned' diff --git a/lib/pasarguard-restore.sh b/lib/pasarguard-restore.sh index 1efa88e..9f8fd24 100644 --- a/lib/pasarguard-restore.sh +++ b/lib/pasarguard-restore.sh @@ -894,6 +894,10 @@ wait_for_recovery_database() { return 1 } +# Left in APP_DIR by a --fresh run that provisioned the installation but did not +# finish; holds the SHA256 of that backup's checksum inventory. +FRESH_RESTORE_MARKER=".pasarguard-fresh-restore" + # Prepare a fresh deployment using the source images. Refuse existing app/data, # containers and database storage; this mode is never an in-place downgrade. # The archived Compose file is installed unchanged. Each recorded source image @@ -1069,6 +1073,10 @@ prepare_fresh_restore() { mkdir -p "$APP_DIR" "$DATA_DIR" || return 1 install -m 600 "$stage/.env" "$ENV_FILE" || return 1 install -m 600 "$recovery_compose" "$COMPOSE_FILE" || return 1 + # From here on the directories are no longer empty. Record which backup + # provisioned them, so running the same --fresh command again continues + # this attempt; a successful restore removes the marker. + (umask 077 && sha256sum <"$stage/backup-files.sha256" | awk '{print $1}' >"$APP_DIR/$FRESH_RESTORE_MARKER") || return 1 if [ -n "$db_service" ]; then colorized_echo blue "Starting only the database; panel migrations stay stopped until the import finishes." $COMPOSE -f "$COMPOSE_FILE" -p "$APP_NAME" up -d --no-deps "$db_service" >>"$log" 2>&1 || return 1 @@ -1128,6 +1136,17 @@ restore_command() { fi colorized_echo blue "Starting restore process..." + # A previous --fresh run provisioned this installation and then failed. + # The same command continues it as an ordinary restore (checked below to be + # the same backup) instead of refusing the now non-empty directories. + local fresh_resume=false + if [ "$fresh_restore" = true ] && [ "$check_only" = false ] && \ + [ -f "$APP_DIR/$FRESH_RESTORE_MARKER" ] && [ ! -L "$APP_DIR/$FRESH_RESTORE_MARKER" ]; then + colorized_echo yellow "A previous --fresh run stopped after provisioning $APP_DIR. Continuing it as an ordinary restore; the database it created will be overwritten." + fresh_restore=false + fresh_resume=true + fi + if [ "$fresh_restore" = false ] && [ "$check_only" = false ]; then if ! is_pasarguard_installed || [ ! -f "$COMPOSE_FILE" ]; then colorized_echo red "No installation found. To recover onto an empty server, use: pasarguard restore --fresh /path/to/backup.zip" @@ -1230,7 +1249,7 @@ restore_command() { # application services if they were shut down, and terminate with the error code. cleanup_and_exit_restore_error() { local code="${1:-1}" - if [ "$services_stopped" = true ] && [ "$fresh_restore" = false ]; then + if [ "$services_stopped" = true ] && [ "$fresh_restore" = false ] && [ "$fresh_resume" = false ]; then if [[ "$db_type" == "sqlite" ]]; then up_pasarguard || echo "Failed to restart pasarguard after SQLite restore failure" >>"$log_file" else @@ -1548,6 +1567,17 @@ restore_command() { cleanup_and_exit_restore_error 1 fi if ! print_backup_runtime "$temp_restore_dir"; then cleanup_and_exit_restore_error 1; fi + if [ "$fresh_resume" = true ]; then + local previous_backup="" this_backup="" + previous_backup=$(cat "$APP_DIR/$FRESH_RESTORE_MARKER" 2>/dev/null) || previous_backup="" + if [ -f "$temp_restore_dir/backup-files.sha256" ]; then + this_backup=$(sha256sum <"$temp_restore_dir/backup-files.sha256" | awk '{print $1}') + fi + if [ -z "$this_backup" ] || [ "$previous_backup" != "$this_backup" ]; then + colorized_echo red "The unfinished --fresh run in $APP_DIR used a different backup. Retry with that backup, or restore this one without --fresh." + cleanup_and_exit_restore_error 1 + fi + fi # Load environment variables from extracted .env colorized_echo blue "Loading configuration from backup..." @@ -2249,7 +2279,8 @@ restore_command() { colorized_echo red "The data directory exclude list is missing (mismatched script libraries). Refusing to sync the data directory." cleanup_and_exit_restore_error 1 fi - if [ "$fresh_restore" = false ] && [ "$(ls -A "$DATA_DIR" 2>/dev/null)" ]; then + # A continued --fresh run (fresh_resume) has only its own half-provisioned files to copy. + if [ "$fresh_restore" = false ] && [ "$fresh_resume" = false ] && [ "$(ls -A "$DATA_DIR" 2>/dev/null)" ]; then colorized_echo blue "Backing up current data directory before restore..." if ! save_data_dir_safety_copy "$DATA_DIR.backup.$(date +%Y%m%d%H%M%S)" "$log_file"; then colorized_echo red "Failed to save the current data directory before replacement." @@ -2303,7 +2334,7 @@ restore_command() { install_package rsync fi mkdir -p "$APP_DIR" - if [ "$fresh_restore" = false ] && [ "$(ls -A "$APP_DIR" 2>/dev/null)" ]; then + if [ "$fresh_restore" = false ] && [ "$fresh_resume" = false ] && [ "$(ls -A "$APP_DIR" 2>/dev/null)" ]; then colorized_echo blue "Backing up current app directory before restore..." if ! rsync -a --exclude backup "$APP_DIR/" "$APP_DIR.backup.$(date +%Y%m%d%H%M%S)/" 2>>"$log_file"; then colorized_echo red "Failed to save the current application files before replacement." @@ -2313,7 +2344,7 @@ restore_command() { if ! rsync -av --exclude 'pasarguard_data' --exclude 'db_backup.sql' --exclude 'db_backup.sqlite' \ --exclude 'db_backup.timescaledb-version' --exclude 'pg_dump' \ --exclude 'backup-runtime.tsv' --exclude 'backup-files.sha256' --exclude '.pasarguard-recovery-compose.json' \ - --exclude '.pasarguard-destination-compose.yml' --exclude 'pasarguard_ts_compat.*' \ + --exclude '.pasarguard-destination-compose.yml' --exclude "$FRESH_RESTORE_MARKER" --exclude 'pasarguard_ts_compat.*' \ --exclude '*_combined.zip' --exclude 'pasarguard_env_cleaned' \ --exclude 'pasarguard_restore_error.log' --exclude "$sqlite_basename" \ "$temp_restore_dir/" "$APP_DIR/" >>"$log_file" 2>&1; then @@ -2370,6 +2401,7 @@ restore_command() { fi harden_secret_file "$ENV_FILE" harden_secret_file "$COMPOSE_FILE" + rm -f "$APP_DIR/$FRESH_RESTORE_MARKER" rm -rf "$temp_restore_dir" colorized_echo green "Restore completed successfully!" colorized_echo green "PasarGuard services have been started. Check panel login, subscriptions and node connections before upgrading." diff --git a/pasarguard.sh b/pasarguard.sh index 9f98098..385197a 100755 --- a/pasarguard.sh +++ b/pasarguard.sh @@ -1141,6 +1141,9 @@ install_pasarguard() { mkdir -p "$DATA_DIR" mkdir -p "$APP_DIR" + # This installation replaces anything an unfinished `restore --fresh` left + # here; its retry marker must not turn a later --fresh into a restore over it. + rm -f "$APP_DIR/${FRESH_RESTORE_MARKER:-.pasarguard-fresh-restore}" colorized_echo blue "Fetching .env file" # Pre-create .env as 0600 (and tighten any pre-existing copy) so the DB, diff --git a/tests/fresh_recovery_roundtrip.sh b/tests/fresh_recovery_roundtrip.sh index 77818f0..654be70 100644 --- a/tests/fresh_recovery_roundtrip.sh +++ b/tests/fresh_recovery_roundtrip.sh @@ -4,9 +4,12 @@ # Back up a source installation, remove it completely, recover it with --fresh, # then check the data, the installed Compose file and the images that run. # -# Usage: bash tests/fresh_recovery_roundtrip.sh ENGINE [healthcheck|no-healthcheck] +# Usage: bash tests/fresh_recovery_roundtrip.sh ENGINE [healthcheck|no-healthcheck|failing-healthcheck] # ENGINE: sqlite | mysql | mariadb | postgresql | timescaledb # no-healthcheck: the archived database service has no Compose healthcheck. +# failing-healthcheck: the archived healthcheck always fails, so the first +# --fresh stops after provisioning; the same command is then run again and +# must finish the recovery. # # Needs root (restore --fresh requires it), Docker with Compose v2, sqlite3, jq, # rsync, zip and unzip. Everything it creates is its own: the Compose project @@ -26,9 +29,13 @@ ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" engine="${1:-}" healthcheck="${2:-healthcheck}" case "$healthcheck" in - healthcheck | no-healthcheck) ;; - *) echo "usage: $0 ENGINE [healthcheck|no-healthcheck]" >&2; exit 2 ;; + healthcheck | no-healthcheck | failing-healthcheck) ;; + *) echo "usage: $0 ENGINE [healthcheck|no-healthcheck|failing-healthcheck]" >&2; exit 2 ;; esac +if [ "${1:-}" = sqlite ] && [ "$healthcheck" = failing-healthcheck ]; then + echo "failing-healthcheck needs a database service" >&2 + exit 2 +fi app_image="${FRESH_APP_IMAGE:-alpine:3.20}" panel_ref="pasarguard-recovery-fixture/panel:latest" @@ -39,7 +46,7 @@ case "$engine" in mariadb) db_image="${FRESH_DB_IMAGE:-mariadb:11.4}"; target=/var/lib/mysql; dbservice=mariadb ;; postgresql) db_image="${FRESH_DB_IMAGE:-postgres:16}"; target=/var/lib/postgresql/data; dbservice=postgresql ;; timescaledb) db_image="${FRESH_DB_IMAGE:-timescale/timescaledb:2.27.2-pg17}"; target=/var/lib/postgresql/data; dbservice=timescaledb ;; - *) echo "usage: $0 ENGINE [healthcheck|no-healthcheck]" >&2; exit 2 ;; + *) echo "usage: $0 ENGINE [healthcheck|no-healthcheck|failing-healthcheck]" >&2; exit 2 ;; esac root=$(mktemp -d "${TMPDIR:-/tmp}/pasarguard-fresh-recovery.XXXXXX") @@ -83,7 +90,7 @@ DB_NAME=appdb SQLALCHEMY_DATABASE_URL="$url" EOF -# Write the Compose file; $1 is "healthcheck" or "no-healthcheck" for the database. +# Write the Compose file; $1 is the database healthcheck mode (see usage). write_compose() { cat >"$COMPOSE_FILE" <>"$COMPOSE_FILE" + return 0 + ;; + esac case "$engine" in mysql) printf ' healthcheck:\n test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root --password=fixture-password"]\n' ;; mariadb) printf ' healthcheck:\n test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]\n' ;; @@ -129,6 +142,18 @@ run_sql() { esac } +# Wait up to 120 s until the database answers over TCP inside its container, +# whatever its healthcheck says. +wait_until_database_answers() { + local cid="$1" port=5432 attempt + case "$engine" in mysql | mariadb) port=3306 ;; esac + for ((attempt = 0; attempt < 60; attempt++)); do + if recovery_database_responds "$cid" "$engine" "$port" >/dev/null 2>&1; then return 0; fi + sleep 2 + done + return 1 +} + # The source always starts with a healthcheck so the fixture data can be written # once the database is ready; the archived Compose file decides what --fresh sees. write_compose healthcheck @@ -162,7 +187,18 @@ fi [ ! -e "$APP_DIR" ] || fail "--check created the application directory" started=$(date +%s) +if [ "$healthcheck" = failing-healthcheck ]; then + if (restore_command "$root/recovery.zip" --fresh --yes); then + fail "--fresh succeeded although the database never became healthy" + fi + [ -f "$APP_DIR/.pasarguard-fresh-restore" ] || fail "no retry marker after the failed --fresh" + # As an administrator would after reading the log: let the database finish + # starting, then run the same command again. + wait_until_database_answers "$(dc ps -q "$dbservice")" || fail "database did not answer after the failed --fresh" + echo "first --fresh stopped as expected; running the same command again" +fi (restore_command "$root/recovery.zip" --fresh --yes) +[ ! -e "$APP_DIR/.pasarguard-fresh-restore" ] || fail "retry marker left after a successful restore" echo "fresh recovery took $(($(date +%s) - started))s" [ "$(cat "$DATA_DIR/sentinel.txt")" = original-state ] || fail "data file not restored" @@ -179,10 +215,10 @@ done <"$root/runtime.tsv" if [ "$engine" != sqlite ]; then dc restart "$dbservice" - dc up -d --wait --wait-timeout 180 "$dbservice" + [ "$healthcheck" = failing-healthcheck ] || dc up -d --wait --wait-timeout 180 "$dbservice" cid=$(dc ps -q "$dbservice") - if [ "$healthcheck" = no-healthcheck ]; then - wait_for_recovery_database "$cid" "$engine" "" "$root/wait.log" || fail "database did not answer after restart" + if [ "$healthcheck" != healthcheck ]; then + wait_until_database_answers "$cid" || fail "database did not answer after restart" fi fi [ "$(run_sql "$cid" 'SELECT value FROM ci_recovery;')" = 42 ] || fail "database row not restored" diff --git a/tests/unit_restore_recovery.sh b/tests/unit_restore_recovery.sh index c4494f8..3e8cb4e 100644 --- a/tests/unit_restore_recovery.sh +++ b/tests/unit_restore_recovery.sh @@ -105,6 +105,8 @@ fail() { echo "✗ $1"; FAIL=$((FAIL + 1)); } assert_true() { local l="$1"; shift; if "$@"; then pass "$l"; else fail "$l"; fi; } # Assert that the given command evaluates to false (nonzero exit status). assert_false() { local l="$1"; shift; if ! "$@"; then pass "$l"; else fail "$l"; fi; } +# True when a pre-restore safety copy of APP_DIR or DATA_DIR exists. +has_safety_copies() { compgen -G "$APP_DIR.backup.*" >/dev/null || compgen -G "$DATA_DIR.backup.*" >/dev/null; } # Run a command with its output discarded. quiet() { "$@" >/dev/null 2>&1; } # Assert equality between actual and expected values. @@ -278,6 +280,41 @@ if command -v sqlite3 >/dev/null 2>&1 && command -v zip >/dev/null 2>&1 && comma make_sqlite_archive noinventory drop out=$(run_restore "$WORK_DIR/archives/noinventory.zip" --check 2>&1) assert_eq "$?" 1 "restore --check: recovery metadata without inventory rejected" + + # A --fresh run that stopped after provisioning is continued by the same command. + ensure_docker_service_running() { :; } + make_sqlite_archive resume keep + mkdir -p "$APP_DIR" "$DATA_DIR" + unzip -p "$WORK_DIR/archives/resume.zip" .env >"$ENV_FILE" + unzip -p "$WORK_DIR/archives/resume.zip" docker-compose.yml >"$COMPOSE_FILE" + unzip -p "$WORK_DIR/archives/resume.zip" backup-files.sha256 | sha256sum | awk '{print $1}' >"$APP_DIR/.pasarguard-fresh-restore" + printf 'half provisioned\n' >"$DATA_DIR/leftover" + rm -rf "$APP_DIR".backup.* "$DATA_DIR".backup.* + : >"$FAKE_DOCKER_DIR/calls.log" + out=$(run_restore "$WORK_DIR/archives/resume.zip" --fresh --yes 2>&1) + rc=$? + assert_eq "$rc" 0 "fresh retry: same command continues the unfinished run" + assert_false "fresh retry: no safety copies of the half-provisioned install" has_safety_copies + case "$out" in *"Continuing it as an ordinary restore"*) pass "fresh retry: continuation announced" ;; *) fail "fresh retry: continuation announced" ;; esac + assert_true "fresh retry: database restored" test -s "$DATA_DIR/db.sqlite3" + assert_true "fresh retry: data restored" test -f "$DATA_DIR/cert.pem" + assert_false "fresh retry: marker removed after success" test -e "$APP_DIR/.pasarguard-fresh-restore" + assert_true "fresh retry: services started at the end" grep -q ' up -d' "$FAKE_DOCKER_DIR/calls.log" + + # The marker of another backup is not continued. + rm -rf "$APP_DIR" "$DATA_DIR" + mkdir -p "$APP_DIR" + unzip -p "$WORK_DIR/archives/resume.zip" .env >"$ENV_FILE" + unzip -p "$WORK_DIR/archives/resume.zip" docker-compose.yml >"$COMPOSE_FILE" + printf '%s\n' "$(printf '0%.0s' {1..64})" >"$APP_DIR/.pasarguard-fresh-restore" + : >"$FAKE_DOCKER_DIR/calls.log" + out=$(run_restore "$WORK_DIR/archives/resume.zip" --fresh --yes 2>&1) + rc=$? + assert_eq "$rc" 1 "fresh retry: marker from another backup refused" + case "$out" in *"used a different backup"*) pass "fresh retry: other backup explained" ;; *) fail "fresh retry: other backup explained" ;; esac + assert_false "fresh retry: refused retry stops no services" grep -qE ' (stop|down)( |$)' "$FAKE_DOCKER_DIR/calls.log" + assert_false "fresh retry: refused retry restores no data" test -e "$DATA_DIR/cert.pem" + rm -rf "$APP_DIR" "$DATA_DIR" rm -rf "$WORK_DIR/archives" "$WORK_DIR/edit" "$APP_DIR" "$DATA_DIR" else echo "(skipped restore --check cases: sqlite3/zip/unzip unavailable)" @@ -440,6 +477,7 @@ make_fresh_stage() { '{services: ({pasarguard: {image: $panel, volumes: [{type: "bind", source: $data, target: "/var/lib/pasarguard"}]}, postgresql: {image: "postgres:16"}} + (if $worker == "" then {} else {worker: {image: $panel}} end)), volumes: {}}' >"$FAKE_DOCKER_DIR/config.json" + write_backup_checksums "$stage" } reset_fake_docker @@ -462,6 +500,7 @@ assert_file_lacks "$FAKE_DOCKER_DIR/calls.log" "install_yq" "fresh: yq not insta assert_file_has "$FAKE_DOCKER_DIR/calls.log" "up -d --no-deps postgresql" "fresh: only the database started" assert_eq "$fresh_db_container" "cid-postgresql" "fresh: database container from Compose returned to the caller" assert_eq "$(stat -c %a "$ENV_FILE")" 600 "fresh: .env installed 0600" +assert_eq "$(cat "$APP_DIR/.pasarguard-fresh-restore" 2>/dev/null)" "$(sha256sum <"$stage/backup-files.sha256" | awk '{print $1}')" "fresh: provisioned installation marked with the backup's inventory hash" # Offline: no registry digest was recorded, the original image ID is loaded. reset_fake_docker @@ -492,6 +531,7 @@ prepare_fresh_restore "$stage" postgresql "$WORK_DIR/fresh.log" 5432 fresh_db_co assert_eq "$?" 1 "fresh: unreproducible image refused" assert_file_has "$WORK_DIR/fresh.out" "No reproducible image for service 'pasarguard'" "fresh: refusal names the service" assert_false "fresh: refusal installs no .env" test -e "$ENV_FILE" +assert_false "fresh: refusal leaves no retry marker" test -e "$APP_DIR/.pasarguard-fresh-restore" assert_file_lacks "$FAKE_DOCKER_DIR/calls.log" "tag " "fresh: refusal tags nothing" assert_file_lacks "$FAKE_DOCKER_DIR/calls.log" " up " "fresh: refusal starts nothing" @@ -564,6 +604,16 @@ prepare_fresh_restore "$stage" postgresql "$WORK_DIR/fresh.log" 5432 fresh_db_co assert_eq "$?" 0 "fresh: database without healthcheck accepted once it answers" assert_eq "$fresh_db_container" "cid-postgresql" "fresh: probed database container returned to the caller" +# ----------------------------------------------------------------------- +# install_pasarguard drops a retry marker left by an unfinished --fresh +# ----------------------------------------------------------------------- +rm -rf "$APP_DIR" "$DATA_DIR" +mkdir -p "$APP_DIR" +printf '%s\n' "$(printf '0%.0s' {1..64})" >"$APP_DIR/.pasarguard-fresh-restore" +(install_pasarguard latest 1 sqlite) >/dev/null 2>&1 +assert_false "install: retry marker from an unfinished --fresh removed" test -e "$APP_DIR/.pasarguard-fresh-restore" +rm -rf "$APP_DIR" "$DATA_DIR" + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1