From 051f6813cffc824d8bdbe37c6e25e3e5f0535b90 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 28 May 2026 15:17:26 +0530 Subject: [PATCH] MDEV-39061 mariadb-backup compatible wrappers for BACKUP SERVER scripts/mariabackup/mariabackup.sh: a drop-in wrapper that lets existing mariabackup invocations drive the server-side BACKUP SERVER command without changing user scripts. mariabackup.sh covers all four mariabackup modes. --backup translates into "BACKUP SERVER TO ''" via the mariadb client, forwarding connection options, and layers --stream/--compress as tar/gzip pipelines on the result. --prepare runs mariadbd --bootstrap on backup.cnf so InnoDB applies the archived redo log. --copy-back / --move-back drop a prepared backup into the datadir via cp -r / mv. --prepare --incremental-dir copies the incremental's ib_logfile* into the base and advances innodb_log_recovery_target; innodb_log_recovery_start stays pinned to the base checkpoint. --apply-log-only maps to --innodb-force-recovery=3 to skip rollback between incrementals. --rollback-xa runs two passes: normal recovery, then a second bootstrap with --tc-heuristic-recover=ROLLBACK. --copy-back / --move-back refuse a non-empty datadir unless --force-non-empty-directories is set, and print the post-action chown / systemctl start commands. For incremental --backup, innodb_log_archive_start is treated as a startup-only, read-only server invariant: the wrapper reads @@global.innodb_log_archive_start and fails fast if the archive floor exceeds the base backup's end LSN. Limitations: --export is accepted but not yet implemented; the wrapper prints a warning and runs plain recovery without producing the per-table .cfg files needed for ALTER TABLE ... IMPORT TABLESPACE. mbstream.sh shims the mbstream CLI onto tar, dropping mbstream-only flags (-p/--parallel) so legacy pipelines keep working. README.md maps every supported option per mode to its BACKUP SERVER equivalent and documents the backup.cnf format. Add include/have_mariabackup_wrapper.inc redirects $XTRABACKUP to the wrapper so a test opts in by sourcing one file; skips when the wrapper, bash, or the mariadb client is unavailable. wrapper_basic.test: exercises full backup, streaming, compression, the ignored legacy options. --- .../include/have_mariabackup_wrapper.inc | 56 ++ .../suite/mariabackup/wrapper_basic.result | 31 ++ .../suite/mariabackup/wrapper_basic.test | 82 +++ scripts/mariabackup/README.md | 517 ++++++++++++++++++ scripts/mariabackup/mariabackup.sh | 367 +++++++++++++ scripts/mariabackup/mbstream.sh | 19 + 6 files changed, 1072 insertions(+) create mode 100644 mysql-test/include/have_mariabackup_wrapper.inc create mode 100644 mysql-test/suite/mariabackup/wrapper_basic.result create mode 100644 mysql-test/suite/mariabackup/wrapper_basic.test create mode 100644 scripts/mariabackup/README.md create mode 100755 scripts/mariabackup/mariabackup.sh create mode 100755 scripts/mariabackup/mbstream.sh diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..8771fb50b078e --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,56 @@ +# ==== Purpose ==== +# +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through scripts/mariabackup/mariabackup.sh — the BACKUP SERVER +# compatibility wrapper — without any change to the test body. +# +# +# Skip the test if any of these are missing: +# - the wrapper script +# - bash +# - the mariadb client (wrapper shells out to it) +# +# ==== Usage ==== +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# ==== Exposed variables ==== +# +# $XTRABACKUP — now points at mariabackup.sh + +--source include/linux.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../scripts/mariabackup/mariabackup.sh + +--error 0,1 +perl; +use strict; +use warnings; +use File::Basename; + +my $wrapper = $ENV{MARIABACKUP_WRAPPER}; +exit 1 unless $wrapper && -x $wrapper; + +chomp(my $bash = `command -v bash 2>/dev/null`); +exit 1 unless $bash && -x $bash; + +# Prepend its directory to PATH so the bare `mariadb` invocation +# inside the wrapper resolves. +my ($mariadb) = split /\s+/, ($ENV{MYSQL} // ''); +exit 1 unless $mariadb && -x $mariadb; +$ENV{PATH} = dirname($mariadb) . ":$ENV{PATH}"; + +exit 0; +EOF + +if ($errno) +{ + --skip mariabackup.sh wrapper unavailable (script, bash, or mariadb client missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER diff --git a/mysql-test/suite/mariabackup/wrapper_basic.result b/mysql-test/suite/mariabackup/wrapper_basic.result new file mode 100644 index 0000000000000..ae903685801ce --- /dev/null +++ b/mysql-test/suite/mariabackup/wrapper_basic.result @@ -0,0 +1,31 @@ +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); +# +# Full backup succeeds and runs BACKUP SERVER +# +FOUND 1 /Executing: BACKUP SERVER TO/ in wrapper.log +# +# (--parallel/--throttle/--no-lock/--safe-slave-backup) +# +FOUND 1 /Executing: BACKUP SERVER TO/ in wrapper.log +# +# --stream=mbstream emits a valid tar archive to stdout +# +FOUND 1 /Creating tar stream/ in wrapper.log +# +# --compress produces a valid gzip stream +# +FOUND 1 /Compressing with gzip/ in wrapper.log +# +# Backup into an already-existing target directory is rejected +# +FOUND 1 /Target directory already exists/ in wrapper.log +# +# Missing --target-dir is rejected +# +FOUND 1 /--target-dir required/ in wrapper.log +# +# Non-existent parent directory is rejected +# +FOUND 1 /Parent directory does not exist/ in wrapper.log +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/wrapper_basic.test b/mysql-test/suite/mariabackup/wrapper_basic.test new file mode 100644 index 0000000000000..744d39265eeee --- /dev/null +++ b/mysql-test/suite/mariabackup/wrapper_basic.test @@ -0,0 +1,82 @@ +--source include/have_mariabackup_wrapper.inc +--source include/have_innodb.inc + +--let $defaults=--defaults-file=$MYSQLTEST_VARDIR/my.cnf +--let $logfile=$MYSQLTEST_VARDIR/tmp/wrapper.log +--let SEARCH_FILE=$logfile +--let SEARCH_ABORT=NOT FOUND + +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); + +--echo # +--echo # Full backup succeeds and runs BACKUP SERVER +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_full +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir > $logfile 2>&1 +--let SEARCH_PATTERN=Executing: BACKUP SERVER TO +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # (--parallel/--throttle/--no-lock/--safe-slave-backup) +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_legacy +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --parallel=4 --throttle=100 --no-lock --safe-slave-backup > $logfile 2>&1 +--let SEARCH_PATTERN=Executing: BACKUP SERVER TO +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # --stream=mbstream emits a valid tar archive to stdout +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_stream +--let $streamfile=$MYSQLTEST_VARDIR/tmp/bk_stream.tar +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --stream=mbstream > $streamfile 2>$logfile +--exec tar -tf $streamfile > /dev/null +--let SEARCH_PATTERN=Creating tar stream +--source include/search_pattern_in_file.inc +--rmdir $targetdir +--remove_file $streamfile + +--echo # +--echo # --compress produces a valid gzip stream +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_gz +--let $gzfile=$MYSQLTEST_VARDIR/tmp/bk.tar.gz +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir --compress > $gzfile 2>$logfile +--exec gzip -t $gzfile +--let SEARCH_PATTERN=Compressing with gzip +--source include/search_pattern_in_file.inc +--rmdir $targetdir +--remove_file $gzfile + +--echo # +--echo # Backup into an already-existing target directory is rejected +--echo # +--let $targetdir=$MYSQLTEST_VARDIR/tmp/bk_exists +--mkdir $targetdir +--error 1 +--exec $XTRABACKUP $defaults --backup --target-dir=$targetdir > $logfile 2>&1 +--let SEARCH_PATTERN=Target directory already exists +--source include/search_pattern_in_file.inc +--rmdir $targetdir + +--echo # +--echo # Missing --target-dir is rejected +--echo # +--error 1 +--exec $XTRABACKUP $defaults --backup > $logfile 2>&1 +--let SEARCH_PATTERN=--target-dir required +--source include/search_pattern_in_file.inc + +--echo # +--echo # Non-existent parent directory is rejected +--echo # +--error 1 +--exec $XTRABACKUP $defaults --backup --target-dir=$MYSQLTEST_VARDIR/tmp/no_such_parent/bk > $logfile 2>&1 +--let SEARCH_PATTERN=Parent directory does not exist +--source include/search_pattern_in_file.inc + +DROP TABLE t1; +--remove_file $logfile diff --git a/scripts/mariabackup/README.md b/scripts/mariabackup/README.md new file mode 100644 index 0000000000000..21f67c54a2191 --- /dev/null +++ b/scripts/mariabackup/README.md @@ -0,0 +1,517 @@ +# MariaDB Backup Wrapper + +A drop-in `mariabackup`-compatible shell wrapper that translates the +familiar CLI into MariaDB's server-side `BACKUP SERVER` SQL command. +Lets DBAs migrate to BACKUP SERVER without changing existing scripts. + +## Overview + +`mariabackup.sh` masks the traditional `mariabackup` binary. With +`--backup`, it parses MariaBackup options, sets `backup_include` and +`backup_exclude` via the `mariadb` client, then issues +`BACKUP SERVER TO ''`. Optional streaming, compression, and +encryption are shell pipelines layered on the resulting directory. +All actual backup work happens server-side. + +**Prerequisites:** MariaDB with BACKUP SERVER support, `mariadb` +client in `PATH`, an account with `BACKUP SERVER` + `SET GLOBAL` +privileges, `innodb_log_archive=ON` for incrementals, and the +server's `innodb_log_archive_start` (startup-only) set no higher +than the base backup's end LSN. + +--- + +## --backup + +### Description + +Creates a backup using `BACKUP SERVER`. Produces a backup directory +with data files, redo logs, and `backup.cnf` carrying LSN metadata. + +### Structure + +``` +mariabackup.sh --backup --target-dir=DIRECTORY [OPTIONS] +``' + +### Options + +| Option | Description | +| ---------------------------- | ------------------------------------------------------------------------ | +| `--target-dir=DIR` | **(required)** Backup destination | +| `--incremental-basedir=DIR` | Incremental backup based on a prior full backup | +| `--stream=mbstream` | Stream the backup to stdout as a tar archive | +| `--databases=REGEX` | Include pattern (comma-separated list supported) | +| `--databases-exclude=REGEX` | Exclude pattern (comma-separated list supported) | +| `--tables=REGEX` | Table-level include (used only if `--databases` not set) | +| `--tables-exclude=REGEX` | Table-level exclude (used only if `--databases-exclude` not set) | +| `--tables-file=FILE` | File of `database.table` entries, one per line, merged into `--tables` | +| `--compress` | Pipe stream through `gzip` (or `pigz` if `--compress-threads` is set) | +| `--compress-threads=N` | Use `pigz -p N` instead of `gzip` | +| `--encrypt=ALG` | Pipe stream through `openssl enc -ALG -salt -pbkdf2` | + +**Connection options** (forwarded to the `mariadb` client): +`--user`/`-u`, `--password`/`-p`, `--host`/`-h`, `--port`/`-P`, +`--socket`/`-S`, `--defaults-file`, `--defaults-extra-file`. + +**Silently ignored** (BACKUP SERVER handles server-side): +`--parallel`, `--throttle`, `--no-lock`, `--safe-slave-backup`. + + +**Precedence:** + +- `--databases` wins over `--tables`; `--databases-exclude` wins over + `--tables-exclude` (the loser is ignored with a warning). +- `--tables-exclude` wins over `--tables`. +- `--tables-file` is merged into `--tables`. + +### BACKUP SERVER Mapping + +```sql +SET GLOBAL backup_include=''; -- only if include built +SET GLOBAL backup_exclude=''; -- only if exclude built +BACKUP SERVER TO '/path/to/backup'; +``` + +The patterns land in `backup.cnf` inside the target directory along +with `innodb_log_recovery_start` / `innodb_log_recovery_target`. + +--- + +### --target-dir + +#### Description + +Backup destination directory. Required. Must not already exist; parent +must exist and be writable. + +#### BACKUP SERVER Mapping + +```sql +BACKUP SERVER TO '/path/to/backup'; +``` +--- + +### --incremental-basedir + +#### Description + +Creates an incremental backup containing only redo logs since the base +backup. The wrapper reads `innodb_log_recovery_target` from the base +`backup.cnf` and verifies it is **≥** the server's +`@@innodb_log_archive_start`: i.e., the archive still covers from +the base's end LSN forward. If the archive has been pruned past that +point, the incremental is impossible and the wrapper fails fast. + +Requires `innodb_log_archive=ON` on the server. The archive floor +(`innodb_log_archive_start`) is a startup-only, read-only variable +configured by the DBA; the wrapper never tries to mutate it. + +#### BACKUP SERVER Mapping + +```bash +BASE_LSN=$(grep ^innodb_log_recovery_target /base/backup.cnf | cut -d= -f2) +FLOOR=$(mariadb -BN -e "SELECT @@global.innodb_log_archive_start") +[ "$FLOOR" -le "$BASE_LSN" ] || exit 1 # archive pruned past base +mariadb -e "BACKUP SERVER TO '/incremental/path'" +``` +--- + +### --stream + +#### Description + +Streams the backup directory to stdout as a tar archive. Only +`mbstream` is supported (mapped to `tar`). The included `mbstream.sh` +wrapper drops mbstream-specific flags (`-p`/`--parallel`) so legacy +pipelines keep working. + +#### BACKUP SERVER Mapping + +```bash +mariadb -e "BACKUP SERVER TO '/tmp/backup'" +tar -c -f - -C /tmp/backup . +``` +--- + +### --compress + +#### Description + +Pipes the stream through `gzip` (or `pigz` if `--compress-threads` is +set). Implies `--stream=mbstream`. The compression algorithm argument +(e.g. `--compress=quicklz`) is accepted for CLI compatibility but +ignored; output is always gzip-compatible. + +#### BACKUP SERVER Mapping + +```bash +mariadb -e "BACKUP SERVER TO '/tmp/backup'" +tar -c -f - -C /tmp/backup . | gzip +``` + +--- + +### --compress-threads + +#### Description + +Switches compression from `gzip` to `pigz -p N`. Implies `--compress`. + +#### BACKUP SERVER Mapping + +```bash +tar -c -f - -C /tmp/backup . | pigz -p N +``` + +--- + +### --encrypt + +#### Description + +Pipes the stream through `openssl enc -ALG -salt -pbkdf2`. Implies +`--stream=mbstream`. Combines with `--compress`: compression runs +before encryption. + +#### BACKUP SERVER Mapping + +```bash +tar -c -f - -C /tmp/backup . \ + | gzip \ # if --compress + | openssl enc -aes-256-cbc -salt -pbkdf2 +``` +--- + + +## --prepare + +### Description + +Prepares a BACKUP SERVER backup directory for restore by running +`mariadbd --bootstrap` against its `backup.cnf`, so InnoDB applies the +archived redo log to the data files and exits. For incrementals, +copies the increment's redo logs into the base directory and advances +the LSN bounds in `backup.cnf` before bootstrap. + +### Structure + +``` +mariabackup.sh --prepare --target-dir=DIRECTORY [OPTIONS] +``` + +### Options + +| Option | Description | +| ------------------------------- | ------------------------------------------------------------------------ | +| `--target-dir=DIR` | **(required)** Backup directory to prepare | +| `--incremental-dir=DIR` | Merge an incremental backup into `--target-dir` before recovery | +| `--apply-log` | Synonym for `--prepare` | +| `--apply-log-only` | Apply redo only; skip rollback (use between incrementals in a chain) | +| `--export` | Produce per-table `.cfg` files for `IMPORT TABLESPACE` | +| `--rollback-xa` | Roll back prepared XA transactions during recovery | +| `--use-memory=N` | InnoDB buffer pool size during recovery (default 96 MiB) | +| `--parallel=N` | Threads for redo apply | +| `--force-non-empty-directories` | Allow `--target-dir` to contain unrelated files | + +**Forwarded to the bootstrap `mariadbd`:** all `--innodb-*` tunables, +`--tmpdir`/`-t`, `--datadir`/`-h`, `--defaults-file`, +`--defaults-extra-file`, `--defaults-group`, +`--log-innodb-page-corruption`, `--mysqld`. + +### BACKUP SERVER Mapping + +Full prepare: + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +Incremental prepare: + +```bash +cp /ib_logfile* / +# atomic backup.cnf rewrite (write temp + mv): +# innodb_log_recovery_start unchanged (still base's original checkpoint) +# innodb_log_recovery_target ← _target +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +`--apply-log-only` adds `--innodb-force-recovery=3`. +`--export` pipes `FLUSH TABLES ... FOR EXPORT` statements to bootstrap stdin. + +--- + +### --target-dir + +#### Description + +Backup directory to prepare. Required. Must already exist and contain +a `backup.cnf` produced by `BACKUP SERVER`. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --incremental-dir + +#### Description + +Applies an incremental backup on top of `--target-dir`. The wrapper +copies the incremental's `ib_logfile*` into the base and atomically +rewrites `backup.cnf` to advance `innodb_log_recovery_target` to the +incremental's `_target`. `innodb_log_recovery_start` stays pinned to +base's original checkpoint, so recovery always replays from there. + +Order-dependent: apply incrementals in the order they were taken. + +#### BACKUP SERVER Mapping + +```bash +cp /backup/inc1/ib_logfile* /backup/base/ +# rewrite /backup/base/backup.cnf: +# innodb_log_recovery_start unchanged (base's original checkpoint) +# innodb_log_recovery_target= +mariadbd --bootstrap --defaults-file=/backup/base/backup.cnf < /dev/null +``` + +--- + +### --apply-log-only + +#### Description + +Applies redo but skips rollback of uncommitted transactions. Use only +between incrementals in a chain: the **final** `--prepare` must omit +this option so the rollback phase actually runs. Implemented via +`innodb_force_recovery=3`, which keeps writes enabled (below the +read-only threshold at level 4) and leaves undo logs intact for the +next incremental. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --innodb-force-recovery=3 \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --export + +#### Description + +Produces per-table `.cfg` files alongside the data files so individual +tables can be restored on another server via +`ALTER TABLE ... IMPORT TABLESPACE`. The wrapper enumerates the backed-up +tables and feeds `FLUSH TABLES ... FOR EXPORT` to bootstrap stdin after +recovery. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap --defaults-file=/backup.cnf </backup.cnf < /dev/null +``` + +--- + +### --rollback-xa + +#### Description + +Rolls back prepared XA transactions during recovery. Off by default: +prepared XA state survives the prepare unless this option is set. + +Implemented as a **two-pass** bootstrap because `tc-heuristic-recover` +and automatic crash recovery are mutually exclusive in the server +(`sql/log.cc:12285`): + +1. **Pass 1**: normal recovery. Applies redo, rolls back uncommitted + non-XA transactions. +2. **Pass 2**: heuristic XA cleanup. Starts again with + `--tc-heuristic-recover=ROLLBACK`, which force-rolls-back **all** + prepared XA transactions and exits. + +#### BACKUP SERVER Mapping + +```bash +# Pass 1: normal recovery +mariadbd --bootstrap --defaults-file=/backup.cnf < /dev/null + +# Pass 2: heuristic XA rollback +mariadbd --bootstrap --tc-heuristic-recover=ROLLBACK \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + +### --innodb-\* tunables + +#### Description + +All `--innodb-*` options accepted by `mariadbd` are forwarded +verbatim. Required when the source server used non-default page size, +log group home dir, or data file path: recovery needs to read the +files back under the same geometry. + +#### BACKUP SERVER Mapping + +```bash +mariadbd --bootstrap \ + --innodb-page-size=16K \ + --innodb-log-files-in-group=2 \ + --defaults-file=/backup.cnf < /dev/null +``` + +--- + + +## --copy-back + +### Description + +Copies a prepared backup into the server's datadir. The source backup +directory is preserved. Run after `--prepare` has applied redo logs +and the backup is consistent. The server must be **stopped** during +the copy. + +### Structure + +``` +mariabackup.sh --copy-back --target-dir=DIRECTORY --datadir=DATADIR [OPTIONS] +``` + +### Options + +| Option | Description | +| ------------------------------- | ---------------------------------------------------------------- | +| `--target-dir=DIR` | **(required)** Prepared backup directory (source) | +| `--datadir=DIR` | **(required)** Server datadir (destination) | +| `--force-non-empty-directories` | Allow `--datadir` to contain pre-existing files | +| `--parallel=N` | Ignored: `cp -r` is single-threaded | + +**Forwarded for split-path layouts:** `--innodb-data-home-dir`, +`--innodb-undo-directory`, `--innodb-log-group-home-dir`, +`--defaults-file`, `--defaults-extra-file`, `--defaults-group`. + +### BACKUP SERVER Mapping + +```bash +cp -r /backup/base/* /var/lib/mysql/ +# post-action: +chown -R mysql:mysql /var/lib/mysql/ +systemctl start mariadb +``` + +The wrapper refuses a non-empty `--datadir` unless +`--force-non-empty-directories` is set, and prints the post-action +`chown` and server-start commands to stderr after the copy completes. + +--- + +## --move-back + +### Description + +Moves a prepared backup into the server's datadir. The source backup +is consumed (its files are renamed onto the datadir). Faster than +`--copy-back` when source and destination share a filesystem: each +file becomes a single `rename(2)` instead of a full copy. + +### Structure + +``` +mariabackup.sh --move-back --target-dir=DIRECTORY --datadir=DATADIR [OPTIONS] +``` + +### Options + +Same as `--copy-back`. `mv` preserves the source file ownership, so +the post-action `chown` is still required before starting the server. + +### BACKUP SERVER Mapping + +```bash +mv /backup/base/* /var/lib/mysql/ +# post-action: +chown -R mysql:mysql /var/lib/mysql/ +systemctl start mariadb +``` + +--- + + +## backup.cnf Format + +Auto-generated by `BACKUP SERVER` inside the target directory. + +```ini +[mariadbd] +datadir=/backup/partial +innodb_log_recovery_start=12288 +innodb_log_recovery_target=15000 +backup_include=^prod\..* +backup_exclude=^prod\.temp.*,^prod\.cache.* +``` + +| Field | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------ | +| `datadir` | Backup directory path | +| `innodb_log_recovery_start` | Latest checkpoint LSN at the start of the base backup. Recovery begins scanning here. | +| | Pinned: does not advance when incrementals are merged in. | +| `innodb_log_recovery_target` | End LSN of the backup. Recovery stops here, ignoring any extra archive records on disk. | +| | Advances with each merged incremental. | +| `backup_include` | Include pattern, partial backups only | +| `backup_exclude` | Exclude pattern, partial backups only | + +Both `_start` and `_target` are written by `BACKUP SERVER`; `_start` +stays fixed across the prepare chain while `_target` advances as +incrementals are applied. The include/exclude lines are omitted when +no filter was applied. + +--- + +## BACKUP SERVER Variables + +| Variable | Type | Access | Description | +| --------------------------- | -------------- | ------ | ---------------------------------------------------------------------------- | +| `innodb_log_archive` | Boolean | RW | Enables redo log archiving. Must be `ON` for incremental backups. | +| `innodb_log_archive_start` | Integer (LSN) | Read-only, startup-only | Floor for `innodb_log_recovery_start`: declares where the | +| | | | on-disk redo archive begins. Set by the DBA at server | +| | | | startup (`mariadbd --innodb-log-archive-start=N`) after | +| | | | pruning old archive files; Wrapper only reads it | +| | | | (`SELECT @@global.innodb_log_archive_start`) to verify | +| | | | an incremental is still possible. | +| `backup_include` | String (POSIX ERE) | RW | Comma-separated patterns matched against `db.table` (literal `.`). | +| `backup_exclude` | String (POSIX ERE) | RW | Comma-separated patterns matched against `db.table`. | + +The wrapper sets `backup_include` and `backup_exclude` via +`SET GLOBAL`, then runs `BACKUP SERVER`. `innodb_log_archive_start` +is read-only and configured at server startup. Final include/exclude +patterns are also written into `backup.cnf` for +restore tooling. + +--- diff --git a/scripts/mariabackup/mariabackup.sh b/scripts/mariabackup/mariabackup.sh new file mode 100755 index 0000000000000..4dcb71a34616f --- /dev/null +++ b/scripts/mariabackup/mariabackup.sh @@ -0,0 +1,367 @@ +#!/bin/bash +# mariabackup.sh: BACKUP SERVER-compatible mariabackup wrapper. + +MODE="" +TARGET_DIR="" +STREAM_FORMAT="" +INCREMENTAL_BASEDIR="" +COMPRESS="" +COMPRESS_THREADS="" +ENCRYPT="" +DATABASES_PATTERN="" +DATABASES_EXCLUDE_PATTERN="" +TABLES_PATTERN="" +TABLES_EXCLUDE_PATTERN="" +TABLES_FILE="" +MARIADB_OPTS="" +INCREMENTAL_DIR="" +APPLY_LOG_ONLY="" +EXPORT="" +ROLLBACK_XA="" +USE_MEMORY="" +FORCE_NON_EMPTY="" +INNODB_OPTS="" +MYSQLD_EXTRA="" +MYSQLD_BIN="mariadbd" +DATADIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --backup) MODE="backup"; shift ;; + --prepare|--apply-log) MODE="prepare"; shift ;; + --copy-back) MODE="copy-back"; shift ;; + --move-back) MODE="move-back"; shift ;; + + --target-dir=*) TARGET_DIR="${1#*=}"; shift ;; + --datadir=*) DATADIR="${1#*=}"; shift ;; + --stream=*) STREAM_FORMAT="${1#*=}"; shift ;; + --incremental-basedir=*) INCREMENTAL_BASEDIR="${1#*=}"; shift ;; + --incremental-dir=*) INCREMENTAL_DIR="${1#*=}"; shift ;; + --use-memory=*) USE_MEMORY="${1#*=}"; shift ;; + --mysqld=*) MYSQLD_BIN="${1#*=}"; shift ;; + + --apply-log-only) APPLY_LOG_ONLY="yes"; shift ;; + --export) EXPORT="yes"; shift ;; + --rollback-xa) ROLLBACK_XA="yes"; shift ;; + --force-non-empty-directories) FORCE_NON_EMPTY="yes"; shift ;; + + --innodb-*=*|--innodb-*) INNODB_OPTS="$INNODB_OPTS $1"; shift ;; + --tmpdir=*|--log-innodb-page-corruption) MYSQLD_EXTRA="$MYSQLD_EXTRA $1"; shift ;; + + --databases=*) DATABASES_PATTERN="${1#*=}"; shift ;; + --databases-exclude=*) DATABASES_EXCLUDE_PATTERN="${1#*=}"; shift ;; + --tables=*) TABLES_PATTERN="${1#*=}"; shift ;; + --tables-exclude=*) TABLES_EXCLUDE_PATTERN="${1#*=}"; shift ;; + --tables-file=*) TABLES_FILE="${1#*=}"; shift ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + -u|-p|-h|-P|-S) + # Bare `-p` is a password prompt: only consume the next argv if it looks like a value. + if [[ -n "${2-}" && "$2" != -* ]]; then + MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift 2 + else + MARIADB_OPTS="$MARIADB_OPTS $1"; shift + fi + ;; + -u*|-p*|-h*|-P*|-S*) + MARIADB_OPTS="$MARIADB_OPTS $1"; shift ;; + + --compress|--compress=*) + # Compression algorithm value is ignored: output is always gzip/pigz. + COMPRESS="yes"; shift ;; + --compress-threads=*) COMPRESS_THREADS="${1#*=}"; shift ;; + --encrypt=*) ENCRYPT="${1#*=}"; shift ;; + + --parallel=*|--throttle=*|--no-lock|--safe-slave-backup) + # Handled server-side by BACKUP SERVER. + shift ;; + + *) shift ;; + esac +done + +if [[ -z "$TARGET_DIR" ]]; then + echo "Error: --target-dir required" >&2 + exit 1 +fi + +# --prepare +if [[ "$MODE" == "prepare" ]]; then + if [[ ! -d "$TARGET_DIR" ]]; then + echo "Error: Target directory does not exist: $TARGET_DIR" >&2 + exit 1 + fi + if [[ ! -f "$TARGET_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in target directory: $TARGET_DIR" >&2 + exit 1 + fi + + if [[ -n "$INCREMENTAL_DIR" ]]; then + if [[ ! -d "$INCREMENTAL_DIR" ]]; then + echo "Error: Incremental directory does not exist: $INCREMENTAL_DIR" >&2 + exit 1 + fi + if [[ ! -f "$INCREMENTAL_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in incremental directory: $INCREMENTAL_DIR" >&2 + exit 1 + fi + INC_TARGET=$(grep "^innodb_log_recovery_target" "$INCREMENTAL_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + if [[ -z "$INC_TARGET" ]]; then + echo "Error: Could not read innodb_log_recovery_target from $INCREMENTAL_DIR/backup.cnf" >&2 + exit 1 + fi + echo "Merging incremental: advancing _target to $INC_TARGET" >&2 + + cp "$INCREMENTAL_DIR"/ib_logfile* "$TARGET_DIR/" || { + echo "Error: Failed to copy redo logs from $INCREMENTAL_DIR" >&2 + exit 1 + } + + # _start stays pinned to base's original checkpoint; only _target advances. + TMP_CNF="$TARGET_DIR/backup.cnf.tmp.$$" + sed -e "s/^innodb_log_recovery_target=.*/innodb_log_recovery_target=$INC_TARGET/" \ + "$TARGET_DIR/backup.cnf" > "$TMP_CNF" \ + && mv "$TMP_CNF" "$TARGET_DIR/backup.cnf" || { + echo "Error: Failed to update $TARGET_DIR/backup.cnf" >&2 + rm -f "$TMP_CNF" + exit 1 + } + fi + + BOOTSTRAP_OPTS="" + [[ -n "$APPLY_LOG_ONLY" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS --innodb-force-recovery=3" + [[ -n "$USE_MEMORY" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS --innodb-buffer-pool-size=$USE_MEMORY" + [[ -n "$INNODB_OPTS" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS$INNODB_OPTS" + [[ -n "$MYSQLD_EXTRA" ]] && BOOTSTRAP_OPTS="$BOOTSTRAP_OPTS$MYSQLD_EXTRA" + + if [[ -n "$EXPORT" ]]; then + echo "Warning: --export is not yet implemented; running plain recovery" >&2 + fi + + # Pass 1: normal recovery. + echo "Pass 1: $MYSQLD_BIN --bootstrap --defaults-file=$TARGET_DIR/backup.cnf$BOOTSTRAP_OPTS" >&2 + $MYSQLD_BIN --bootstrap --defaults-file="$TARGET_DIR/backup.cnf" $BOOTSTRAP_OPTS < /dev/null + PREP_STATUS=$? + if [[ $PREP_STATUS -ne 0 ]]; then + echo "Error: prepare pass 1 failed (exit $PREP_STATUS)" >&2 + exit $PREP_STATUS + fi + + # Pass 2: heuristic XA rollback. tc-heuristic-recover conflicts with + # automatic crash recovery, so it has to run separately after pass 1. + if [[ -n "$ROLLBACK_XA" ]]; then + echo "Pass 2: $MYSQLD_BIN --bootstrap --tc-heuristic-recover=ROLLBACK --defaults-file=$TARGET_DIR/backup.cnf" >&2 + $MYSQLD_BIN --bootstrap --tc-heuristic-recover=ROLLBACK \ + --defaults-file="$TARGET_DIR/backup.cnf" < /dev/null + XA_STATUS=$? + if [[ $XA_STATUS -ne 0 ]]; then + echo "Error: prepare pass 2 (XA rollback) failed (exit $XA_STATUS)" >&2 + exit $XA_STATUS + fi + fi + + echo "Prepare completed: $TARGET_DIR" >&2 + exit 0 +fi + +# --copy-back / --move-back +if [[ "$MODE" == "copy-back" || "$MODE" == "move-back" ]]; then + if [[ ! -d "$TARGET_DIR" ]]; then + echo "Error: Target directory does not exist: $TARGET_DIR" >&2 + exit 1 + fi + if [[ ! -f "$TARGET_DIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in $TARGET_DIR (not a prepared backup?)" >&2 + exit 1 + fi + if [[ -z "$DATADIR" ]]; then + echo "Error: --datadir required for --$MODE" >&2 + exit 1 + fi + if [[ ! -d "$DATADIR" ]]; then + echo "Error: Datadir does not exist: $DATADIR" >&2 + exit 1 + fi + if [[ -z "$FORCE_NON_EMPTY" ]] && [[ -n "$(ls -A "$DATADIR" 2>/dev/null)" ]]; then + echo "Error: Datadir is not empty: $DATADIR" >&2 + echo "Pass --force-non-empty-directories to override" >&2 + exit 1 + fi + + if [[ "$MODE" == "copy-back" ]]; then + echo "Copying $TARGET_DIR/ to $DATADIR/" >&2 + cp -r "$TARGET_DIR"/. "$DATADIR"/ || { + echo "Error: copy-back failed" >&2 + exit 1 + } + else + echo "Moving $TARGET_DIR/ to $DATADIR/" >&2 + ( shopt -s dotglob nullglob + mv "$TARGET_DIR"/* "$DATADIR"/ ) || { + echo "Error: move-back failed" >&2 + exit 1 + } + fi + + echo "Restore completed: $DATADIR" >&2 + echo "Post-action required:" >&2 + echo " chown -R mysql:mysql $DATADIR" >&2 + echo " systemctl start mariadb" >&2 + exit 0 +fi + +# --backup + +if [[ -e "$TARGET_DIR" ]]; then + echo "Error: Target directory already exists: $TARGET_DIR" >&2 + echo "Remove it first or choose a different target directory" >&2 + exit 1 +fi + +PARENT_DIR="$(dirname "$TARGET_DIR")" +if [[ ! -d "$PARENT_DIR" ]]; then + echo "Error: Parent directory does not exist: $PARENT_DIR" >&2 + exit 1 +fi +if [[ ! -w "$PARENT_DIR" ]]; then + echo "Error: Parent directory is not writable: $PARENT_DIR" >&2 + exit 1 +fi + +if [[ -n "$COMPRESS" || -n "$ENCRYPT" ]] && [[ -z "$STREAM_FORMAT" ]]; then + STREAM_FORMAT="mbstream" +fi + +if [[ -n "$INCREMENTAL_BASEDIR" ]]; then + if [[ ! -d "$INCREMENTAL_BASEDIR" ]]; then + echo "Error: Base backup directory does not exist: $INCREMENTAL_BASEDIR" >&2 + exit 1 + fi + if [[ ! -f "$INCREMENTAL_BASEDIR/backup.cnf" ]]; then + echo "Error: backup.cnf not found in base backup directory: $INCREMENTAL_BASEDIR" >&2 + exit 1 + fi + BASE_LSN=$(grep "^innodb_log_recovery_target" "$INCREMENTAL_BASEDIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + if [[ -z "$BASE_LSN" ]]; then + echo "Error: Could not read innodb_log_recovery_target from $INCREMENTAL_BASEDIR/backup.cnf" >&2 + exit 1 + fi + echo "Base backup LSN: $BASE_LSN" >&2 + + # innodb_log_archive_start is startup-only and read-only on the server. + # Verify the archive floor still covers the base before kicking off the + # incremental: if older logs have been pruned, the request is impossible. + SERVER_FLOOR=$(mariadb $MARIADB_OPTS -BN -e "SELECT @@global.innodb_log_archive_start" 2>/dev/null) + if [[ -z "$SERVER_FLOOR" ]]; then + echo "Error: Could not read @@global.innodb_log_archive_start from server" >&2 + exit 1 + fi + if (( SERVER_FLOOR > BASE_LSN )); then + echo "Error: server's innodb_log_archive_start=$SERVER_FLOOR exceeds base backup's" >&2 + echo " end LSN=$BASE_LSN. Archive files needed for this incremental have" >&2 + echo " been pruned. Take a fresh full backup instead." >&2 + exit 1 + fi + echo "Archive floor OK: server $SERVER_FLOOR <= base $BASE_LSN" >&2 +fi + +# Build backup_include / backup_exclude with precedence: +# --databases beats --tables; --databases-exclude beats --tables-exclude. +# --tables-file is escaped (`.` -> `[.]`) and merged into --tables. +# BACKUP SERVER has a single include / single exclude variable, so --databases +# and --tables cannot both apply: combine them into one --databases regex. + +FINAL_INCLUDE="" +FINAL_EXCLUDE="" + +if [[ -n "$TABLES_FILE" ]]; then + if [[ ! -f "$TABLES_FILE" ]]; then + echo "Error: Tables file not found: $TABLES_FILE" >&2 + exit 1 + fi + TABLES_FROM_FILE="" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + # Escape `.` to `[.]` so prod.users does not accidentally match prodxusers. + table_pattern="${line//./[.]}" + if [[ -z "$TABLES_FROM_FILE" ]]; then + TABLES_FROM_FILE="$table_pattern" + else + TABLES_FROM_FILE="$TABLES_FROM_FILE,$table_pattern" + fi + done < "$TABLES_FILE" + if [[ -n "$TABLES_PATTERN" ]]; then + TABLES_PATTERN="$TABLES_PATTERN,$TABLES_FROM_FILE" + else + TABLES_PATTERN="$TABLES_FROM_FILE" + fi +fi + +if [[ -n "$DATABASES_PATTERN" ]]; then + FINAL_INCLUDE="$DATABASES_PATTERN" + if [[ -n "$TABLES_PATTERN" ]]; then + echo "Warning: --tables='$TABLES_PATTERN' is ignored because --databases takes precedence" >&2 + echo " To filter both database and tables, combine them into one --databases pattern." >&2 + fi +elif [[ -n "$TABLES_PATTERN" ]]; then + FINAL_INCLUDE="$TABLES_PATTERN" +fi + +if [[ -n "$DATABASES_EXCLUDE_PATTERN" ]]; then + FINAL_EXCLUDE="$DATABASES_EXCLUDE_PATTERN" +elif [[ -n "$TABLES_EXCLUDE_PATTERN" ]]; then + FINAL_EXCLUDE="$TABLES_EXCLUDE_PATTERN" +fi + +if [[ -n "$FINAL_INCLUDE" ]]; then + echo "Setting backup_include='$FINAL_INCLUDE'" >&2 + mariadb $MARIADB_OPTS -e "SET GLOBAL backup_include='$FINAL_INCLUDE'" +fi + +if [[ -n "$FINAL_EXCLUDE" ]]; then + echo "Setting backup_exclude='$FINAL_EXCLUDE'" >&2 + mariadb $MARIADB_OPTS -e "SET GLOBAL backup_exclude='$FINAL_EXCLUDE'" +fi + +SQL="BACKUP SERVER TO '$TARGET_DIR'" +echo "Executing: $SQL" >&2 +mariadb $MARIADB_OPTS -e "$SQL" + +if [[ -n "$STREAM_FORMAT" ]]; then + case "$STREAM_FORMAT" in + mbstream) + echo "Creating tar stream from $TARGET_DIR" >&2 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + STREAM_CMD=("$SCRIPT_DIR/mbstream.sh" -c -f - -C "$TARGET_DIR" .) + if [[ -n "$COMPRESS" && -n "$ENCRYPT" ]]; then + if [[ -n "$COMPRESS_THREADS" ]]; then + echo "Compressing with pigz -p $COMPRESS_THREADS and encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | pigz -p "$COMPRESS_THREADS" | openssl enc -"$ENCRYPT" -salt -pbkdf2 + else + echo "Compressing with gzip and encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | gzip | openssl enc -"$ENCRYPT" -salt -pbkdf2 + fi + elif [[ -n "$COMPRESS" ]]; then + if [[ -n "$COMPRESS_THREADS" ]]; then + echo "Compressing with pigz -p $COMPRESS_THREADS" >&2 + "${STREAM_CMD[@]}" | pigz -p "$COMPRESS_THREADS" + else + echo "Compressing with gzip" >&2 + "${STREAM_CMD[@]}" | gzip + fi + elif [[ -n "$ENCRYPT" ]]; then + echo "Encrypting with $ENCRYPT" >&2 + "${STREAM_CMD[@]}" | openssl enc -"$ENCRYPT" -salt -pbkdf2 + else + "${STREAM_CMD[@]}" + fi + ;; + *) + echo "Error: Unsupported stream format: $STREAM_FORMAT (only mbstream is supported)" >&2 + exit 1 + ;; + esac +fi diff --git a/scripts/mariabackup/mbstream.sh b/scripts/mariabackup/mbstream.sh new file mode 100755 index 0000000000000..bcd47ba1b2b42 --- /dev/null +++ b/scripts/mariabackup/mbstream.sh @@ -0,0 +1,19 @@ +#!/bin/bash +ARGS=() +SKIP_NEXT=0 +for arg in "$@"; do + [[ $SKIP_NEXT -eq 1 ]] && { SKIP_NEXT=0; continue; } + case "$arg" in + -p|--parallel) + SKIP_NEXT=1 + ;; + -p*) + ;; + --parallel=*) + ;; + *) + ARGS+=("$arg") + ;; + esac +done +exec tar "${ARGS[@]}"