feat(email): protect retained recovery before maintenance

.github/workflows/prepare-retained-email-reader-inputs.yaml

@@ -0,0 +1,120 @@
+name: Prepare Retained Email Reader Inputs
+
+"on":
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: "Type exactly: prepare-retained-email-reader-inputs"
+        required: true
+        type: string
+
+permissions: {}
+concurrency:
+  group: prod-deploy
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  prepare:
+    name: Preserve the fixed retained reader inputs
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    environment: prod
+    permissions:
+      contents: read
+    steps:
+      - name: Require the confirmed first-attempt main dispatch
+        shell: bash
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+        run: |
+          if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != prepare-retained-email-reader-inputs ]]; then
+            printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=dispatch restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2
+            exit 2
+          fi
+
+      - name: Check out the confirmed immutable revision
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Set up the repository Go version
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Build the reviewed command before restoring credentials
+        shell: bash
+        run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery
+
+      - name: Install the repository-approved Kubernetes client
+        shell: bash
+        env:
+          KUBECTL_VERSION: v1.36.2
+          KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
+        run: |
+          set -euo pipefail
+          kubectl_path="$RUNNER_TEMP/email-reader-kubectl"
+          curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \
+            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path"
+          printf '%s  %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check
+          mkdir -p "$RUNNER_TEMP/email-reader-tools"
+          install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-reader-tools/kubectl"
+          printf '%s\n' "$RUNNER_TEMP/email-reader-tools" >> "$GITHUB_PATH"
+
+      - name: Copy and verify the exact retained inputs without pausing
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+          EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }}
+          EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }}
+          EMAIL_READER_INPUTS_CONFIRM: ${{ inputs.confirm }}
+        run: |
+          set +x
+          set -euo pipefail
+          umask 077
+          fail() { printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=workspace restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; }
+          [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
+          private_dir="$(mktemp -d /dev/shm/email-reader-inputs.XXXXXXXX)"
+          forward_pid=
+          cleanup() {
+            if [[ -n "$forward_pid" ]]; then
+              kill "$forward_pid" 2>/dev/null || true
+              wait "$forward_pid" 2>/dev/null || true
+            fi
+            rm -rf "$private_dir"
+          }
+          trap cleanup EXIT
+          [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail
+          export KUBECONFIG="$private_dir/kubeconfig"
+          printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG"
+          unset KUBE_CONFIG
+          # The helper sends GitHub's masking commands on stderr. Preserve
+          # that stream so later native-client output remains redacted.
+          if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then
+            fail
+          fi
+          unset HCLOUD_TOKEN
+          if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" > /dev/null 2>"$private_dir/source.log"; then fail; fi
+          kc=(kubectl --context=admin@prod --request-timeout=20s)
+          if ! "${kc[@]}" -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$private_dir/reader.log" |
+              base64 --decode > "$private_dir/token" 2>"$private_dir/decode.log"; then fail; fi
+          [[ -s "$private_dir/token" ]] || fail
+          "${kc[@]}" -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$private_dir/forward.log" 2>&1 &
+          forward_pid=$!
+          ready=false
+          for ((attempt=0; attempt<20; attempt++)); do
+            kill -0 "$forward_pid" 2>/dev/null || fail
+            if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \
+                http://127.0.0.1:38200/v1/sys/health > "$private_dir/health.json" 2>"$private_dir/health.log"; then
+              if jq -e '.initialized==true and .sealed==false and .standby==false' "$private_dir/health.json" > /dev/null; then ready=true; break; fi
+            fi
+            sleep 1
+          done
+          [[ "$ready" == true ]] || fail
+          export EMAIL_READER_INPUTS_WORK_DIR="$private_dir"
+          "$RUNNER_TEMP/verify-email-data-recovery" --prepare-reader-inputs

.github/workflows/protect-retained-email-volume.yaml

@@ -0,0 +1,100 @@
+name: Protect Retained Email Volume
+
+"on":
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: "Type exactly: protect-retained-email-volume"
+        required: true
+        type: string
+
+permissions: {}
+concurrency:
+  group: prod-deploy
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  protect:
+    name: Retain the fixed recovery volume without stopping its reader
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: prod
+    permissions:
+      contents: read
+    steps:
+      - name: Require the confirmed first-attempt main dispatch
+        shell: bash
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+        run: |
+          if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != protect-retained-email-volume ]]; then
+            printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=dispatch reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2
+            exit 2
+          fi
+
+      - name: Check out the confirmed immutable revision
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Set up the repository Go version
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Build the reviewed command before restoring credentials
+        shell: bash
+        run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery
+
+      - name: Install the repository-approved Kubernetes client
+        shell: bash
+        env:
+          KUBECTL_VERSION: v1.36.2
+          KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
+        run: |
+          set -euo pipefail
+          kubectl_path="$RUNNER_TEMP/email-volume-kubectl"
+          curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \
+            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path"
+          printf '%s  %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check
+          mkdir -p "$RUNNER_TEMP/email-volume-tools"
+          install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-volume-tools/kubectl"
+          printf '%s\n' "$RUNNER_TEMP/email-volume-tools" >> "$GITHUB_PATH"
+
+      - name: Protect only the independently witnessed existing volume
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+          EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }}
+          EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }}
+          EMAIL_RETAINED_PV_UID: ${{ secrets.EMAIL_RETAINED_PV_UID }}
+          EMAIL_RETAINED_VOLUME_CONFIRM: ${{ inputs.confirm }}
+        run: |
+          set +x
+          set -euo pipefail
+          umask 077
+          ulimit -c 0
+          fail() { printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=workspace reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; }
+          uid='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+          [[ "$EMAIL_RETAINED_READER_UID" =~ $uid && "$EMAIL_RETAINED_CLAIM_UID" =~ $uid && "$EMAIL_RETAINED_PV_UID" =~ $uid ]] || fail
+          [[ "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_CLAIM_UID" && "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_PV_UID" && "$EMAIL_RETAINED_CLAIM_UID" != "$EMAIL_RETAINED_PV_UID" ]] || fail
+          [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
+          private_dir="$(mktemp -d /dev/shm/email-volume-protection.XXXXXXXX)"
+          trap 'rm -rf "$private_dir"' EXIT
+          [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail
+          export KUBECONFIG="$private_dir/kubeconfig"
+          printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG"
+          unset KUBE_CONFIG
+          if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then
+            fail
+          fi
+          unset HCLOUD_TOKEN
+          if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" >/dev/null 2>"$private_dir/source.log"; then
+            fail
+          fi
+          "$RUNNER_TEMP/verify-email-data-recovery" --protect-retained-volume

scripts/verify-email-data-recovery/README.md

@@ -92,6 +92,48 @@ comparison. `EMAIL_RETAINED_PARITY=VERIFIED` reports present message parity.
 The original trial stays unverified and disposal still requires its own backup
 and ownership checks.
 
+The separately confirmed `Prepare Retained Email Reader Inputs` workflow
+prepares custody for the original retained reader. It runs only on the first
+maintainer-dispatched attempt from current reviewed main, under the production
+deployment lock. It binds the independently recorded reader and claim, reads
+the exact retained encryption key, recovery authentication, configuration and
+declared workload inputs, and rechecks their identities and contents immediately
+before the one conditional OpenBao write. Kubernetes access is read-only.
+
+The destination is a create-only, original-version KV2 record under
+`infrastructure/email/recovery/retained-reader-inputs`. The command uses CAS=0;
+an identical existing record is read back without rewriting it. The command
+never updates or deletes the record. Changed,
+deleted, newer, malformed or unreadable records refuse preparation. A failed
+or lost write response is uncertain and is not retried. Original inputs remain
+in place on every outcome. Neither credentials nor resource identities are
+printed; the short-lived credential files exist only in runner memory.
+
+`EMAIL_READER_INPUT_CUSTODY=STORED` proves the observed record matches those
+retained inputs. Its receipt explicitly says `restart=NOT_AUTHORIZED` and
+`recovery=UNVERIFIED`: independent recovery of the stored inputs, a durable
+restart witness, the separate protected pause/resume operations and native
+detachment proof remain required. This workflow never stops the reader,
+allocates storage, changes the source mailbox or grants disposal authority.
+The expired-backup disposal gate remains unchanged. If custody is refused,
+investigate the retained identities and destination before another confirmed
+first attempt; never replace a conflicting record or regenerate a missing key.
+
+`Protect Retained Email Volume` is a separate, default-off main-only operation
+with confirmation `protect-retained-email-volume`. It requires independent
+reader, claim and PV UID witnesses, including the private
+`EMAIL_RETAINED_PV_UID` setting, and validates the actual restored binding.
+It deliberately does not assume that the original PV name contains the current
+restored claim's UID. The command preserves the running reader and every
+claim, backend, key and configuration object. Its only permitted mutation is
+changing the bound original PV's reclaim policy from `Delete` to `Retain`,
+with atomic UID, name, resource-version and complete-spec tests. It then
+rechecks the original graph and serving source. An already-retained PV is
+verified without a write; an uncertain patch is never retried or rolled back.
+No credential export or reader lifecycle authority is available in this
+command. Protection does not unlock either expired-backup disposal path,
+establish key recovery or authorize a later reader pause.
+
 With confirmation `verify-fresh-retained-email-parity`, the workflow checks
 the separately retained fresh restore. The protected witness binds the
 namespace, reader, claim, backup and download. The backup must still be

scripts/verify-email-data-recovery/main.go

@@ -12,7 +12,18 @@ import (
 	"time"
 )
 
+// main selects one independently authorized recovery operation; unknown flags fail closed.
 func main() {
+	if len(os.Args) == 2 && os.Args[1] == "--protect-retained-volume" {
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		os.Exit(runRetainedVolumeProtection(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, os.Stdout))
+	}
+	if len(os.Args) == 2 && os.Args[1] == "--prepare-reader-inputs" {
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		os.Exit(runReaderInputCustody(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, loadReaderCustodyStore, os.Stdout))
+	}
 	if len(os.Args) == 2 && os.Args[1] == "--dispose-retained" {
 		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 		defer stop()