From c6989e9df364de697a1b6a5e5fc1c7b0d489a12c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 16:11:05 +0100 Subject: [PATCH 1/7] feat(email): bind retained reader recovery inputs Part of #6048. Prepare immutable input custody before the separate lifecycle and persistence operations. --- .../reader_inputs.go | 121 ++++++++++++++ .../reader_inputs_test.go | 149 ++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 scripts/verify-email-data-recovery/reader_inputs.go create mode 100644 scripts/verify-email-data-recovery/reader_inputs_test.go diff --git a/scripts/verify-email-data-recovery/reader_inputs.go b/scripts/verify-email-data-recovery/reader_inputs.go new file mode 100644 index 0000000000..90eb7e0d79 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_inputs.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "reflect" + "regexp" +) + +type retainedReaderInput struct { + Kind, Name, Namespace, UID, Digest string + Values object +} + +type retainedReaderInputs struct { + ReaderUID, ClaimUID string + Inputs []retainedReaderInput +} + +// This captures custody inputs only. It neither establishes durable custody +// nor authorizes a pause, restart, detach or disposal. In particular it does +// not replace the original backup-validity gate with a credential snapshot. +// Values contain credentials and must stay in the protected private workspace. +func captureRetainedReaderInputs(i invocation, image, readerUID, claimUID, sourceClaimUID string, graph []object) (retainedReaderInputs, error) { + fail := func() (retainedReaderInputs, error) { return retainedReaderInputs{}, errors.New("reader-inputs") } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + imagePin := regexp.MustCompile(`^ghcr\.io/devantler-tech/platform-private-mail-bootstrap@sha256:[0-9a-f]{64}$`) + if i != retainedRecovery || !imagePin.MatchString(image) || !uuid.MatchString(readerUID) || !uuid.MatchString(claimUID) || !uuid.MatchString(sourceClaimUID) || readerUID == claimUID || claimUID == sourceClaimUID { + return fail() + } + wanted := append(recoveryResources(i, "", ""), recoveryReader(i, image)) + if len(graph) != len(wanted) { + return fail() + } + actualByName := map[string]object{} + seenUID := map[string]bool{} + for _, actual := range graph { + m := child(actual, "metadata") + uid := text(m, "uid") + ref := ownedReference{kind: text(actual, "kind"), name: text(m, "name"), namespace: text(m, "namespace")} + if !uuid.MatchString(uid) || seenUID[uid] || !regexp.MustCompile(`^[1-9][0-9]{0,31}$`).MatchString(text(m, "resourceVersion")) || m["deletionTimestamp"] != nil || actualByName[ref.key()] != nil || !i.permitsMutation(ref.kind, ref.namespace, ref.name) { + return fail() + } + if owners, exists := m["ownerReferences"]; exists && owners != nil { + list, ok := owners.([]any) + if !ok || len(list) != 0 { + return fail() + } + } + actualByName[ref.key()] = actual + seenUID[uid] = true + } + bundle := retainedReaderInputs{ReaderUID: readerUID, ClaimUID: claimUID} + for _, want := range wanted { + m := child(want, "metadata") + ref := ownedReference{kind: text(want, "kind"), name: text(m, "name"), namespace: text(m, "namespace")} + actual := actualByName[ref.key()] + if actual == nil || text(actual, "apiVersion") != text(want, "apiVersion") { + return fail() + } + am := child(actual, "metadata") + for key, value := range child(m, "labels") { + if child(am, "labels")[key] != value { + return fail() + } + } + switch ref.kind { + case "Pod": + if text(am, "uid") != readerUID || text(child(actual, "status"), "phase") != "Running" || !admittedRecoveryObject(want, actual) || child(actual, "spec")["terminationGracePeriodSeconds"] != float64(30) { + return fail() + } + ready := false + for _, condition := range items(child(actual, "status"), "conditions") { + if text(condition, "type") == "Ready" && text(condition, "status") == "True" { + ready = true + } + } + if !ready { + return fail() + } + case "PersistentVolumeClaim": + spec := child(actual, "spec") + if text(am, "uid") != claimUID || text(child(actual, "status"), "phase") != "Bound" || !kubernetesName.MatchString(text(spec, "volumeName")) || text(spec, "storageClassName") != i.namespace || !reflect.DeepEqual(spec["accessModes"], []any{"ReadWriteOnce"}) || text(child(child(spec, "resources"), "requests"), "storage") != "2Gi" { + return fail() + } + case "Secret": + if !retainedSecret(actual, ref.name) || actual["stringData"] != nil { + return fail() + } + case "ConfigMap": + if !reflect.DeepEqual(actual["data"], want["data"]) || len(child(actual, "binaryData")) != 0 { + return fail() + } + case "StorageClass": + for _, key := range []string{"provisioner", "reclaimPolicy", "volumeBindingMode", "parameters"} { + if !reflect.DeepEqual(actual[key], want[key]) { + return fail() + } + } + case "CiliumNetworkPolicy": + if !reflect.DeepEqual(actual["spec"], want["spec"]) { + return fail() + } + } + // Marshal/unmarshal creates a private snapshot: later API reads cannot + // mutate values already accepted for the conditional custody write. + canonical, err := json.Marshal(retainedMutableFields(actual)) + if err != nil { + return fail() + } + var frozen object + if json.Unmarshal(canonical, &frozen) != nil { + return fail() + } + digest := sha256.Sum256(canonical) + bundle.Inputs = append(bundle.Inputs, retainedReaderInput{Kind: ref.kind, Name: ref.name, Namespace: ref.namespace, UID: text(am, "uid"), Digest: hex.EncodeToString(digest[:]), Values: frozen}) + } + return bundle, nil +} diff --git a/scripts/verify-email-data-recovery/reader_inputs_test.go b/scripts/verify-email-data-recovery/reader_inputs_test.go new file mode 100644 index 0000000000..fe62222c24 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_inputs_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" +) + +// The independent witnesses identify the existing copy; labels alone must +// never select which credentials are saved for a later restart. +func readerInputFixture(t *testing.T) ([]object, string, string) { + t.Helper() + graph := append(recoveryResources(retainedRecovery, strings.Repeat("a", 64), "restore:"+strings.Repeat("b", 64)), recoveryReader(retainedRecovery, fixtureImage)) + var readerUID, claimUID string + for index, o := range graph { + m := child(o, "metadata") + m["uid"] = fmt.Sprintf("%08x-1111-2222-3333-444444444444", index+1) + m["resourceVersion"] = fmt.Sprint(index + 100) + if text(o, "kind") == "Secret" { + data := object{} + for k, v := range child(o, "stringData") { + data[k] = base64.StdEncoding.EncodeToString([]byte(v.(string))) + } + o["data"] = data + delete(o, "stringData") + } + if text(o, "kind") == "Pod" { + readerUID = text(m, "uid") + o["status"] = object{"phase": "Running", "conditions": []any{object{"type": "Ready", "status": "True"}}} + } + if text(o, "kind") == "PersistentVolumeClaim" { + claimUID = text(m, "uid") + child(o, "spec")["volumeName"] = "pvc-existing-recovery" + o["status"] = object{"phase": "Bound"} + } + } + return graph, readerUID, claimUID +} + +func TestReaderInputBundleKeepsTheActualPerCopyCredentials(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + bundle, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil { + t.Fatalf("independently bound retained inputs refused: %v", err) + } + if bundle.ReaderUID != reader || bundle.ClaimUID != claim || len(bundle.Inputs) != 8 { + t.Fatal("input custody lost the independent reader or complete restart inputs") + } + secrets := 0 + for _, input := range bundle.Inputs { + if len(input.Digest) != 64 || input.UID == "" { + t.Fatal("restart input lacks a frozen identity/content binding") + } + if input.Kind == "Secret" { + secrets++ + if !reflect.DeepEqual(child(input.Values, "data"), child(graph[map[string]int{"storage-key": 3, "test-authentication": 4}[input.Name]], "data")) { + t.Fatal("operational credentials substituted for this copy's inputs") + } + } + } + if secrets != 2 { + t.Fatal("both retained encryption and recovery authentication are required") + } + graph[0], graph[7] = graph[7], graph[0] + reordered, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil || !reflect.DeepEqual(bundle, reordered) { + t.Fatal("discovery order changed the retained-input identity") + } + before, err := json.Marshal(bundle) + if err != nil { + t.Fatal(err) + } + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = "changed-after-capture" + after, err := json.Marshal(bundle) + if err != nil || string(before) != string(after) { + t.Fatal("later live mutations rewrote the captured custody inputs") + } +} + +func TestReaderInputBundleBindsValidChangedKeyBytes(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + source := "ffffffff-1111-2222-3333-444444444444" + before, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, source, graph) + if err != nil { + t.Fatal(err) + } + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = base64.StdEncoding.EncodeToString([]byte(strings.Repeat("c", 64))) + after, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, source, graph) + if err != nil { + t.Fatal(err) + } + if before.Inputs[3].Digest == after.Inputs[3].Digest { + t.Fatal("a different valid encryption key inherited the original content binding") + } +} + +func TestReaderInputBundleRefusesBorrowedReplacedOrIncompleteInputs(t *testing.T) { + for _, failure := range []string{"reader", "claim", "source", "missing", "duplicate", "key", "authentication", "configuration", "storage-class", "executable", "not-ready", "terminating", "controller", "namespace", "identity", "resource-version", "labels", "foreign-target"} { + t.Run(failure, func(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + source := "ffffffff-1111-2222-3333-444444444444" + target := retainedRecovery + switch failure { + case "reader": + reader = "eeeeeeee-1111-2222-3333-444444444444" + case "claim": + claim = "eeeeeeee-1111-2222-3333-444444444444" + case "source": + source = claim + case "missing": + graph = graph[:len(graph)-1] + case "duplicate": + graph = append(graph, graph[0]) + case "key": + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = "not-base64" + case "authentication": + child(graph[4], "data")["credential"] = base64.StdEncoding.EncodeToString([]byte("another-account")) + case "configuration": + child(graph[5], "data")["config.json"] = `{}` + case "storage-class": + child(graph[2], "parameters")["csi.storage.k8s.io/node-stage-secret-namespace"] = "private-mail" + case "executable": + items(child(graph[7], "spec"), "containers")[0]["command"] = []any{"unreviewed"} + case "not-ready": + graph[7]["status"] = object{"phase": "Pending"} + case "terminating": + child(graph[3], "metadata")["deletionTimestamp"] = "2026-01-01T00:00:00Z" + case "controller": + child(graph[7], "metadata")["ownerReferences"] = []any{object{"controller": true}} + case "namespace": + child(graph[3], "metadata")["namespace"] = "private-mail" + case "identity": + child(graph[4], "metadata")["uid"] = child(graph[3], "metadata")["uid"] + case "resource-version": + delete(child(graph[3], "metadata"), "resourceVersion") + case "labels": + child(child(graph[3], "metadata"), "labels")[recoveryLabel] = "another-run" + case "foreign-target": + target.run = "12345678" + } + if _, err := captureRetainedReaderInputs(target, fixtureImage, reader, claim, source, graph); err == nil { + t.Fatal("unqualified input bundle could enter recovery custody") + } + }) + } +} From 13dcc20d1f68acf874806695ebf34cc03dbfcff2 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 17:08:43 +0100 Subject: [PATCH 2/7] feat(email): prepare retained reader inputs in protected custody Part of #6048. Add the confirmed main-only OpenBao preparation path, exact create-only readback, fresh source and input binding, read-only native capability, and bounded private credential staging. Keep restart and recovery authority separate. --- .../prepare-retained-email-reader-inputs.yaml | 116 +++++++ scripts/verify-email-data-recovery/README.md | 27 ++ scripts/verify-email-data-recovery/main.go | 5 + .../reader_custody_entry.go | 164 ++++++++++ .../reader_custody_entry_test.go | 194 ++++++++++++ .../reader_custody_runtime.go | 78 +++++ .../reader_custody_runtime_test.go | 98 ++++++ .../reader_custody_store.go | 189 +++++++++++ .../reader_custody_store_test.go | 296 ++++++++++++++++++ .../retained_workflow_test.go | 24 +- 10 files changed, 1185 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/prepare-retained-email-reader-inputs.yaml create mode 100644 scripts/verify-email-data-recovery/reader_custody_entry.go create mode 100644 scripts/verify-email-data-recovery/reader_custody_entry_test.go create mode 100644 scripts/verify-email-data-recovery/reader_custody_runtime.go create mode 100644 scripts/verify-email-data-recovery/reader_custody_runtime_test.go create mode 100644 scripts/verify-email-data-recovery/reader_custody_store.go create mode 100644 scripts/verify-email-data-recovery/reader_custody_store_test.go diff --git a/.github/workflows/prepare-retained-email-reader-inputs.yaml b/.github/workflows/prepare-retained-email-reader-inputs.yaml new file mode 100644 index 0000000000..13c9d697da --- /dev/null +++ b/.github/workflows/prepare-retained-email-reader-inputs.yaml @@ -0,0 +1,116 @@ +name: Prepare Retained Email Reader Inputs + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: prepare-retained-email-reader-inputs" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + prepare: + name: Preserve the fixed retained reader inputs + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != prepare-retained-email-reader-inputs ]]; then + printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=dispatch restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Build the reviewed command before restoring credentials + shell: bash + run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery + + - name: Install the repository-approved Kubernetes client + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-reader-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-reader-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-reader-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-reader-tools" >> "$GITHUB_PATH" + + - name: Copy and verify the exact retained inputs without pausing + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} + EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_READER_INPUTS_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + fail() { printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=workspace restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; } + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + private_dir="$(mktemp -d /dev/shm/email-reader-inputs.XXXXXXXX)" + forward_pid= + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$private_dir" + } + trap cleanup EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + if ! ./scripts/use-prod-stable-api-endpoint.sh > /dev/null 2>"$private_dir/endpoint.log"; then fail; fi + unset HCLOUD_TOKEN + if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" > /dev/null 2>"$private_dir/source.log"; then fail; fi + kc=(kubectl --context=admin@prod --request-timeout=20s) + if ! "${kc[@]}" -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$private_dir/reader.log" | + base64 --decode > "$private_dir/token" 2>"$private_dir/decode.log"; then fail; fi + [[ -s "$private_dir/token" ]] || fail + "${kc[@]}" -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$private_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > "$private_dir/health.json" 2>"$private_dir/health.log"; then + if jq -e '.initialized==true and .sealed==false and .standby==false' "$private_dir/health.json" > /dev/null; then ready=true; break; fi + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + export EMAIL_READER_INPUTS_WORK_DIR="$private_dir" + "$RUNNER_TEMP/verify-email-data-recovery" --prepare-reader-inputs diff --git a/scripts/verify-email-data-recovery/README.md b/scripts/verify-email-data-recovery/README.md index 8f38a7f065..9ff5c5391b 100644 --- a/scripts/verify-email-data-recovery/README.md +++ b/scripts/verify-email-data-recovery/README.md @@ -92,6 +92,33 @@ comparison. `EMAIL_RETAINED_PARITY=VERIFIED` reports present message parity. The original trial stays unverified and disposal still requires its own backup and ownership checks. +The separately confirmed `Prepare Retained Email Reader Inputs` workflow +prepares custody for the original retained reader. It runs only on the first +maintainer-dispatched attempt from current reviewed main, under the production +deployment lock. It binds the independently recorded reader and claim, reads +the exact retained encryption key, recovery authentication, configuration and +declared workload inputs, and rechecks their identities and contents immediately +before the one conditional OpenBao write. Kubernetes access is read-only. + +The destination is a create-only, original-version KV2 record under +`infrastructure/email/recovery/retained-reader-inputs`. The command uses CAS=0; +an identical existing record is read back without rewriting it. The command +never updates or deletes the record. Changed, +deleted, newer, malformed or unreadable records refuse preparation. A failed +or lost write response is uncertain and is not retried. Original inputs remain +in place on every outcome. Neither credentials nor resource identities are +printed; the short-lived credential files exist only in runner memory. + +`EMAIL_READER_INPUT_CUSTODY=STORED` proves the observed record matches those +retained inputs. Its receipt explicitly says `restart=NOT_AUTHORIZED` and +`recovery=UNVERIFIED`: independent recovery of the stored inputs, a durable +restart witness, the separate protected pause/resume operations and native +detachment proof remain required. This workflow never stops the reader, +allocates storage, changes the source mailbox or grants disposal authority. +The expired-backup disposal gate remains unchanged. If custody is refused, +investigate the retained identities and destination before another confirmed +first attempt; never replace a conflicting record or regenerate a missing key. + With confirmation `verify-fresh-retained-email-parity`, the workflow checks the separately retained fresh restore. The protected witness binds the namespace, reader, claim, backup and download. The backup must still be diff --git a/scripts/verify-email-data-recovery/main.go b/scripts/verify-email-data-recovery/main.go index 2563cc74f6..c3c9259e20 100644 --- a/scripts/verify-email-data-recovery/main.go +++ b/scripts/verify-email-data-recovery/main.go @@ -13,6 +13,11 @@ import ( ) func main() { + if len(os.Args) == 2 && os.Args[1] == "--prepare-reader-inputs" { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + os.Exit(runReaderInputCustody(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, loadReaderCustodyStore, os.Stdout)) + } if len(os.Args) == 2 && os.Args[1] == "--dispose-retained" { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/scripts/verify-email-data-recovery/reader_custody_entry.go b/scripts/verify-email-data-recovery/reader_custody_entry.go new file mode 100644 index 0000000000..96952f4ade --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_entry.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "strings" + "time" +) + +var errReaderInputRead = errors.New("reader-input-read") + +type readerInputKubernetes struct{ inner kubernetes } + +// This command has only fixed native reads. Lifecycle operations cannot borrow +// its credential-custody authority to mutate a Pod or persistent object. +func (k readerInputKubernetes) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + if k.inner == nil || len(input) != 0 || (len(args) != 5 && len(args) != 6) || args[0] != "get" || args[3] != "--ignore-not-found=true" || args[4] != "-o=json" { + return nil, errReaderInputRead + } + namespace := "" + if len(args) == 6 { + if !strings.HasPrefix(args[5], "--namespace=") { + return nil, errReaderInputRead + } + namespace = strings.TrimPrefix(args[5], "--namespace=") + } + name, allowed := args[2], false + switch args[1] { + case "Pod": + allowed = (namespace == retainedRecovery.namespace && name == "reader") || (namespace == "private-mail" && name == "private-mail-0") + case "PersistentVolumeClaim": + allowed = (namespace == retainedRecovery.namespace && name == "data") || (namespace == "private-mail" && name == "private-mail-data") + case "PersistentVolume": + allowed = namespace == "" && strings.HasPrefix(name, "pvc-") && kubernetesName.MatchString(name) + case "Namespace", "StorageClass": + allowed = namespace == "" && name == retainedRecovery.namespace + case "CiliumNetworkPolicy": + allowed = namespace == retainedRecovery.namespace && name == "isolation" + case "ConfigMap": + allowed = namespace == retainedRecovery.namespace && name == "datastore" + case "Secret": + allowed = namespace == retainedRecovery.namespace && (name == "storage-key" || name == "test-authentication") + } + if !allowed { + return nil, errReaderInputRead + } + return k.inner.call(ctx, nil, args...) +} + +func authorizeReaderInputCustody(env func(string) string) error { + for key, expected := range map[string]string{ + "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", + "GITHUB_ACTOR": "devantler", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main", + "EMAIL_READER_INPUTS_CONFIRM": "prepare-retained-email-reader-inputs", + } { + if env(key) != expected { + return errors.New("reader-input-dispatch") + } + } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + reader, claim := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + if !recoveryRunPattern.MatchString(env("GITHUB_RUN_ID")) || !recoveryRevisionPattern.MatchString(env("GITHUB_SHA")) || !uuid.MatchString(reader) || !uuid.MatchString(claim) || reader == claim { + return errors.New("reader-input-dispatch") + } + return nil +} + +func runReaderInputCustody(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, loadStore func(func(string) string) (*readerCustodyStore, error), output io.Writer) int { + stage := "dispatch" + fail := func() int { + if output != nil { + _, _ = fmt.Fprintf(output, "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=%s restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", stage) + } + return 2 + } + if output == nil || verifyMain == nil || loadStore == nil || authorizeReaderInputCustody(env) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_READER_INPUT_CUSTODY=STARTED source_writes=0 restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + stage = "source" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + d := &recoveryDriver{i: retainedRecovery, k: readerInputKubernetes{inner: k}} + var err error + d.source, err = d.readSource(ctx) + if err != nil { + return fail() + } + reader, claim := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + stage = "inputs" + original, err := d.captureCustodyInputs(ctx, reader, claim) + if err != nil { + return fail() + } + stage = "credentials" + store, err := loadStore(env) + if err != nil || store == nil || store.client == nil { + return fail() + } + defer store.client.CloseIdleConnections() + verify := func(ctx context.Context) error { + // Rebind reviewed source before the last native reads. The store calls + // this after its final non-destructive preflight, just before CAS=0. + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return errors.New("custody-source-changed") + } + fresh, err := d.captureCustodyInputs(ctx, reader, claim) + if err != nil || !reflect.DeepEqual(original, fresh) { + return errors.New("custody-inputs-changed") + } + return nil + } + stage = "custody" + if store.ensure(ctx, original, verify) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_READER_INPUT_CUSTODY=STORED source_writes=0 restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + return 0 +} + +func (d *recoveryDriver) captureCustodyInputs(ctx context.Context, readerUID, claimUID string) (retainedReaderInputs, error) { + fail := func() (retainedReaderInputs, error) { return retainedReaderInputs{}, errors.New("reader-inputs") } + reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) + if err != nil || !found || text(child(reader, "metadata"), "uid") != readerUID { + return fail() + } + claim, found, err := d.lookup(ctx, "PersistentVolumeClaim", "data", retainedRecovery.namespace) + if err != nil || !found || text(child(claim, "metadata"), "uid") != claimUID || claimUID == d.source.claimUID || text(child(claim, "spec"), "volumeName") == d.source.pv { + return fail() + } + var graph []object + for _, want := range append(recoveryResources(retainedRecovery, "", ""), recoveryReader(retainedRecovery, d.image)) { + m := child(want, "metadata") + actual := reader + switch text(want, "kind") { + case "Pod": + case "PersistentVolumeClaim": + actual = claim + default: + actual, found, err = d.lookup(ctx, text(want, "kind"), text(m, "name"), text(m, "namespace")) + if err != nil || !found { + return fail() + } + } + graph = append(graph, actual) + } + bundle, err := captureRetainedReaderInputs(retainedRecovery, d.image, readerUID, claimUID, d.source.claimUID, graph) + if err != nil || d.sourceRetained(ctx) != nil { + return fail() + } + return bundle, nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_entry_test.go b/scripts/verify-email-data-recovery/reader_custody_entry_test.go new file mode 100644 index 0000000000..7030d09898 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_entry_test.go @@ -0,0 +1,194 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func readerCustodyDispatch() map[string]string { + env := confirmedInvocation() + env["GITHUB_ACTIONS"] = "true" + env["GITHUB_WORKFLOW_REF"] = "devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main" + env["EMAIL_READER_INPUTS_CONFIRM"] = "prepare-retained-email-reader-inputs" + return env +} + +func readerCustodyNativeFixture(t *testing.T) (*cleanupAPI, string, string) { + t.Helper() + t.Chdir("../..") + graph, reader, claim := readerInputFixture(t) + image, err := loadRecoveryImage() + if err != nil { + t.Fatal(err) + } + items(child(graph[7], "spec"), "containers")[0]["image"] = image + a := &cleanupAPI{objects: map[string]object{}} + for _, o := range graph { + m := child(o, "metadata") + a.objects[cleanupKey(text(o, "kind"), text(m, "namespace"), text(m, "name"))] = o + } + pod, pvc, pv := servingFixture() + items(child(pod, "spec"), "containers")[0]["image"] = image + child(pvc, "metadata")["uid"] = "ffffffff-1111-2222-3333-444444444444" + child(pvc, "spec")["volumeName"] = "pvc-serving-volume" + child(pv, "metadata")["name"] = "pvc-serving-volume" + child(child(pv, "spec"), "claimRef")["uid"] = child(pvc, "metadata")["uid"] + child(child(pv, "spec"), "csi")["volumeHandle"] = "pvc-serving-volume" + a.objects[cleanupKey("Pod", "private-mail", "private-mail-0")] = pod + a.objects[cleanupKey("PersistentVolumeClaim", "private-mail", "private-mail-data")] = pvc + a.objects[cleanupKey("PersistentVolume", "", "pvc-serving-volume")] = pv + return a, reader, claim +} + +func TestReaderCustodyDispatchRefusesBeforeAnyAccess(t *testing.T) { + for _, invalid := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_RUN_ATTEMPT", "GITHUB_ACTOR", "GITHUB_WORKFLOW_REF", "RUNNER_ENVIRONMENT", "GITHUB_SHA", "GITHUB_RUN_ID", "EMAIL_READER_INPUTS_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID"} { + t.Run(invalid, func(t *testing.T) { + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "11111111-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = "22222222-1111-2222-3333-444444444444" + env[invalid] = "invalid" + k := &refusedAPI{} + callbacks := 0 + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, k, + func(context.Context, string) error { callbacks++; return nil }, + func(func(string) string) (*readerCustodyStore, error) { callbacks++; return nil, nil }, &out) + if code != 2 || k.calls != 0 || callbacks != 0 || !strings.Contains(out.String(), "stage=dispatch") { + t.Fatal("foreign dispatch reached credentials, native resources or source execution") + } + }) + } +} + +func TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject(t *testing.T) { + for _, changeAt := range []int{0, 2, 3} { + t.Run(map[int]string{0: "stable", 2: "changed-before-write", 3: "changed-after-write"}[changeAt], func(t *testing.T) { + a, reader, claim := readerCustodyNativeFixture(t) + before, err := json.Marshal(a.objects) + if err != nil { + t.Fatal(err) + } + posts, sourceChecks := 0, 0 + var saved object + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + var payload object + if json.NewDecoder(r.Body).Decode(&payload) != nil || child(payload, "options")["cas"] != float64(0) { + t.Error("operator escaped create-only custody") + } + saved = child(payload, "data") + _, _ = w.Write([]byte(`{"data":{"version":1}}`)) + return + } + if r.URL.Path == retainedReaderDataPath && saved != nil { + custodyResponse(t, w, saved, 1) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"], env["EMAIL_RETAINED_CLAIM_UID"] = reader, claim + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, a, + func(context.Context, string) error { + sourceChecks++ + if changeAt != 0 && sourceChecks == changeAt { + child(a.objects[cleanupKey("ConfigMap", retainedRecovery.namespace, "datastore")], "data")["config.json"] = `{}` + } + return nil + }, func(func(string) string) (*readerCustodyStore, error) { + return newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + }, &out) + if changeAt != 0 { + expectedPosts := 0 + if changeAt == 3 { + expectedPosts = 1 + } + if code != 2 || posts != expectedPosts || strings.Contains(out.String(), "CUSTODY=STORED") { + t.Fatal("changed native inputs produced a write or a successful custody receipt") + } + return + } + after, err := json.Marshal(a.objects) + if code != 0 || posts != 1 || sourceChecks != 3 || err != nil || !bytes.Equal(before, after) { + t.Fatalf("operator failed its read-only native boundary: code=%d posts=%d checks=%d output=%q", code, posts, sourceChecks, out.String()) + } + if !strings.Contains(out.String(), "restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(out.String(), reader) || strings.Contains(out.String(), "CRYPTO_KEY_VALUE") || strings.Contains(out.String(), "fixture-reader-custody-token") { + t.Fatal("custody receipt exposed inputs or granted lifecycle/recovery authority") + } + }) + } +} + +func TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead(t *testing.T) { + for _, args := range [][]string{ + {"delete", "Pod", "reader"}, {"create", "-f=-"}, {"patch", "Secret", "storage-key"}, + {"get", "Secret", "openbao-unseal", "--ignore-not-found=true", "-o=json", "--namespace=openbao"}, + {"get", "Pod", "reader", "--ignore-not-found=true", "-o=json", "--namespace=private-mail"}, + {"get", "Secret", "storage-key", "--ignore-not-found=true", "-o=json", "--namespace=" + retainedRecovery.namespace, "--raw=/"}, + } { + k := &refusedAPI{} + _, err := (readerInputKubernetes{inner: k}).call(context.Background(), nil, args...) + if err == nil || k.calls != 0 { + t.Fatal("input custody capability reached a mutation or foreign resource") + } + } + if _, err := (readerInputKubernetes{}).call(context.Background(), nil); !errors.Is(err, errReaderInputRead) { + t.Fatal("missing native reader did not fail closed") + } +} + +func TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess(t *testing.T) { + for _, failure := range []string{"output", "source"} { + t.Run(failure, func(t *testing.T) { + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "11111111-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = "22222222-1111-2222-3333-444444444444" + k := &refusedAPI{} + var out bytes.Buffer + writer := &observationFailingWriter{base: &out, remaining: 10} + if failure == "output" { + writer.remaining = 0 + } + loaded := 0 + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, k, + func(context.Context, string) error { return errors.New("unverified current main") }, + func(func(string) string) (*readerCustodyStore, error) { loaded++; return nil, nil }, writer) + if code != 2 || k.calls != 0 || loaded != 0 || strings.Contains(out.String(), "CUSTODY=STORED") { + t.Fatal("lost output or source proof reached native credentials") + } + }) + } +} + +func TestReaderCustodyWrongIndependentReaderStopsBeforeKeyAccess(t *testing.T) { + a, _, claim := readerCustodyNativeFixture(t) + secretReads, loaded := 0, 0 + a.onGet = func(key string) { + if strings.HasPrefix(key, "Secret/") { + secretReads++ + } + } + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "eeeeeeee-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = claim + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, a, + func(context.Context, string) error { return nil }, + func(func(string) string) (*readerCustodyStore, error) { loaded++; return nil, nil }, &out) + if code != 2 || secretReads != 0 || loaded != 0 || !strings.Contains(out.String(), "stage=inputs") { + t.Fatal("a replaced reader reached encryption or store credentials") + } +} diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime.go b/scripts/verify-email-data-recovery/reader_custody_runtime.go new file mode 100644 index 0000000000..afadefe641 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_runtime.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "time" +) + +func readReaderCustodyToken(directory string) (token string, err error) { + fail := func() (string, error) { return "", errors.New("reader-custody-token") } + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + return fail() + } + info, err = os.Lstat(filepath.Join(directory, "token")) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() < 16 || info.Size() > 256 { + return fail() + } + root, err := os.OpenRoot(directory) + if err != nil { + return fail() + } + defer func() { + if root.Close() != nil { + token, err = fail() + } + }() + file, err := root.Open("token") + if err != nil { + return fail() + } + defer func() { + if file.Close() != nil { + token, err = fail() + } + }() + info, err = file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { + return fail() + } + raw, err := io.ReadAll(io.LimitReader(file, 257)) + if err != nil || !regexp.MustCompile(`^[A-Za-z0-9._-]{16,256}$`).Match(raw) { + return fail() + } + return string(raw), nil +} + +func loadReaderCustodyStore(env func(string) string) (*readerCustodyStore, error) { + directory := env("EMAIL_READER_INPUTS_WORK_DIR") + if !regexp.MustCompile(`^/dev/shm/email-reader-inputs\.[A-Za-z0-9]{8}$`).MatchString(directory) || filepath.Clean(directory) != directory { + return nil, errors.New("reader-custody-workspace") + } + token, err := readReaderCustodyToken(directory) + if err != nil { + return nil, errors.New("reader-custody-token") + } + return newReaderCustodyStore("http://127.0.0.1:38200", token) +} + +func verifyReaderCustodySource(ctx context.Context, revision string) error { + if !recoveryRevisionPattern.MatchString(revision) { + return errors.New("reader-custody-source") + } + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "./scripts/verify-prod-recovery-source.sh", revision) + command.Stdout, command.Stderr = io.Discard, io.Discard + command.WaitDelay = 5 * time.Second + if command.Run() != nil { + return errors.New("reader-custody-source") + } + return nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime_test.go b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go new file mode 100644 index 0000000000..cec0be8cc3 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestReaderCustodyTokenRequiresPrivateRegularLocalFiles(t *testing.T) { + for _, failure := range []string{"", "public-directory", "public-file", "file-symlink", "directory-symlink", "empty", "oversized", "newline"} { + t.Run(failure, func(t *testing.T) { + dir := t.TempDir() + if os.Chmod(dir, 0700) != nil { + t.Fatal("private fixture directory") + } + value := "fixture-reader-custody-token" + switch failure { + case "empty": + value = "" + case "oversized": + value = strings.Repeat("x", 257) + case "newline": + value += "\n" + } + path := filepath.Join(dir, "token") + if os.WriteFile(path, []byte(value), 0600) != nil { + t.Fatal("private fixture credential") + } + switch failure { + case "public-directory": + if os.Chmod(dir, 0755) != nil { + t.Fatal("fixture mode") + } + case "public-file": + if os.Chmod(path, 0644) != nil { + t.Fatal("fixture mode") + } + case "file-symlink": + if os.Rename(path, path+"-real") != nil || os.Symlink(path+"-real", path) != nil { + t.Fatal("fixture symlink") + } + case "directory-symlink": + link := dir + "-link" + if os.Symlink(dir, link) != nil { + t.Fatal("fixture directory symlink") + } + t.Cleanup(func() { _ = os.Remove(link) }) + dir = link + } + token, err := readReaderCustodyToken(dir) + if failure == "" { + if err != nil || token != value { + t.Fatal("private bounded credential refused") + } + } else if err == nil || token != "" || strings.Contains(err.Error(), value) && value != "" { + t.Fatal("unqualified credential accepted or exposed") + } + }) + } +} + +func TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints(t *testing.T) { + for _, endpoint := range []string{"https://127.0.0.1:38200", "http://example.invalid:38200", "http://localhost:38200", "http://127.0.0.1", "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://user@127.0.0.1:38200", "http://127.0.0.1:38200/path", "http://127.0.0.1:38200?", "http://127.0.0.1:38200#other"} { + if store, err := newReaderCustodyStore(endpoint, "fixture-reader-custody-token"); err == nil || store != nil { + t.Fatal("custody credential could reach an unqualified endpoint") + } + } + if _, err := newReaderCustodyStore("http://127.0.0.1:38200", "token\r\nInjected: value"); err == nil { + t.Fatal("malformed authentication accepted") + } +} + +func TestReaderCustodyCommandUsesTheSeparateProtectedDispatch(t *testing.T) { + if os.Getenv("EMAIL_READER_CUSTODY_CHILD") == "1" { + os.Args = []string{"verify-email-data-recovery", "--prepare-reader-inputs"} + main() + return + } + for _, confirmed := range []bool{false, true} { + command := exec.Command(os.Args[0], "-test.run=^TestReaderCustodyCommandUsesTheSeparateProtectedDispatch$") + command.Env = []string{"EMAIL_READER_CUSTODY_CHILD=1", "PATH=/nonexistent", "GOCOVERDIR=" + t.TempDir()} + stage := "dispatch" + if confirmed { + for key, value := range readerCustodyDispatch() { + command.Env = append(command.Env, key+"="+value) + } + command.Env = append(command.Env, "EMAIL_RETAINED_READER_UID=11111111-1111-2222-3333-444444444444", "EMAIL_RETAINED_CLAIM_UID=22222222-1111-2222-3333-444444444444") + stage = "source" + } + output, err := command.CombinedOutput() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage="+stage+" restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + t.Fatalf("custody command lost its distinct dispatch boundary: %q", output) + } + } +} diff --git a/scripts/verify-email-data-recovery/reader_custody_store.go b/scripts/verify-email-data-recovery/reader_custody_store.go new file mode 100644 index 0000000000..9ef343e8d5 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_store.go @@ -0,0 +1,189 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/url" + "regexp" + "strconv" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const retainedReaderDataPath = "/v1/secret/data/infrastructure/email/recovery/retained-reader-inputs" +const retainedReaderMetadataPath = "/v1/secret/metadata/infrastructure/email/recovery/retained-reader-inputs" + +type readerCustodyStore struct { + client *http.Client + endpoint string + token string +} + +// Only the protected workflow's loopback tunnel is accepted. No environment +// proxy, redirect, URL credentials or arbitrary remote destination receives +// the OpenBao token. The caller pins the production tunnel's exact port. +func newReaderCustodyStore(endpoint, token string) (*readerCustodyStore, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "http" || u.Hostname() != "127.0.0.1" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.Opaque != "" || !regexp.MustCompile(`^[A-Za-z0-9._-]{16,256}$`).MatchString(token) { + return nil, errors.New("custody-endpoint") + } + port, err := strconv.Atoi(u.Port()) + if err != nil || port < 1 || port > 65535 { + return nil, errors.New("custody-endpoint") + } + transport := &http.Transport{Proxy: nil, DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, MaxResponseHeaderBytes: 64 << 10} + client := &http.Client{Transport: transport, Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + return &readerCustodyStore{client: client, endpoint: endpoint, token: token}, nil +} + +func readerCustodyDocument(bundle retainedReaderInputs) (object, error) { + fail := func() (object, error) { return nil, errors.New("custody-document") } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + if !uuid.MatchString(bundle.ReaderUID) || !uuid.MatchString(bundle.ClaimUID) || bundle.ReaderUID == bundle.ClaimUID || len(bundle.Inputs) != 8 { + return fail() + } + allowed := map[string]bool{} + for _, want := range append(recoveryResources(retainedRecovery, "", ""), recoveryReader(retainedRecovery, "")) { + m := child(want, "metadata") + allowed[(ownedReference{kind: text(want, "kind"), name: text(m, "name"), namespace: text(m, "namespace")}).key()] = true + } + seen := map[string]bool{} + for _, input := range bundle.Inputs { + ref := ownedReference{kind: input.Kind, name: input.Name, namespace: input.Namespace} + if !allowed[ref.key()] || !uuid.MatchString(input.UID) || seen[input.UID] || input.Values == nil || (input.Kind == "Pod" && input.UID != bundle.ReaderUID) || (input.Kind == "PersistentVolumeClaim" && input.UID != bundle.ClaimUID) { + return fail() + } + delete(allowed, ref.key()) + seen[input.UID] = true + raw, err := json.Marshal(input.Values) + if err != nil { + return fail() + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != input.Digest { + return fail() + } + } + raw, err := json.Marshal(bundle) + if err != nil || len(raw) > 512<<10 { + return fail() + } + var value object + if json.Unmarshal(raw, &value) != nil { + return fail() + } + return object{"schema": "email-reader-inputs/v1", "trial_run": retainedRecovery.run, "trial_revision": retainedRecovery.revision, "bundle": value}, nil +} + +// ensure creates the one fixed record with CAS=0, or verifies an identical +// original version. It has no update, delete, metadata-write or retry-write +// path. verify runs after the final API preflight, immediately before POST, +// then again after readback. A stored record alone authorizes no lifecycle. +func (s *readerCustodyStore) ensure(ctx context.Context, bundle retainedReaderInputs, verify func(context.Context) error) error { + fail := func() error { return errors.New("reader-custody") } + expected, err := readerCustodyDocument(bundle) + if err != nil || verify == nil || s == nil || s.client == nil || ctx.Err() != nil { + return fail() + } + mounts, status, err := s.request(ctx, http.MethodGet, "/v1/sys/mounts", nil) + mount := child(child(mounts, "data"), "secret/") + if err != nil || status != http.StatusOK || text(mount, "type") != "kv" || text(child(mount, "options"), "version") != "2" { + return fail() + } + found, err := s.matches(ctx, expected) + if err != nil { + return fail() + } + if !found { + // A soft-deleted value is not a new key. Preserve its metadata and + // refuse before any write, rather than trying to replace its history. + _, status, err = s.request(ctx, http.MethodGet, retainedReaderMetadataPath, nil) + if err != nil || status != http.StatusNotFound { + return fail() + } + } + if verify(ctx) != nil { + return fail() + } + if !found { + payload, err := json.Marshal(object{"options": object{"cas": 0}, "data": expected}) + if err != nil { + return fail() + } + _, status, err = s.request(ctx, http.MethodPost, retainedReaderDataPath, payload) + // A lost or failed response is uncertain and is never retried. A CAS + // conflict can succeed only by reading the identical original record. + if err != nil || (status != http.StatusOK && status != http.StatusBadRequest) { + return fail() + } + } + found, err = s.matches(ctx, expected) + if err != nil || !found || verify(ctx) != nil { + return fail() + } + return nil +} + +func (s *readerCustodyStore) matches(ctx context.Context, expected object) (bool, error) { + o, status, err := s.request(ctx, http.MethodGet, retainedReaderDataPath, nil) + if err != nil { + return false, errors.New("custody-readback") + } + if status == http.StatusNotFound { + return false, nil + } + data := child(o, "data") + metadata := child(data, "metadata") + deleted, hasDeleted := metadata["deletion_time"].(string) + destroyed, hasDestroyed := metadata["destroyed"].(bool) + _, timeErr := time.Parse(time.RFC3339Nano, text(metadata, "created_time")) + actualBytes, actualErr := json.Marshal(data["data"]) + expectedBytes, expectedErr := json.Marshal(expected) + if status != http.StatusOK || metadata["version"] != float64(1) || !hasDeleted || deleted != "" || !hasDestroyed || destroyed || timeErr != nil || actualErr != nil || expectedErr != nil || !bytes.Equal(actualBytes, expectedBytes) { + return false, errors.New("custody-readback") + } + return true, nil +} + +func (s *readerCustodyStore) request(ctx context.Context, method, path string, payload []byte) (object, int, error) { + fail := func() (object, int, error) { return nil, 0, errors.New("custody-request") } + if (method != http.MethodGet || (path != "/v1/sys/mounts" && path != retainedReaderDataPath && path != retainedReaderMetadataPath)) && (method != http.MethodPost || path != retainedReaderDataPath) { + return fail() + } + req, err := http.NewRequestWithContext(ctx, method, s.endpoint+path, bytes.NewReader(payload)) + if err != nil { + return fail() + } + req.Header.Set("X-Vault-Token", s.token) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/json") + } + response, err := s.client.Do(req) + if err != nil { + return fail() + } + defer func() { _ = response.Body.Close() }() + raw, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil || len(raw) > 1<<20 { + return fail() + } + if len(bytes.TrimSpace(raw)) == 0 { + return object{}, response.StatusCode, nil + } + if strictjson.Validate(raw, nil) != nil { + return fail() + } + var value object + if json.Unmarshal(raw, &value) != nil || value == nil { + return fail() + } + return value, response.StatusCode, nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_store_test.go b/scripts/verify-email-data-recovery/reader_custody_store_test.go new file mode 100644 index 0000000000..78e4cb1bd0 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_store_test.go @@ -0,0 +1,296 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func custodyBundle(t *testing.T) retainedReaderInputs { + t.Helper() + graph, reader, claim := readerInputFixture(t) + bundle, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil { + t.Fatal(err) + } + return bundle +} + +func custodyResponse(t *testing.T, w http.ResponseWriter, data any, version int) { + t.Helper() + if err := json.NewEncoder(w).Encode(object{"data": object{"data": data, "metadata": object{"version": version, "destroyed": false, "deletion_time": "", "created_time": "2026-01-01T00:00:00Z"}}}); err != nil { + t.Error(err) + } +} + +func TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite(t *testing.T) { + bundle := custodyBundle(t) + var stored object + posts, preflight, rebound := 0, false, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Vault-Token") != "fixture-reader-custody-token" { + t.Error("request lost its private authentication header") + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/sys/mounts": + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + case r.Method == http.MethodGet && r.URL.Path == retainedReaderMetadataPath: + preflight = true + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == retainedReaderDataPath: + if stored == nil { + w.WriteHeader(http.StatusNotFound) + return + } + custodyResponse(t, w, stored, 1) + case r.Method == http.MethodPost && r.URL.Path == retainedReaderDataPath: + posts++ + if !preflight || rebound != 1 { + t.Error("custody mutation preceded final native-input rebinding") + } + var payload object + if json.NewDecoder(r.Body).Decode(&payload) != nil || child(payload, "options")["cas"] != float64(0) || len(child(payload, "options")) != 1 || len(payload) != 2 { + t.Error("write lost its create-only boundary") + } + stored = child(payload, "data") + _, _ = w.Write([]byte(`{"data":{"version":1}}`)) + default: + t.Errorf("unexpected credential operation: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusForbidden) + } + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + verify := func(context.Context) error { + if !preflight { + t.Error("native verification happened before the final non-destructive preflight") + } + rebound++ + return nil + } + if err = store.ensure(context.Background(), bundle, verify); err != nil { + t.Fatalf("valid create-only custody failed: %v", err) + } + if posts != 1 || rebound != 2 || stored["schema"] != "email-reader-inputs/v1" { + t.Fatal("custody did not read back and rebind the exact stored inputs") + } + expected, err := readerCustodyDocument(bundle) + if err != nil || !reflect.DeepEqual(stored, expected) { + t.Fatal("stored credentials differ from the independently captured copy") + } + // A new invocation may verify the same immutable record, never overwrite it. + if err = store.ensure(context.Background(), bundle, verify); err != nil || posts != 1 { + t.Fatal("idempotent verification rewrote the existing custody record") + } +} + +func TestReaderCustodyRejectsChangedInputsAfterFinalPreflight(t *testing.T) { + posts, preflight := 0, false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + posts++ + } + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.URL.Path == retainedReaderMetadataPath { + preflight = true + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { + if preflight { + return errors.New("native inputs changed") + } + return nil + }) + if err == nil || posts != 0 { + t.Fatal("changed native inputs authorized a credential write") + } +} + +func TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords(t *testing.T) { + for _, failure := range []string{"different", "newer-version", "deleted", "destroyed", "absent-metadata", "unreadable", "duplicate-json", "oversized", "wrong-mount"} { + t.Run(failure, func(t *testing.T) { + posts := 0 + bundle := custodyBundle(t) + document, err := readerCustodyDocument(bundle) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + posts++ + } + if r.URL.Path == "/v1/sys/mounts" { + version := "2" + if failure == "wrong-mount" { + version = "1" + } + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"` + version + `"}}}}`)) + return + } + if r.URL.Path == retainedReaderMetadataPath { + _, _ = w.Write([]byte(`{"data":{"current_version":1}}`)) + return + } + switch failure { + case "deleted": + w.WriteHeader(http.StatusNotFound) + case "destroyed": + _, _ = w.Write([]byte(`{"data":{"data":{},"metadata":{"version":1,"destroyed":true,"deletion_time":""}}}`)) + case "absent-metadata": + _ = json.NewEncoder(w).Encode(object{"data": object{"data": document}}) + case "unreadable": + w.WriteHeader(http.StatusForbidden) + case "duplicate-json": + _, _ = w.Write([]byte(`{"data":{},"data":{}}`)) + case "oversized": + _, _ = w.Write([]byte(strings.Repeat("x", (1<<20)+1))) + case "newer-version": + custodyResponse(t, w, document, 2) + default: + custodyResponse(t, w, object{"schema": "foreign"}, 1) + } + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), bundle, func(context.Context) error { return nil }) + if err == nil || posts != 0 || strings.Contains(err.Error(), "fixture-reader-custody-token") { + t.Fatal("ambiguous custody state caused a write or exposed authentication") + } + }) + } +} + +func TestReaderCustodyNeverFollowsCredentialRedirects(t *testing.T) { + forwarded := 0 + other := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { forwarded++ })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + if store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { return nil }) == nil || forwarded != 0 { + t.Fatal("a redirect could relay the custody authentication") + } +} + +func TestReaderCustodyCASConflictCannotOverwriteAnotherRecord(t *testing.T) { + for _, identical := range []bool{false, true} { + t.Run(map[bool]string{false: "foreign-winner", true: "identical-winner"}[identical], func(t *testing.T) { + bundle := custodyBundle(t) + winner, err := readerCustodyDocument(bundle) + if err != nil { + t.Fatal(err) + } + if !identical { + winner = object{"schema": "another-recovery-point"} + } + posts := 0 + var stored object + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + stored = winner + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"errors":["check-and-set conflict"]}`)) + return + } + if r.URL.Path == retainedReaderDataPath && stored != nil { + custodyResponse(t, w, stored, 1) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), bundle, func(context.Context) error { return nil }) + if (err == nil) != identical || posts != 1 || !reflect.DeepEqual(stored, winner) { + t.Fatal("CAS conflict rewrote a winner or accepted different recovery inputs") + } + }) + } +} + +func TestReaderCustodyLostWriteResponseIsNotRetriedOrReportedStored(t *testing.T) { + posts, postWriteReads := 0, 0 + committed := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("HTTP fixture cannot model a lost committed response") + w.WriteHeader(http.StatusInternalServerError) + close(committed) + return + } + connection, _, err := hijacker.Hijack() + if err != nil { + t.Error(err) + } else { + _ = connection.Close() + } + close(committed) + return + } + if posts != 0 { + postWriteReads++ + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { return nil }) + select { + case <-committed: + case <-time.After(2 * time.Second): + t.Fatal("fixture write never reached its committed-response boundary") + } + if err == nil || posts != 1 || postWriteReads != 0 { + t.Fatal("uncertain write was retried or reclassified as verified") + } +} diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 55cd26df7e..7cfcfd2983 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -11,7 +11,16 @@ import ( // The actual workflow guard must reject branch, actor and rerun changes before // checkout or credentials; metadata checks preserve the fixed disposal handoff. func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/dispose-retained-email-copy.yaml") + checkRetainedWorkflow(t, "dispose-retained-email-copy.yaml", "dispose", "dispose-retained-email-copy", "--dispose-retained", "EMAIL_RETAINED_COPY_CONFIRM", retainedDispatch()) +} + +func TestReaderInputCustodyWorkflowMainGuardAndCustody(t *testing.T) { + checkRetainedWorkflow(t, "prepare-retained-email-reader-inputs.yaml", "prepare", "prepare-retained-email-reader-inputs", "--prepare-reader-inputs", "EMAIL_READER_INPUTS_CONFIRM", readerCustodyDispatch()) +} + +func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confirmEnv string, env map[string]string) { + t.Helper() + data, err := os.ReadFile("../../.github/workflows/" + filename) if err != nil { t.Fatal(err) } @@ -36,14 +45,17 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { if yaml.Unmarshal(data, &w) != nil || len(w.On) != 1 || w.On["workflow_dispatch"] == nil || len(w.Permissions) != 0 || len(w.Jobs) != 1 || w.Concurrency.Group != "prod-deploy" || w.Concurrency.Cancel { t.Fatal("workflow widened its trigger, permission or concurrency boundary") } - job := w.Jobs["dispose"] + job := w.Jobs[jobName] if job.Environment != "prod" || job.Timeout > 25 || job.Timeout < 3 || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) != 6 { t.Fatal("observation custody changed") } guard := job.Steps[0] - env := retainedDispatch() - env["CONFIRM"] = "dispose-retained-email-copy" - for _, key := range append([]string{""}, "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "CONFIRM") { + env["CONFIRM"] = confirm + guardKeys := []string{"", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "CONFIRM"} + if jobName == "prepare" { + guardKeys = append(guardKeys, "GITHUB_ACTIONS") + } + for _, key := range guardKeys { t.Run("guard/"+key, func(t *testing.T) { cmd := exec.Command("/bin/bash", "--noprofile", "--norc", "-c", guard.Run+"\nprintf DISPOSAL_ADMITTED\n") for k, v := range env { @@ -72,7 +84,7 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { } } final := job.Steps[5] - if final.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || final.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || final.Env["EMAIL_RETAINED_COPY_CONFIRM"] != "${{ inputs.confirm }}" || !strings.HasSuffix(strings.TrimSpace(final.Run), `"$RUNNER_TEMP/verify-email-data-recovery" --dispose-retained`) { + if final.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || final.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || final.Env[confirmEnv] != "${{ inputs.confirm }}" || !strings.HasSuffix(strings.TrimSpace(final.Run), `"$RUNNER_TEMP/verify-email-data-recovery" `+flag) { t.Fatal("workflow invoked recovery writes or lost credential binding") } if final.Env["EMAIL_RETAINED_READER_UID"] != "${{ secrets.EMAIL_RETAINED_READER_UID }}" || final.Env["EMAIL_RETAINED_CLAIM_UID"] != "${{ secrets.EMAIL_RETAINED_CLAIM_UID }}" || !strings.Contains(final.Run, `./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA"`) { From 761fab41d5b3eab5097937116118aef74380fd19 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 17:18:36 +0100 Subject: [PATCH 3/7] fix(email): preserve runner masking during custody preparation Part of #6048. Keep the stable endpoint helper stderr visible to the runner so its masking commands protect subsequent native-client output. The shared endpoint guard reproduces the rejected form and passes with the reviewed invocation. --- .github/workflows/prepare-retained-email-reader-inputs.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prepare-retained-email-reader-inputs.yaml b/.github/workflows/prepare-retained-email-reader-inputs.yaml index 13c9d697da..e501a78c59 100644 --- a/.github/workflows/prepare-retained-email-reader-inputs.yaml +++ b/.github/workflows/prepare-retained-email-reader-inputs.yaml @@ -93,7 +93,11 @@ jobs: export KUBECONFIG="$private_dir/kubeconfig" printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" unset KUBE_CONFIG - if ! ./scripts/use-prod-stable-api-endpoint.sh > /dev/null 2>"$private_dir/endpoint.log"; then fail; fi + # The helper sends GitHub's masking commands on stderr. Preserve + # that stream so later native-client output remains redacted. + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi unset HCLOUD_TOKEN if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" > /dev/null 2>"$private_dir/source.log"; then fail; fi kc=(kubectl --context=admin@prod --request-timeout=20s) From 3e2b71c9656b99113c1f054b71765f7d1df82101 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 17:47:12 +0100 Subject: [PATCH 4/7] feat(email): retain the fixed recovery volume before maintenance Part of #6048. Keep this restored binding separate from generic provider-source retention. Add an independently witnessed main-only operation with atomic UID, resource-version and full-spec JSON patch tests, fresh source and graph readback, and no key export or reader lifecycle permission. Preserve all disposal and backup validity gates. --- .../protect-retained-email-volume.yaml | 100 +++++++ scripts/verify-email-data-recovery/README.md | 15 ++ scripts/verify-email-data-recovery/main.go | 5 + .../reader_volume_entry_test.go | 217 ++++++++++++++++ .../reader_volume_retention.go | 244 ++++++++++++++++++ .../reader_volume_retention_test.go | 124 +++++++++ .../retained_workflow_test.go | 9 +- 7 files changed, 713 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/protect-retained-email-volume.yaml create mode 100644 scripts/verify-email-data-recovery/reader_volume_entry_test.go create mode 100644 scripts/verify-email-data-recovery/reader_volume_retention.go create mode 100644 scripts/verify-email-data-recovery/reader_volume_retention_test.go diff --git a/.github/workflows/protect-retained-email-volume.yaml b/.github/workflows/protect-retained-email-volume.yaml new file mode 100644 index 0000000000..54574a32fa --- /dev/null +++ b/.github/workflows/protect-retained-email-volume.yaml @@ -0,0 +1,100 @@ +name: Protect Retained Email Volume + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: protect-retained-email-volume" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + protect: + name: Retain the fixed recovery volume without stopping its reader + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != protect-retained-email-volume ]]; then + printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=dispatch reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Build the reviewed command before restoring credentials + shell: bash + run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery + + - name: Install the repository-approved Kubernetes client + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-volume-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-volume-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-volume-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-volume-tools" >> "$GITHUB_PATH" + + - name: Protect only the independently witnessed existing volume + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} + EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_RETAINED_PV_UID: ${{ secrets.EMAIL_RETAINED_PV_UID }} + EMAIL_RETAINED_VOLUME_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + ulimit -c 0 + fail() { printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=workspace reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; } + uid='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + [[ "$EMAIL_RETAINED_READER_UID" =~ $uid && "$EMAIL_RETAINED_CLAIM_UID" =~ $uid && "$EMAIL_RETAINED_PV_UID" =~ $uid ]] || fail + [[ "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_CLAIM_UID" && "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_PV_UID" && "$EMAIL_RETAINED_CLAIM_UID" != "$EMAIL_RETAINED_PV_UID" ]] || fail + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + private_dir="$(mktemp -d /dev/shm/email-volume-protection.XXXXXXXX)" + trap 'rm -rf "$private_dir"' EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" >/dev/null 2>"$private_dir/source.log"; then + fail + fi + "$RUNNER_TEMP/verify-email-data-recovery" --protect-retained-volume diff --git a/scripts/verify-email-data-recovery/README.md b/scripts/verify-email-data-recovery/README.md index 9ff5c5391b..2d3e93a3d9 100644 --- a/scripts/verify-email-data-recovery/README.md +++ b/scripts/verify-email-data-recovery/README.md @@ -119,6 +119,21 @@ The expired-backup disposal gate remains unchanged. If custody is refused, investigate the retained identities and destination before another confirmed first attempt; never replace a conflicting record or regenerate a missing key. +`Protect Retained Email Volume` is a separate, default-off main-only operation +with confirmation `protect-retained-email-volume`. It requires independent +reader, claim and PV UID witnesses, including the private +`EMAIL_RETAINED_PV_UID` setting, and validates the actual restored binding. +It deliberately does not assume that the original PV name contains the current +restored claim's UID. The command preserves the running reader and every +claim, backend, key and configuration object. Its only permitted mutation is +changing the bound original PV's reclaim policy from `Delete` to `Retain`, +with atomic UID, name, resource-version and complete-spec tests. It then +rechecks the original graph and serving source. An already-retained PV is +verified without a write; an uncertain patch is never retried or rolled back. +No credential export or reader lifecycle authority is available in this +command. Protection does not unlock either expired-backup disposal path, +establish key recovery or authorize a later reader pause. + With confirmation `verify-fresh-retained-email-parity`, the workflow checks the separately retained fresh restore. The protected witness binds the namespace, reader, claim, backup and download. The backup must still be diff --git a/scripts/verify-email-data-recovery/main.go b/scripts/verify-email-data-recovery/main.go index c3c9259e20..1b9c372045 100644 --- a/scripts/verify-email-data-recovery/main.go +++ b/scripts/verify-email-data-recovery/main.go @@ -13,6 +13,11 @@ import ( ) func main() { + if len(os.Args) == 2 && os.Args[1] == "--protect-retained-volume" { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + os.Exit(runRetainedVolumeProtection(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, os.Stdout)) + } if len(os.Args) == 2 && os.Args[1] == "--prepare-reader-inputs" { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/scripts/verify-email-data-recovery/reader_volume_entry_test.go b/scripts/verify-email-data-recovery/reader_volume_entry_test.go new file mode 100644 index 0000000000..3ccffd78eb --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_entry_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "reflect" + "strings" + "testing" + + jsonpatch "gopkg.in/evanphx/json-patch.v4" +) + +type readerVolumeAPI struct { + inner *cleanupAPI + name string + patches int + beforePatch, afterPatch func() + lostResponse bool +} + +func (a *readerVolumeAPI) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + if args[0] != "patch" { + if len(args) > 1 && args[1] == "Secret" { + return nil, errors.New("secret-read-forbidden") + } + return a.inner.call(ctx, input, args...) + } + a.patches++ + if !reflect.DeepEqual(args, []string{"patch", "PersistentVolume", a.name, "--type=json", "--patch-file=/dev/stdin", "-o=json"}) { + return nil, errors.New("patch-capability") + } + if a.beforePatch != nil { + a.beforePatch() + } + key := cleanupKey("PersistentVolume", "", a.name) + raw, _ := json.Marshal(a.inner.objects[key]) + patch, err := jsonpatch.DecodePatch(input) + if err != nil { + return nil, err + } + updated, err := patch.Apply(raw) + if err != nil { + return nil, err + } + var o object + if json.Unmarshal(updated, &o) != nil { + return nil, errors.New("patch-result") + } + child(o, "metadata")["resourceVersion"] = "501" + a.inner.objects[key] = o + if a.afterPatch != nil { + a.afterPatch() + } + if a.lostResponse { + return nil, errors.New("uncertain-patch-response") + } + return json.Marshal(a.inner.objects[key]) +} + +func readerVolumeDispatch() map[string]string { + env := readerCustodyDispatch() + delete(env, "EMAIL_READER_INPUTS_CONFIRM") + env["GITHUB_WORKFLOW_REF"] = "devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main" + env["EMAIL_RETAINED_VOLUME_CONFIRM"] = "protect-retained-email-volume" + return env +} + +func readerVolumeNativeFixture(t *testing.T) (*readerVolumeAPI, map[string]string) { + t.Helper() + a, _, _ := readerCustodyNativeFixture(t) + reader, claim, pv, _, witness := retainedVolumeFixture(t) + image, err := loadRecoveryImage() + if err != nil { + t.Fatal(err) + } + items(child(reader, "spec"), "containers")[0]["image"] = image + for _, o := range []object{reader, claim, pv} { + m := child(o, "metadata") + a.objects[cleanupKey(text(o, "kind"), text(m, "namespace"), text(m, "name"))] = o + } + env := readerVolumeDispatch() + env["EMAIL_RETAINED_READER_UID"], env["EMAIL_RETAINED_CLAIM_UID"], env["EMAIL_RETAINED_PV_UID"] = witness.ReaderUID, witness.ClaimUID, witness.PVUID + return &readerVolumeAPI{inner: a, name: text(child(pv, "metadata"), "name")}, env +} + +func TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + before, _ := json.Marshal(a.inner.objects) + for pass := 0; pass < 2; pass++ { + var out bytes.Buffer + checks := 0 + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { checks++; return nil }, &out) + if code != 0 || a.patches != 1 || checks < 2 || !strings.Contains(out.String(), "EMAIL_RETAINED_VOLUME=PROTECTED") || !strings.Contains(out.String(), "reader=RETAINED restart=NOT_AUTHORIZED") { + t.Fatalf("fixed metadata protection was not verified: code=%d patches=%d checks=%d output=%q", code, a.patches, checks, out.String()) + } + if strings.Contains(out.String(), env["EMAIL_RETAINED_PV_UID"]) || strings.Contains(out.String(), a.name) { + t.Fatal("private identity reached output") + } + } + var original map[string]object + if json.Unmarshal(before, &original) != nil { + t.Fatal("fixture") + } + key := cleanupKey("PersistentVolume", "", a.name) + child(original[key], "spec")["persistentVolumeReclaimPolicy"] = "Retain" + child(original[key], "metadata")["resourceVersion"] = "501" + if !reflect.DeepEqual(original, a.inner.objects) { + t.Fatal("protection changed data, readers, keys, configuration or another storage field") + } +} + +func TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess(t *testing.T) { + for _, field := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "GITHUB_WORKFLOW_REF", "GITHUB_RUN_ID", "GITHUB_SHA", "EMAIL_RETAINED_VOLUME_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID", "EMAIL_RETAINED_PV_UID"} { + t.Run(field, func(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + env[field] = "foreign" + calls := 0 + a.inner.onGet = func(string) { calls++ } + var out bytes.Buffer + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { calls++; return nil }, &out) + if code != 2 || calls != 0 || a.patches != 0 || !strings.Contains(out.String(), "stage=dispatch") { + t.Fatal("foreign dispatch reached native state or source execution") + } + }) + } +} + +func TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation(t *testing.T) { + for _, failure := range []string{"stale-version", "replacement", "lost-response", "changed-readback", "changed-before-write", "changed-after-write"} { + t.Run(failure, func(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + key := cleanupKey("PersistentVolume", "", a.name) + switch failure { + case "stale-version": + a.beforePatch = func() { child(a.inner.objects[key], "metadata")["resourceVersion"] = "502" } + case "replacement": + a.beforePatch = func() { child(a.inner.objects[key], "metadata")["uid"] = "replaced" } + case "lost-response": + a.lostResponse = true + case "changed-readback": + a.afterPatch = func() { child(child(a.inner.objects[key], "spec"), "csi")["volumeHandle"] = "foreign" } + } + checks := 0 + var out bytes.Buffer + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { + checks++ + if failure == "changed-before-write" && checks == 2 { + child(a.inner.objects[cleanupKey("PersistentVolumeClaim", retainedRecovery.namespace, "data")], "metadata")["uid"] = "changed" + } + if failure == "changed-after-write" && checks == 3 { + return errors.New("source-changed") + } + return nil + }, &out) + wantPatches := 1 + if failure == "changed-before-write" { + wantPatches = 0 + } + if code != 2 || a.patches != wantPatches || strings.Contains(out.String(), "VOLUME=PROTECTED") { + t.Fatalf("uncertain protection was retried or certified: code=%d patches=%d output=%q", code, a.patches, out.String()) + } + }) + } +} + +func TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions(t *testing.T) { + for _, args := range [][]string{ + {"get", "Secret", "storage-key", "--ignore-not-found=true", "-o=json", "--namespace=" + retainedRecovery.namespace}, + {"delete", "Pod", "reader"}, {"create", "-f=-"}, + {"patch", "PersistentVolumeClaim", "data", "--type=json", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-foreign", "--type=json", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-bound", "--type=merge", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-bound", "--type=json", "--patch-file=/dev/stdin", "-o=json", "--subresource=status"}, + } { + inner := &refusedAPI{} + native := &retainedVolumeKubernetes{inner: inner, name: "pvc-bound", patch: []byte("approved-patch")} + if _, err := native.call(context.Background(), []byte("approved-patch"), args...); err == nil || inner.calls != 0 { + t.Fatal("metadata protection borrowed secret, reader, claim or arbitrary patch authority") + } + } + for _, input := range [][]byte{nil, []byte("foreign-patch")} { + inner := &refusedAPI{} + native := &retainedVolumeKubernetes{inner: inner, name: "pvc-bound", patch: []byte("approved-patch")} + if _, err := native.call(context.Background(), input, "patch", "PersistentVolume", "pvc-bound", "--type=json", "--patch-file=/dev/stdin", "-o=json"); err == nil || inner.calls != 0 { + t.Fatal("unqualified patch payload reached Kubernetes") + } + } +} + +func TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch(t *testing.T) { + if os.Getenv("EMAIL_RETAINED_VOLUME_CHILD") == "1" { + os.Args = []string{"verify-email-data-recovery", "--protect-retained-volume"} + main() + return + } + for _, confirmed := range []bool{false, true} { + command := exec.Command(os.Args[0], "-test.run=^TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch$") + command.Env = []string{"EMAIL_RETAINED_VOLUME_CHILD=1", "PATH=/nonexistent", "GOCOVERDIR=" + t.TempDir()} + stage := "dispatch" + if confirmed { + for key, value := range readerVolumeDispatch() { + command.Env = append(command.Env, key+"="+value) + } + command.Env = append(command.Env, "EMAIL_RETAINED_READER_UID=11111111-1111-2222-3333-444444444444", "EMAIL_RETAINED_CLAIM_UID=22222222-1111-2222-3333-444444444444", "EMAIL_RETAINED_PV_UID=33333333-1111-2222-3333-444444444444") + stage = "source" + } + output, err := command.CombinedOutput() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_RETAINED_VOLUME=UNVERIFIED stage="+stage+" reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + t.Fatalf("volume command lost its separate dispatch boundary: %q", output) + } + } +} diff --git a/scripts/verify-email-data-recovery/reader_volume_retention.go b/scripts/verify-email-data-recovery/reader_volume_retention.go new file mode 100644 index 0000000000..25eeedb935 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_retention.go @@ -0,0 +1,244 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +type retainedVolumeWitness struct{ ReaderUID, ClaimUID, PVUID string } + +func runRetainedVolumeProtection(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, output io.Writer) int { + stage := "dispatch" + fail := func() int { + if output != nil { + _, _ = fmt.Fprintf(output, "EMAIL_RETAINED_VOLUME=UNVERIFIED stage=%s reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", stage) + } + return 2 + } + witness := retainedVolumeWitness{env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID"), env("EMAIL_RETAINED_PV_UID")} + if output == nil || verifyMain == nil || authorizeRetainedVolumeProtection(env, witness) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_RETAINED_VOLUME=STARTED source_writes=0 reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + stage = "source" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + native := &retainedVolumeKubernetes{inner: k} + d := &recoveryDriver{i: retainedRecovery, k: native} + var err error + d.source, err = d.readSource(ctx) + if err != nil { + return fail() + } + stage = "binding" + original, err := d.readRetainedVolume(ctx, witness) + if err != nil || verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + fresh, err := d.readRetainedVolume(ctx, witness) + if err != nil || !sameRetainedVolumeInputs(original, fresh) || d.sourceRetained(ctx) != nil { + return fail() + } + expected, err := cloneRetainedVolumeObjects(fresh) + if err != nil { + return fail() + } + child(expected[2], "spec")["persistentVolumeReclaimPolicy"] = "Retain" + state := "EXISTING" + if text(child(fresh[2], "spec"), "persistentVolumeReclaimPolicy") == "Delete" { + m := child(fresh[2], "metadata") + payload, err := json.Marshal([]object{ + {"op": "test", "path": "/metadata/uid", "value": witness.PVUID}, + {"op": "test", "path": "/metadata/name", "value": text(m, "name")}, + {"op": "test", "path": "/metadata/resourceVersion", "value": text(m, "resourceVersion")}, + {"op": "test", "path": "/spec", "value": child(fresh[2], "spec")}, + {"op": "replace", "path": "/spec/persistentVolumeReclaimPolicy", "value": "Retain"}, + }) + if err != nil { + return fail() + } + stage = "mutation" + native.name, native.patch = text(m, "name"), payload + _, err = native.call(ctx, payload, "patch", "PersistentVolume", native.name, "--type=json", "--patch-file=/dev/stdin", "-o=json") + native.patch = nil + // An uncertain response never retries or reclassifies a mutation as + // success. No rollback can turn an already-protected PV back to Delete. + if err != nil { + return fail() + } + state = "UPDATED" + } + stage = "readback" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + after, err := d.readRetainedVolume(ctx, witness) + if err != nil || !sameRetainedVolumeInputs(expected, after) || d.sourceRetained(ctx) != nil { + return fail() + } + if _, err := fmt.Fprintf(output, "EMAIL_RETAINED_VOLUME=PROTECTED state=%s source_writes=0 reader=RETAINED restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", state); err != nil { + return 2 + } + return 0 +} + +func authorizeRetainedVolumeProtection(env func(string) string, witness retainedVolumeWitness) error { + for key, want := range map[string]string{ + "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", + "GITHUB_ACTOR": "devantler", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main", + "EMAIL_RETAINED_VOLUME_CONFIRM": "protect-retained-email-volume", + } { + if env(key) != want { + return errors.New("retained-volume-dispatch") + } + } + if !recoveryRunPattern.MatchString(env("GITHUB_RUN_ID")) || !recoveryRevisionPattern.MatchString(env("GITHUB_SHA")) || !validRetainedVolumeWitness(witness) { + return errors.New("retained-volume-dispatch") + } + return nil +} + +// This operation reads no Secret and cannot stop a reader or retire storage. +// Its only mutation is the exact JSON patch produced after final binding. +type retainedVolumeKubernetes struct { + inner kubernetes + name string + patch []byte +} + +func (k *retainedVolumeKubernetes) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + fail := func() ([]byte, error) { return nil, errors.New("retained-volume-capability") } + if k.inner == nil || len(args) < 2 { + return fail() + } + var raw []byte + var err error + if args[0] == "get" && (args[1] == "Pod" || args[1] == "PersistentVolumeClaim" || args[1] == "PersistentVolume") { + raw, err = (readerInputKubernetes{inner: k.inner}).call(ctx, input, args...) + } else if len(k.patch) != 0 && bytes.Equal(input, k.patch) && reflect.DeepEqual(args, []string{"patch", "PersistentVolume", k.name, "--type=json", "--patch-file=/dev/stdin", "-o=json"}) { + raw, err = k.inner.call(ctx, input, args...) + } else { + return fail() + } + if err != nil || len(raw) > 4<<20 || (len(bytes.TrimSpace(raw)) != 0 && strictjson.Validate(raw, nil) != nil) { + return fail() + } + return raw, nil +} + +func (d *recoveryDriver) readRetainedVolume(ctx context.Context, witness retainedVolumeWitness) ([]object, error) { + fail := func() ([]object, error) { return nil, errors.New("retained-volume-read") } + reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) + if err != nil || !found || text(child(reader, "metadata"), "uid") != witness.ReaderUID { + return fail() + } + claim, found, err := d.lookup(ctx, "PersistentVolumeClaim", "data", retainedRecovery.namespace) + if err != nil || !found || text(child(claim, "metadata"), "uid") != witness.ClaimUID { + return fail() + } + name := text(child(claim, "spec"), "volumeName") + if !kubernetesName.MatchString(name) || name == d.source.pv { + return fail() + } + volume, found, err := d.lookup(ctx, "PersistentVolume", name, "") + if err != nil || !found || qualifyRetainedVolume(reader, claim, volume, d.source, d.image, witness) != nil { + return fail() + } + return []object{reader, claim, volume}, nil +} + +func cloneRetainedVolumeObjects(objects []object) ([]object, error) { + raw, err := json.Marshal(objects) + var copy []object + if err != nil || json.Unmarshal(raw, ©) != nil { + return nil, errors.New("retained-volume-snapshot") + } + return copy, nil +} + +func sameRetainedVolumeInputs(before, after []object) bool { + if len(before) != 3 || len(after) != 3 { + return false + } + for index := range before { + if !reflect.DeepEqual(retainedMutableFields(before[index]), retainedMutableFields(after[index])) || !reflect.DeepEqual(child(child(before[index], "metadata"), "annotations"), child(child(after[index], "metadata"), "annotations")) { + return false + } + } + return true +} + +func validRetainedVolumeWitness(witness retainedVolumeWitness) bool { + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + seen := map[string]bool{} + for _, uid := range []string{witness.ReaderUID, witness.ClaimUID, witness.PVUID} { + if !uuid.MatchString(uid) || seen[uid] { + return false + } + seen[uid] = true + } + return true +} + +func qualifyRetainedVolume(reader, claim, volume object, source sourceIdentity, image string, witness retainedVolumeWitness) error { + fail := func() error { return errors.New("retained-volume-binding") } + if !validRetainedVolumeWitness(witness) { + return fail() + } + name := text(child(claim, "spec"), "volumeName") + if source.podUID == "" || source.claimUID == "" || source.pv == "" || source.pvUID == "" || source.podUID == witness.ReaderUID || source.claimUID == witness.ClaimUID || source.pvUID == witness.PVUID || source.pv == name || !regexp.MustCompile(`^pvc-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`).MatchString(name) { + return fail() + } + for _, target := range []struct { + actual object + kind, name, namespace, uid string + }{{reader, "Pod", "reader", retainedRecovery.namespace, witness.ReaderUID}, {claim, "PersistentVolumeClaim", "data", retainedRecovery.namespace, witness.ClaimUID}, {volume, "PersistentVolume", name, "", witness.PVUID}} { + m := child(target.actual, "metadata") + if text(target.actual, "apiVersion") != "v1" || text(target.actual, "kind") != target.kind || text(m, "name") != target.name || text(m, "namespace") != target.namespace || text(m, "uid") != target.uid || !regexp.MustCompile(`^[1-9][0-9]{0,31}$`).MatchString(text(m, "resourceVersion")) || m["deletionTimestamp"] != nil { + return fail() + } + if owners := m["ownerReferences"]; owners != nil { + list, ok := owners.([]any) + if !ok || len(list) != 0 { + return fail() + } + } + if target.namespace != "" && text(child(m, "labels"), recoveryLabel) != retainedRecovery.run { + return fail() + } + } + if !admittedRecoveryObject(recoveryReader(retainedRecovery, image), reader) || child(reader, "spec")["terminationGracePeriodSeconds"] != float64(30) || text(child(reader, "status"), "phase") != "Running" { + return fail() + } + ready := false + for _, condition := range items(child(reader, "status"), "conditions") { + ready = ready || text(condition, "type") == "Ready" && text(condition, "status") == "True" + } + claimSpec := child(claim, "spec") + if !ready || text(child(claim, "status"), "phase") != "Bound" || text(claimSpec, "storageClassName") != retainedRecovery.namespace || !reflect.DeepEqual(claimSpec["accessModes"], []any{"ReadWriteOnce"}) || text(child(child(claimSpec, "resources"), "requests"), "storage") != "2Gi" { + return fail() + } + spec := child(volume, "spec") + ref, csi := child(spec, "claimRef"), child(spec, "csi") + key := child(csi, "nodeStageSecretRef") + policy := text(spec, "persistentVolumeReclaimPolicy") + if text(child(volume, "status"), "phase") != "Bound" || text(spec, "storageClassName") != retainedRecovery.namespace || text(ref, "apiVersion") != "v1" || text(ref, "kind") != "PersistentVolumeClaim" || text(ref, "name") != "data" || text(ref, "namespace") != retainedRecovery.namespace || text(ref, "uid") != witness.ClaimUID || text(csi, "driver") != "driver.longhorn.io" || text(csi, "volumeHandle") != name || text(child(csi, "volumeAttributes"), "encrypted") != "true" || text(key, "name") != "storage-key" || text(key, "namespace") != retainedRecovery.namespace || policy != "Delete" && policy != "Retain" { + return fail() + } + return nil +} diff --git a/scripts/verify-email-data-recovery/reader_volume_retention_test.go b/scripts/verify-email-data-recovery/reader_volume_retention_test.go new file mode 100644 index 0000000000..1d72a354ab --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_retention_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func retainedVolumeFixture(t *testing.T) (object, object, object, sourceIdentity, retainedVolumeWitness) { + t.Helper() + graph, readerUID, claimUID := readerInputFixture(t) + reader, claim := graph[7], graph[6] + name := "pvc-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + child(claim, "spec")["volumeName"] = name + witness := retainedVolumeWitness{readerUID, claimUID, "99999999-1111-2222-3333-444444444444"} + volume := object{ + "apiVersion": "v1", "kind": "PersistentVolume", + "metadata": object{"name": name, "uid": witness.PVUID, "resourceVersion": "500"}, + "spec": object{ + "claimRef": object{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "name": "data", "namespace": retainedRecovery.namespace, "uid": claimUID}, + "storageClassName": retainedRecovery.namespace, "persistentVolumeReclaimPolicy": "Delete", + "csi": object{"driver": "driver.longhorn.io", "volumeHandle": name, "volumeAttributes": object{"encrypted": "true"}, "nodeStageSecretRef": object{"name": "storage-key", "namespace": retainedRecovery.namespace}}, + }, + "status": object{"phase": "Bound"}, + } + source := sourceIdentity{podUID: "serving-pod", claimUID: "ffffffff-1111-2222-3333-444444444444", pv: "pvc-serving-volume", pvUID: "serving-pv"} + return reader, claim, volume, source, witness +} + +func TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt(t *testing.T) { + for _, policy := range []string{"Delete", "Retain"} { + t.Run(policy, func(t *testing.T) { + reader, claim, volume, source, witness := retainedVolumeFixture(t) + child(volume, "spec")["persistentVolumeReclaimPolicy"] = policy + before, _ := json.Marshal([]object{reader, claim, volume}) + if err := qualifyRetainedVolume(reader, claim, volume, source, fixtureImage, witness); err != nil { + t.Fatalf("the independently bound restored PV was refused: %v", err) + } + after, _ := json.Marshal([]object{reader, claim, volume}) + if string(before) != string(after) || text(child(volume, "metadata"), "name") == "pvc-"+witness.ClaimUID { + t.Fatal("qualification modified inputs or relied on the original-provisioning UID formula") + } + }) + } +} + +func TestRetainedVolumeQualificationRejectsChangedOrForeignBindings(t *testing.T) { + for _, change := range []string{"reader-witness", "claim-witness", "pv-witness", "same-witness", "reader-uid", "claim-uid", "pv-uid", "reader-version", "pv-version", "pv-terminating", "claim-terminating", "reader-terminating", "reader-command", "reader-not-ready", "reader-owner", "source-claim", "source-volume", "source-pv-uid", "missing-source", "claim-namespace", "claim-unbound", "pv-name", "pv-kind", "pv-unbound", "claim-reference", "foreign-reference", "driver", "handle", "unencrypted", "key-reference", "foreign-key-namespace", "unknown-policy", "storage-class"} { + t.Run(change, func(t *testing.T) { + reader, claim, volume, source, witness := retainedVolumeFixture(t) + spec := child(volume, "spec") + switch change { + case "reader-witness": + witness.ReaderUID = "" + case "claim-witness": + witness.ClaimUID = "foreign" + case "pv-witness": + witness.PVUID = "" + case "same-witness": + witness.PVUID = witness.ClaimUID + case "reader-uid": + child(reader, "metadata")["uid"] = "replacement" + case "claim-uid": + child(claim, "metadata")["uid"] = "replacement" + case "pv-uid": + child(volume, "metadata")["uid"] = "replacement" + case "reader-version": + delete(child(reader, "metadata"), "resourceVersion") + case "pv-version": + child(volume, "metadata")["resourceVersion"] = "invalid" + case "pv-terminating": + child(volume, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "claim-terminating": + child(claim, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "reader-terminating": + child(reader, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "reader-command": + items(child(reader, "spec"), "containers")[0]["command"] = []any{"foreign"} + case "reader-not-ready": + reader["status"] = object{"phase": "Pending"} + case "reader-owner": + child(reader, "metadata")["ownerReferences"] = []any{object{"uid": "foreign"}} + case "source-claim": + source.claimUID = witness.ClaimUID + case "source-volume": + source.pv = text(child(volume, "metadata"), "name") + case "source-pv-uid": + source.pvUID = witness.PVUID + case "missing-source": + source = sourceIdentity{} + case "claim-namespace": + child(claim, "metadata")["namespace"] = "private-mail" + case "claim-unbound": + child(claim, "status")["phase"] = "Pending" + case "pv-name": + child(volume, "metadata")["name"] = "pvc-foreign" + case "pv-kind": + volume["kind"] = "Secret" + case "pv-unbound": + child(volume, "status")["phase"] = "Released" + case "claim-reference": + child(spec, "claimRef")["uid"] = source.claimUID + case "foreign-reference": + child(spec, "claimRef")["namespace"] = "private-mail" + case "driver": + child(spec, "csi")["driver"] = "foreign.invalid" + case "handle": + child(spec, "csi")["volumeHandle"] = source.pv + case "unencrypted": + child(child(spec, "csi"), "volumeAttributes")["encrypted"] = "false" + case "key-reference": + child(child(spec, "csi"), "nodeStageSecretRef")["name"] = "foreign" + case "foreign-key-namespace": + child(child(spec, "csi"), "nodeStageSecretRef")["namespace"] = "private-mail" + case "unknown-policy": + spec["persistentVolumeReclaimPolicy"] = "Recycle" + case "storage-class": + spec["storageClassName"] = "foreign" + } + if qualifyRetainedVolume(reader, claim, volume, source, fixtureImage, witness) == nil { + t.Fatal("changed or foreign restored binding was accepted") + } + }) + } +} diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 7cfcfd2983..01eccca930 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -18,6 +18,10 @@ func TestReaderInputCustodyWorkflowMainGuardAndCustody(t *testing.T) { checkRetainedWorkflow(t, "prepare-retained-email-reader-inputs.yaml", "prepare", "prepare-retained-email-reader-inputs", "--prepare-reader-inputs", "EMAIL_READER_INPUTS_CONFIRM", readerCustodyDispatch()) } +func TestRetainedVolumeProtectionWorkflowMainGuardAndCustody(t *testing.T) { + checkRetainedWorkflow(t, "protect-retained-email-volume.yaml", "protect", "protect-retained-email-volume", "--protect-retained-volume", "EMAIL_RETAINED_VOLUME_CONFIRM", readerVolumeDispatch()) +} + func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confirmEnv string, env map[string]string) { t.Helper() data, err := os.ReadFile("../../.github/workflows/" + filename) @@ -52,7 +56,7 @@ func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confi guard := job.Steps[0] env["CONFIRM"] = confirm guardKeys := []string{"", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "CONFIRM"} - if jobName == "prepare" { + if jobName != "dispose" { guardKeys = append(guardKeys, "GITHUB_ACTIONS") } for _, key := range guardKeys { @@ -84,6 +88,9 @@ func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confi } } final := job.Steps[5] + if jobName == "protect" && (final.Env["EMAIL_RETAINED_PV_UID"] != "${{ secrets.EMAIL_RETAINED_PV_UID }}" || strings.Contains(final.Run, "get secret") || strings.Contains(final.Run, "port-forward")) { + t.Fatal("metadata protection lost its independent volume witness or gained credential export") + } if final.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || final.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || final.Env[confirmEnv] != "${{ inputs.confirm }}" || !strings.HasSuffix(strings.TrimSpace(final.Run), `"$RUNNER_TEMP/verify-email-data-recovery" `+flag) { t.Fatal("workflow invoked recovery writes or lost credential binding") } From 0d4e5e5dd5f917e1fe60c71a699debcb72aaf46c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 18:26:20 +0100 Subject: [PATCH 5/7] fix(email): avoid shadowing a built-in in recovery snapshots Part of #6048. Rename the local snapshot result without changing behavior. The exact required Revive v1.16.0 diagnostic reproduced before the rename; configured package lint and the actual retention operator regressions now pass. --- .../verify-email-data-recovery/reader_volume_retention.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/verify-email-data-recovery/reader_volume_retention.go b/scripts/verify-email-data-recovery/reader_volume_retention.go index 25eeedb935..89f8abd6cc 100644 --- a/scripts/verify-email-data-recovery/reader_volume_retention.go +++ b/scripts/verify-email-data-recovery/reader_volume_retention.go @@ -164,11 +164,11 @@ func (d *recoveryDriver) readRetainedVolume(ctx context.Context, witness retaine func cloneRetainedVolumeObjects(objects []object) ([]object, error) { raw, err := json.Marshal(objects) - var copy []object - if err != nil || json.Unmarshal(raw, ©) != nil { + var cloned []object + if err != nil || json.Unmarshal(raw, &cloned) != nil { return nil, errors.New("retained-volume-snapshot") } - return copy, nil + return cloned, nil } func sameRetainedVolumeInputs(before, after []object) bool { From b6ae4acfc1562a7654ba9ad9a572b36a549af500 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 20:19:08 +0100 Subject: [PATCH 6/7] docs(email): explain retained recovery safety boundaries Document the preparation and regression contracts identified in review. Check fixture serialization and type assertions and preserve wrapped CLI exit errors. Runtime behavior and disposal gates are unchanged. --- scripts/verify-email-data-recovery/main.go | 1 + .../reader_custody_entry.go | 3 +++ .../reader_custody_entry_test.go | 7 +++++++ .../reader_custody_runtime.go | 3 +++ .../reader_custody_runtime_test.go | 8 +++++-- .../reader_custody_store.go | 3 +++ .../reader_custody_store_test.go | 8 +++++++ .../reader_inputs_test.go | 9 +++++++- .../reader_volume_entry_test.go | 21 +++++++++++++++---- .../reader_volume_retention.go | 7 +++++++ .../reader_volume_retention_test.go | 3 +++ .../retained_workflow_test.go | 3 +++ 12 files changed, 69 insertions(+), 7 deletions(-) diff --git a/scripts/verify-email-data-recovery/main.go b/scripts/verify-email-data-recovery/main.go index 1b9c372045..667afbe1b5 100644 --- a/scripts/verify-email-data-recovery/main.go +++ b/scripts/verify-email-data-recovery/main.go @@ -12,6 +12,7 @@ import ( "time" ) +// main selects one independently authorized recovery operation; unknown flags fail closed. func main() { if len(os.Args) == 2 && os.Args[1] == "--protect-retained-volume" { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) diff --git a/scripts/verify-email-data-recovery/reader_custody_entry.go b/scripts/verify-email-data-recovery/reader_custody_entry.go index 96952f4ade..2fb71bf562 100644 --- a/scripts/verify-email-data-recovery/reader_custody_entry.go +++ b/scripts/verify-email-data-recovery/reader_custody_entry.go @@ -51,6 +51,7 @@ func (k readerInputKubernetes) call(ctx context.Context, input []byte, args ...s return k.inner.call(ctx, nil, args...) } +// authorizeReaderInputCustody requires the confirmed main workflow and distinct independently recorded copy identities. func authorizeReaderInputCustody(env func(string) string) error { for key, expected := range map[string]string{ "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", @@ -70,6 +71,7 @@ func authorizeReaderInputCustody(env func(string) string) error { return nil } +// runReaderInputCustody captures and rebinds native inputs before create-only custody without granting restart authority. func runReaderInputCustody(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, loadStore func(func(string) string) (*readerCustodyStore, error), output io.Writer) int { stage := "dispatch" fail := func() int { @@ -130,6 +132,7 @@ func runReaderInputCustody(ctx context.Context, env func(string) string, k kuber return 0 } +// captureCustodyInputs reads the fixed retained graph and excludes the serving claim before freezing credential values. func (d *recoveryDriver) captureCustodyInputs(ctx context.Context, readerUID, claimUID string) (retainedReaderInputs, error) { fail := func() (retainedReaderInputs, error) { return retainedReaderInputs{}, errors.New("reader-inputs") } reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) diff --git a/scripts/verify-email-data-recovery/reader_custody_entry_test.go b/scripts/verify-email-data-recovery/reader_custody_entry_test.go index 7030d09898..7494fb5aa6 100644 --- a/scripts/verify-email-data-recovery/reader_custody_entry_test.go +++ b/scripts/verify-email-data-recovery/reader_custody_entry_test.go @@ -11,6 +11,7 @@ import ( "testing" ) +// readerCustodyDispatch supplies the fixed first-attempt workflow identity for custody tests. func readerCustodyDispatch() map[string]string { env := confirmedInvocation() env["GITHUB_ACTIONS"] = "true" @@ -19,6 +20,7 @@ func readerCustodyDispatch() map[string]string { return env } +// readerCustodyNativeFixture separates retained inputs from a distinct serving mailbox graph. func readerCustodyNativeFixture(t *testing.T) (*cleanupAPI, string, string) { t.Helper() t.Chdir("../..") @@ -46,6 +48,7 @@ func readerCustodyNativeFixture(t *testing.T) (*cleanupAPI, string, string) { return a, reader, claim } +// TestReaderCustodyDispatchRefusesBeforeAnyAccess proves invalid dispatch never reads credentials or invokes native clients. func TestReaderCustodyDispatchRefusesBeforeAnyAccess(t *testing.T) { for _, invalid := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_RUN_ATTEMPT", "GITHUB_ACTOR", "GITHUB_WORKFLOW_REF", "RUNNER_ENVIRONMENT", "GITHUB_SHA", "GITHUB_RUN_ID", "EMAIL_READER_INPUTS_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID"} { t.Run(invalid, func(t *testing.T) { @@ -66,6 +69,7 @@ func TestReaderCustodyDispatchRefusesBeforeAnyAccess(t *testing.T) { } } +// TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject checks storage readback and rejects input changes around the write. func TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject(t *testing.T) { for _, changeAt := range []int{0, 2, 3} { t.Run(map[int]string{0: "stable", 2: "changed-before-write", 3: "changed-after-write"}[changeAt], func(t *testing.T) { @@ -132,6 +136,7 @@ func TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject(t * } } +// TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead verifies rejected requests never reach the underlying client. func TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead(t *testing.T) { for _, args := range [][]string{ {"delete", "Pod", "reader"}, {"create", "-f=-"}, {"patch", "Secret", "storage-key"}, @@ -150,6 +155,7 @@ func TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead(t *te } } +// TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess prevents silent operation without source and output evidence. func TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess(t *testing.T) { for _, failure := range []string{"output", "source"} { t.Run(failure, func(t *testing.T) { @@ -173,6 +179,7 @@ func TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess(t *tes } } +// TestReaderCustodyWrongIndependentReaderStopsBeforeKeyAccess checks identity before any retained or service credential is read. func TestReaderCustodyWrongIndependentReaderStopsBeforeKeyAccess(t *testing.T) { a, _, claim := readerCustodyNativeFixture(t) secretReads, loaded := 0, 0 diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime.go b/scripts/verify-email-data-recovery/reader_custody_runtime.go index afadefe641..5f9ade4821 100644 --- a/scripts/verify-email-data-recovery/reader_custody_runtime.go +++ b/scripts/verify-email-data-recovery/reader_custody_runtime.go @@ -11,6 +11,7 @@ import ( "time" ) +// readReaderCustodyToken reads a bounded regular token from a private directory without exposing failures or credential contents. func readReaderCustodyToken(directory string) (token string, err error) { fail := func() (string, error) { return "", errors.New("reader-custody-token") } info, err := os.Lstat(directory) @@ -50,6 +51,7 @@ func readReaderCustodyToken(directory string) (token string, err error) { return string(raw), nil } +// loadReaderCustodyStore accepts only the workflow's private tmpfs workspace and fixed loopback service tunnel. func loadReaderCustodyStore(env func(string) string) (*readerCustodyStore, error) { directory := env("EMAIL_READER_INPUTS_WORK_DIR") if !regexp.MustCompile(`^/dev/shm/email-reader-inputs\.[A-Za-z0-9]{8}$`).MatchString(directory) || filepath.Clean(directory) != directory { @@ -62,6 +64,7 @@ func loadReaderCustodyStore(env func(string) string) (*readerCustodyStore, error return newReaderCustodyStore("http://127.0.0.1:38200", token) } +// verifyReaderCustodySource runs the fixed reviewed source guard with a validated revision and bounded execution time. func verifyReaderCustodySource(ctx context.Context, revision string) error { if !recoveryRevisionPattern.MatchString(revision) { return errors.New("reader-custody-source") diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime_test.go b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go index cec0be8cc3..6496e98c2e 100644 --- a/scripts/verify-email-data-recovery/reader_custody_runtime_test.go +++ b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "os" "os/exec" "path/filepath" @@ -8,6 +9,7 @@ import ( "testing" ) +// TestReaderCustodyTokenRequiresPrivateRegularLocalFiles deliberately creates unsafe modes and links to verify refusal without disclosure. func TestReaderCustodyTokenRequiresPrivateRegularLocalFiles(t *testing.T) { for _, failure := range []string{"", "public-directory", "public-file", "file-symlink", "directory-symlink", "empty", "oversized", "newline"} { t.Run(failure, func(t *testing.T) { @@ -61,6 +63,7 @@ func TestReaderCustodyTokenRequiresPrivateRegularLocalFiles(t *testing.T) { } } +// TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints checks destination and authentication syntax before any network access. func TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints(t *testing.T) { for _, endpoint := range []string{"https://127.0.0.1:38200", "http://example.invalid:38200", "http://localhost:38200", "http://127.0.0.1", "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://user@127.0.0.1:38200", "http://127.0.0.1:38200/path", "http://127.0.0.1:38200?", "http://127.0.0.1:38200#other"} { if store, err := newReaderCustodyStore(endpoint, "fixture-reader-custody-token"); err == nil || store != nil { @@ -72,6 +75,7 @@ func TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints(t *testing.T) { } } +// TestReaderCustodyCommandUsesTheSeparateProtectedDispatch executes the real CLI with no tools available to expose an incorrect branch. func TestReaderCustodyCommandUsesTheSeparateProtectedDispatch(t *testing.T) { if os.Getenv("EMAIL_READER_CUSTODY_CHILD") == "1" { os.Args = []string{"verify-email-data-recovery", "--prepare-reader-inputs"} @@ -90,8 +94,8 @@ func TestReaderCustodyCommandUsesTheSeparateProtectedDispatch(t *testing.T) { stage = "source" } output, err := command.CombinedOutput() - exit, ok := err.(*exec.ExitError) - if !ok || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage="+stage+" restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage="+stage+" restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { t.Fatalf("custody command lost its distinct dispatch boundary: %q", output) } } diff --git a/scripts/verify-email-data-recovery/reader_custody_store.go b/scripts/verify-email-data-recovery/reader_custody_store.go index 9ef343e8d5..2f03dcbd11 100644 --- a/scripts/verify-email-data-recovery/reader_custody_store.go +++ b/scripts/verify-email-data-recovery/reader_custody_store.go @@ -44,6 +44,7 @@ func newReaderCustodyStore(endpoint, token string) (*readerCustodyStore, error) return &readerCustodyStore{client: client, endpoint: endpoint, token: token}, nil } +// readerCustodyDocument binds the complete copy-specific input set to its content digests and fixed record schema. func readerCustodyDocument(bundle retainedReaderInputs) (object, error) { fail := func() (object, error) { return nil, errors.New("custody-document") } uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) @@ -132,6 +133,7 @@ func (s *readerCustodyStore) ensure(ctx context.Context, bundle retainedReaderIn return nil } +// matches accepts only an identical, undeleted original KV2 version; missing data is not recovery proof. func (s *readerCustodyStore) matches(ctx context.Context, expected object) (bool, error) { o, status, err := s.request(ctx, http.MethodGet, retainedReaderDataPath, nil) if err != nil { @@ -153,6 +155,7 @@ func (s *readerCustodyStore) matches(ctx context.Context, expected object) (bool return true, nil } +// request restricts custody HTTP methods and destinations and rejects ambiguous or oversized responses. func (s *readerCustodyStore) request(ctx context.Context, method, path string, payload []byte) (object, int, error) { fail := func() (object, int, error) { return nil, 0, errors.New("custody-request") } if (method != http.MethodGet || (path != "/v1/sys/mounts" && path != retainedReaderDataPath && path != retainedReaderMetadataPath)) && (method != http.MethodPost || path != retainedReaderDataPath) { diff --git a/scripts/verify-email-data-recovery/reader_custody_store_test.go b/scripts/verify-email-data-recovery/reader_custody_store_test.go index 78e4cb1bd0..353bc2cf0f 100644 --- a/scripts/verify-email-data-recovery/reader_custody_store_test.go +++ b/scripts/verify-email-data-recovery/reader_custody_store_test.go @@ -12,6 +12,7 @@ import ( "time" ) +// custodyBundle freezes independently witnessed fixture inputs for storage protocol tests. func custodyBundle(t *testing.T) retainedReaderInputs { t.Helper() graph, reader, claim := readerInputFixture(t) @@ -22,6 +23,7 @@ func custodyBundle(t *testing.T) retainedReaderInputs { return bundle } +// custodyResponse models KV2 data with explicit original-version and deletion metadata. func custodyResponse(t *testing.T, w http.ResponseWriter, data any, version int) { t.Helper() if err := json.NewEncoder(w).Encode(object{"data": object{"data": data, "metadata": object{"version": version, "destroyed": false, "deletion_time": "", "created_time": "2026-01-01T00:00:00Z"}}}); err != nil { @@ -29,6 +31,7 @@ func custodyResponse(t *testing.T, w http.ResponseWriter, data any, version int) } } +// TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite checks the final native rebind, readback and zero-overwrite repeat. func TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite(t *testing.T) { bundle := custodyBundle(t) var stored object @@ -94,6 +97,7 @@ func TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite(t *testing } } +// TestReaderCustodyRejectsChangedInputsAfterFinalPreflight models drift in the last interval before a credential write. func TestReaderCustodyRejectsChangedInputsAfterFinalPreflight(t *testing.T) { posts, preflight := 0, false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -126,6 +130,7 @@ func TestReaderCustodyRejectsChangedInputsAfterFinalPreflight(t *testing.T) { } } +// TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords preserves existing history when API evidence cannot establish safe creation. func TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords(t *testing.T) { for _, failure := range []string{"different", "newer-version", "deleted", "destroyed", "absent-metadata", "unreadable", "duplicate-json", "oversized", "wrong-mount"} { t.Run(failure, func(t *testing.T) { @@ -184,6 +189,7 @@ func TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords(t *testing.T) } } +// TestReaderCustodyNeverFollowsCredentialRedirects uses a second server to detect any forwarded authentication request. func TestReaderCustodyNeverFollowsCredentialRedirects(t *testing.T) { forwarded := 0 other := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { forwarded++ })) @@ -202,6 +208,7 @@ func TestReaderCustodyNeverFollowsCredentialRedirects(t *testing.T) { } } +// TestReaderCustodyCASConflictCannotOverwriteAnotherRecord distinguishes an identical concurrent winner from foreign custody. func TestReaderCustodyCASConflictCannotOverwriteAnotherRecord(t *testing.T) { for _, identical := range []bool{false, true} { t.Run(map[bool]string{false: "foreign-winner", true: "identical-winner"}[identical], func(t *testing.T) { @@ -247,6 +254,7 @@ func TestReaderCustodyCASConflictCannotOverwriteAnotherRecord(t *testing.T) { } } +// TestReaderCustodyLostWriteResponseIsNotRetriedOrReportedStored models a committed request whose response connection is lost. func TestReaderCustodyLostWriteResponseIsNotRetriedOrReportedStored(t *testing.T) { posts, postWriteReads := 0, 0 committed := make(chan struct{}) diff --git a/scripts/verify-email-data-recovery/reader_inputs_test.go b/scripts/verify-email-data-recovery/reader_inputs_test.go index fe62222c24..dc7f3b2d0b 100644 --- a/scripts/verify-email-data-recovery/reader_inputs_test.go +++ b/scripts/verify-email-data-recovery/reader_inputs_test.go @@ -22,7 +22,11 @@ func readerInputFixture(t *testing.T) ([]object, string, string) { if text(o, "kind") == "Secret" { data := object{} for k, v := range child(o, "stringData") { - data[k] = base64.StdEncoding.EncodeToString([]byte(v.(string))) + value, ok := v.(string) + if !ok { + t.Fatal("fixture secret value is not a string") + } + data[k] = base64.StdEncoding.EncodeToString([]byte(value)) } o["data"] = data delete(o, "stringData") @@ -40,6 +44,7 @@ func readerInputFixture(t *testing.T) ([]object, string, string) { return graph, readerUID, claimUID } +// TestReaderInputBundleKeepsTheActualPerCopyCredentials verifies complete credentials, stable ordering and independence from later mutations. func TestReaderInputBundleKeepsTheActualPerCopyCredentials(t *testing.T) { graph, reader, claim := readerInputFixture(t) bundle, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) @@ -80,6 +85,7 @@ func TestReaderInputBundleKeepsTheActualPerCopyCredentials(t *testing.T) { } } +// TestReaderInputBundleBindsValidChangedKeyBytes requires a content binding to change when the retained encryption key changes. func TestReaderInputBundleBindsValidChangedKeyBytes(t *testing.T) { graph, reader, claim := readerInputFixture(t) source := "ffffffff-1111-2222-3333-444444444444" @@ -97,6 +103,7 @@ func TestReaderInputBundleBindsValidChangedKeyBytes(t *testing.T) { } } +// TestReaderInputBundleRefusesBorrowedReplacedOrIncompleteInputs exercises foreign identity, missing input and configuration substitutions. func TestReaderInputBundleRefusesBorrowedReplacedOrIncompleteInputs(t *testing.T) { for _, failure := range []string{"reader", "claim", "source", "missing", "duplicate", "key", "authentication", "configuration", "storage-class", "executable", "not-ready", "terminating", "controller", "namespace", "identity", "resource-version", "labels", "foreign-target"} { t.Run(failure, func(t *testing.T) { diff --git a/scripts/verify-email-data-recovery/reader_volume_entry_test.go b/scripts/verify-email-data-recovery/reader_volume_entry_test.go index 3ccffd78eb..863d56de02 100644 --- a/scripts/verify-email-data-recovery/reader_volume_entry_test.go +++ b/scripts/verify-email-data-recovery/reader_volume_entry_test.go @@ -37,7 +37,10 @@ func (a *readerVolumeAPI) call(ctx context.Context, input []byte, args ...string a.beforePatch() } key := cleanupKey("PersistentVolume", "", a.name) - raw, _ := json.Marshal(a.inner.objects[key]) + raw, err := json.Marshal(a.inner.objects[key]) + if err != nil { + return nil, err + } patch, err := jsonpatch.DecodePatch(input) if err != nil { return nil, err @@ -61,6 +64,7 @@ func (a *readerVolumeAPI) call(ctx context.Context, input []byte, args ...string return json.Marshal(a.inner.objects[key]) } +// readerVolumeDispatch supplies the separate confirmation and workflow identity for volume protection tests. func readerVolumeDispatch() map[string]string { env := readerCustodyDispatch() delete(env, "EMAIL_READER_INPUTS_CONFIRM") @@ -69,6 +73,7 @@ func readerVolumeDispatch() map[string]string { return env } +// readerVolumeNativeFixture models a restored binding with independently witnessed reader, claim and volume identities. func readerVolumeNativeFixture(t *testing.T) (*readerVolumeAPI, map[string]string) { t.Helper() a, _, _ := readerCustodyNativeFixture(t) @@ -87,9 +92,13 @@ func readerVolumeNativeFixture(t *testing.T) (*readerVolumeAPI, map[string]strin return &readerVolumeAPI{inner: a, name: text(child(pv, "metadata"), "name")}, env } +// TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent compares the complete graph across two invocations. func TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent(t *testing.T) { a, env := readerVolumeNativeFixture(t) - before, _ := json.Marshal(a.inner.objects) + before, err := json.Marshal(a.inner.objects) + if err != nil { + t.Fatal(err) + } for pass := 0; pass < 2; pass++ { var out bytes.Buffer checks := 0 @@ -113,6 +122,7 @@ func TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent } } +// TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess verifies every admission field before native reads or writes. func TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess(t *testing.T) { for _, field := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "GITHUB_WORKFLOW_REF", "GITHUB_RUN_ID", "GITHUB_SHA", "EMAIL_RETAINED_VOLUME_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID", "EMAIL_RETAINED_PV_UID"} { t.Run(field, func(t *testing.T) { @@ -129,6 +139,7 @@ func TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess(t *testing.T } } +// TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation models identity races, lost responses and changed readback. func TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation(t *testing.T) { for _, failure := range []string{"stale-version", "replacement", "lost-response", "changed-readback", "changed-before-write", "changed-after-write"} { t.Run(failure, func(t *testing.T) { @@ -167,6 +178,7 @@ func TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation(t *t } } +// TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions refuses secret reads and any unqualified mutation payload. func TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions(t *testing.T) { for _, args := range [][]string{ {"get", "Secret", "storage-key", "--ignore-not-found=true", "-o=json", "--namespace=" + retainedRecovery.namespace}, @@ -191,6 +203,7 @@ func TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions(t *te } } +// TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch executes the CLI to distinguish admission from source verification. func TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch(t *testing.T) { if os.Getenv("EMAIL_RETAINED_VOLUME_CHILD") == "1" { os.Args = []string{"verify-email-data-recovery", "--protect-retained-volume"} @@ -209,8 +222,8 @@ func TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch(t *testing.T) { stage = "source" } output, err := command.CombinedOutput() - exit, ok := err.(*exec.ExitError) - if !ok || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_RETAINED_VOLUME=UNVERIFIED stage="+stage+" reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_RETAINED_VOLUME=UNVERIFIED stage="+stage+" reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { t.Fatalf("volume command lost its separate dispatch boundary: %q", output) } } diff --git a/scripts/verify-email-data-recovery/reader_volume_retention.go b/scripts/verify-email-data-recovery/reader_volume_retention.go index 89f8abd6cc..2194d09dad 100644 --- a/scripts/verify-email-data-recovery/reader_volume_retention.go +++ b/scripts/verify-email-data-recovery/reader_volume_retention.go @@ -16,6 +16,7 @@ import ( type retainedVolumeWitness struct{ ReaderUID, ClaimUID, PVUID string } +// runRetainedVolumeProtection rebinds the restored graph, fences the Retain-only patch and verifies actual readback. func runRetainedVolumeProtection(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, output io.Writer) int { stage := "dispatch" fail := func() int { @@ -96,6 +97,7 @@ func runRetainedVolumeProtection(ctx context.Context, env func(string) string, k return 0 } +// authorizeRetainedVolumeProtection admits only the confirmed first-attempt main workflow with three distinct native witnesses. func authorizeRetainedVolumeProtection(env func(string) string, witness retainedVolumeWitness) error { for key, want := range map[string]string{ "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", @@ -141,6 +143,7 @@ func (k *retainedVolumeKubernetes) call(ctx context.Context, input []byte, args return raw, nil } +// readRetainedVolume reads and qualifies the independently witnessed restored binding while excluding serving storage. func (d *recoveryDriver) readRetainedVolume(ctx context.Context, witness retainedVolumeWitness) ([]object, error) { fail := func() ([]object, error) { return nil, errors.New("retained-volume-read") } reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) @@ -162,6 +165,7 @@ func (d *recoveryDriver) readRetainedVolume(ctx context.Context, witness retaine return []object{reader, claim, volume}, nil } +// cloneRetainedVolumeObjects takes an independent JSON snapshot for comparison after protection. func cloneRetainedVolumeObjects(objects []object) ([]object, error) { raw, err := json.Marshal(objects) var cloned []object @@ -171,6 +175,7 @@ func cloneRetainedVolumeObjects(objects []object) ([]object, error) { return cloned, nil } +// sameRetainedVolumeInputs compares stable definitions and annotations while allowing API resource versions to advance. func sameRetainedVolumeInputs(before, after []object) bool { if len(before) != 3 || len(after) != 3 { return false @@ -183,6 +188,7 @@ func sameRetainedVolumeInputs(before, after []object) bool { return true } +// validRetainedVolumeWitness requires distinct well-formed reader, claim and volume identities. func validRetainedVolumeWitness(witness retainedVolumeWitness) bool { uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) seen := map[string]bool{} @@ -195,6 +201,7 @@ func validRetainedVolumeWitness(witness retainedVolumeWitness) bool { return true } +// qualifyRetainedVolume verifies the ready restored reader and encrypted binding without assuming an original provisioning name. func qualifyRetainedVolume(reader, claim, volume object, source sourceIdentity, image string, witness retainedVolumeWitness) error { fail := func() error { return errors.New("retained-volume-binding") } if !validRetainedVolumeWitness(witness) { diff --git a/scripts/verify-email-data-recovery/reader_volume_retention_test.go b/scripts/verify-email-data-recovery/reader_volume_retention_test.go index 1d72a354ab..bf87500550 100644 --- a/scripts/verify-email-data-recovery/reader_volume_retention_test.go +++ b/scripts/verify-email-data-recovery/reader_volume_retention_test.go @@ -5,6 +5,7 @@ import ( "testing" ) +// retainedVolumeFixture models a restored claim whose UID differs from the volume's original provisioning identity. func retainedVolumeFixture(t *testing.T) (object, object, object, sourceIdentity, retainedVolumeWitness) { t.Helper() graph, readerUID, claimUID := readerInputFixture(t) @@ -26,6 +27,7 @@ func retainedVolumeFixture(t *testing.T) (object, object, object, sourceIdentity return reader, claim, volume, source, witness } +// TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt verifies both permitted policies without mutating the graph. func TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt(t *testing.T) { for _, policy := range []string{"Delete", "Retain"} { t.Run(policy, func(t *testing.T) { @@ -43,6 +45,7 @@ func TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt(t } } +// TestRetainedVolumeQualificationRejectsChangedOrForeignBindings rejects drift in each independent identity and storage boundary. func TestRetainedVolumeQualificationRejectsChangedOrForeignBindings(t *testing.T) { for _, change := range []string{"reader-witness", "claim-witness", "pv-witness", "same-witness", "reader-uid", "claim-uid", "pv-uid", "reader-version", "pv-version", "pv-terminating", "claim-terminating", "reader-terminating", "reader-command", "reader-not-ready", "reader-owner", "source-claim", "source-volume", "source-pv-uid", "missing-source", "claim-namespace", "claim-unbound", "pv-name", "pv-kind", "pv-unbound", "claim-reference", "foreign-reference", "driver", "handle", "unencrypted", "key-reference", "foreign-key-namespace", "unknown-policy", "storage-class"} { t.Run(change, func(t *testing.T) { diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 01eccca930..cfd8fef81d 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -14,14 +14,17 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { checkRetainedWorkflow(t, "dispose-retained-email-copy.yaml", "dispose", "dispose-retained-email-copy", "--dispose-retained", "EMAIL_RETAINED_COPY_CONFIRM", retainedDispatch()) } +// TestReaderInputCustodyWorkflowMainGuardAndCustody exercises the authored guard before credentials are restored. func TestReaderInputCustodyWorkflowMainGuardAndCustody(t *testing.T) { checkRetainedWorkflow(t, "prepare-retained-email-reader-inputs.yaml", "prepare", "prepare-retained-email-reader-inputs", "--prepare-reader-inputs", "EMAIL_READER_INPUTS_CONFIRM", readerCustodyDispatch()) } +// TestRetainedVolumeProtectionWorkflowMainGuardAndCustody checks the metadata-only workflow and independent volume witness. func TestRetainedVolumeProtectionWorkflowMainGuardAndCustody(t *testing.T) { checkRetainedWorkflow(t, "protect-retained-email-volume.yaml", "protect", "protect-retained-email-volume", "--protect-retained-volume", "EMAIL_RETAINED_VOLUME_CONFIRM", readerVolumeDispatch()) } +// checkRetainedWorkflow reads only filenames supplied by the fixed workflow tests and executes each authored admission guard. func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confirmEnv string, env map[string]string) { t.Helper() data, err := os.ReadFile("../../.github/workflows/" + filename) From 575ce3b5675db7a0bd1dac113b07b5cc0534827a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 22:21:40 +0100 Subject: [PATCH 7/7] fix(email): bound retained recovery tool download retries Apply the reviewed pinned-download retry policy to both preparation workflows. Retry only the tool transfer; checksum failures and confirmed production operations remain single attempts. --- .github/workflows/prepare-retained-email-reader-inputs.yaml | 2 +- .github/workflows/protect-retained-email-volume.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prepare-retained-email-reader-inputs.yaml b/.github/workflows/prepare-retained-email-reader-inputs.yaml index e501a78c59..27e8801531 100644 --- a/.github/workflows/prepare-retained-email-reader-inputs.yaml +++ b/.github/workflows/prepare-retained-email-reader-inputs.yaml @@ -57,7 +57,7 @@ jobs: run: | set -euo pipefail kubectl_path="$RUNNER_TEMP/email-reader-kubectl" - curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \ "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check mkdir -p "$RUNNER_TEMP/email-reader-tools" diff --git a/.github/workflows/protect-retained-email-volume.yaml b/.github/workflows/protect-retained-email-volume.yaml index 54574a32fa..d02d31c579 100644 --- a/.github/workflows/protect-retained-email-volume.yaml +++ b/.github/workflows/protect-retained-email-volume.yaml @@ -57,7 +57,7 @@ jobs: run: | set -euo pipefail kubectl_path="$RUNNER_TEMP/email-volume-kubectl" - curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \ "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check mkdir -p "$RUNNER_TEMP/email-volume-tools"