diff --git a/.github/workflows/prepare-retained-email-reader-inputs.yaml b/.github/workflows/prepare-retained-email-reader-inputs.yaml new file mode 100644 index 0000000000..27e8801531 --- /dev/null +++ b/.github/workflows/prepare-retained-email-reader-inputs.yaml @@ -0,0 +1,120 @@ +name: Prepare Retained Email Reader Inputs + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: prepare-retained-email-reader-inputs" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + prepare: + name: Preserve the fixed retained reader inputs + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != prepare-retained-email-reader-inputs ]]; then + printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=dispatch restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Build the reviewed command before restoring credentials + shell: bash + run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery + + - name: Install the repository-approved Kubernetes client + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-reader-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-reader-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-reader-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-reader-tools" >> "$GITHUB_PATH" + + - name: Copy and verify the exact retained inputs without pausing + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} + EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_READER_INPUTS_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + fail() { printf '%s\n' 'EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=workspace restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; } + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + private_dir="$(mktemp -d /dev/shm/email-reader-inputs.XXXXXXXX)" + forward_pid= + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$private_dir" + } + trap cleanup EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + # The helper sends GitHub's masking commands on stderr. Preserve + # that stream so later native-client output remains redacted. + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" > /dev/null 2>"$private_dir/source.log"; then fail; fi + kc=(kubectl --context=admin@prod --request-timeout=20s) + if ! "${kc[@]}" -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$private_dir/reader.log" | + base64 --decode > "$private_dir/token" 2>"$private_dir/decode.log"; then fail; fi + [[ -s "$private_dir/token" ]] || fail + "${kc[@]}" -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$private_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > "$private_dir/health.json" 2>"$private_dir/health.log"; then + if jq -e '.initialized==true and .sealed==false and .standby==false' "$private_dir/health.json" > /dev/null; then ready=true; break; fi + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + export EMAIL_READER_INPUTS_WORK_DIR="$private_dir" + "$RUNNER_TEMP/verify-email-data-recovery" --prepare-reader-inputs diff --git a/.github/workflows/protect-retained-email-volume.yaml b/.github/workflows/protect-retained-email-volume.yaml new file mode 100644 index 0000000000..d02d31c579 --- /dev/null +++ b/.github/workflows/protect-retained-email-volume.yaml @@ -0,0 +1,100 @@ +name: Protect Retained Email Volume + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: protect-retained-email-volume" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + protect: + name: Retain the fixed recovery volume without stopping its reader + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_ACTIONS" != true || "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != protect-retained-email-volume ]]; then + printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=dispatch reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Build the reviewed command before restoring credentials + shell: bash + run: go build -o "$RUNNER_TEMP/verify-email-data-recovery" ./scripts/verify-email-data-recovery + + - name: Install the repository-approved Kubernetes client + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-volume-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 --retry 3 --retry-all-errors --retry-delay 5 --retry-max-time 120 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-volume-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-volume-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-volume-tools" >> "$GITHUB_PATH" + + - name: Protect only the independently witnessed existing volume + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} + EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_RETAINED_PV_UID: ${{ secrets.EMAIL_RETAINED_PV_UID }} + EMAIL_RETAINED_VOLUME_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + ulimit -c 0 + fail() { printf '%s\n' 'EMAIL_RETAINED_VOLUME=UNVERIFIED stage=workspace reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED' >&2; exit 2; } + uid='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + [[ "$EMAIL_RETAINED_READER_UID" =~ $uid && "$EMAIL_RETAINED_CLAIM_UID" =~ $uid && "$EMAIL_RETAINED_PV_UID" =~ $uid ]] || fail + [[ "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_CLAIM_UID" && "$EMAIL_RETAINED_READER_UID" != "$EMAIL_RETAINED_PV_UID" && "$EMAIL_RETAINED_CLAIM_UID" != "$EMAIL_RETAINED_PV_UID" ]] || fail + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + private_dir="$(mktemp -d /dev/shm/email-volume-protection.XXXXXXXX)" + trap 'rm -rf "$private_dir"' EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + if ! ./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA" >/dev/null 2>"$private_dir/source.log"; then + fail + fi + "$RUNNER_TEMP/verify-email-data-recovery" --protect-retained-volume diff --git a/scripts/verify-email-data-recovery/README.md b/scripts/verify-email-data-recovery/README.md index 8f38a7f065..2d3e93a3d9 100644 --- a/scripts/verify-email-data-recovery/README.md +++ b/scripts/verify-email-data-recovery/README.md @@ -92,6 +92,48 @@ comparison. `EMAIL_RETAINED_PARITY=VERIFIED` reports present message parity. The original trial stays unverified and disposal still requires its own backup and ownership checks. +The separately confirmed `Prepare Retained Email Reader Inputs` workflow +prepares custody for the original retained reader. It runs only on the first +maintainer-dispatched attempt from current reviewed main, under the production +deployment lock. It binds the independently recorded reader and claim, reads +the exact retained encryption key, recovery authentication, configuration and +declared workload inputs, and rechecks their identities and contents immediately +before the one conditional OpenBao write. Kubernetes access is read-only. + +The destination is a create-only, original-version KV2 record under +`infrastructure/email/recovery/retained-reader-inputs`. The command uses CAS=0; +an identical existing record is read back without rewriting it. The command +never updates or deletes the record. Changed, +deleted, newer, malformed or unreadable records refuse preparation. A failed +or lost write response is uncertain and is not retried. Original inputs remain +in place on every outcome. Neither credentials nor resource identities are +printed; the short-lived credential files exist only in runner memory. + +`EMAIL_READER_INPUT_CUSTODY=STORED` proves the observed record matches those +retained inputs. Its receipt explicitly says `restart=NOT_AUTHORIZED` and +`recovery=UNVERIFIED`: independent recovery of the stored inputs, a durable +restart witness, the separate protected pause/resume operations and native +detachment proof remain required. This workflow never stops the reader, +allocates storage, changes the source mailbox or grants disposal authority. +The expired-backup disposal gate remains unchanged. If custody is refused, +investigate the retained identities and destination before another confirmed +first attempt; never replace a conflicting record or regenerate a missing key. + +`Protect Retained Email Volume` is a separate, default-off main-only operation +with confirmation `protect-retained-email-volume`. It requires independent +reader, claim and PV UID witnesses, including the private +`EMAIL_RETAINED_PV_UID` setting, and validates the actual restored binding. +It deliberately does not assume that the original PV name contains the current +restored claim's UID. The command preserves the running reader and every +claim, backend, key and configuration object. Its only permitted mutation is +changing the bound original PV's reclaim policy from `Delete` to `Retain`, +with atomic UID, name, resource-version and complete-spec tests. It then +rechecks the original graph and serving source. An already-retained PV is +verified without a write; an uncertain patch is never retried or rolled back. +No credential export or reader lifecycle authority is available in this +command. Protection does not unlock either expired-backup disposal path, +establish key recovery or authorize a later reader pause. + With confirmation `verify-fresh-retained-email-parity`, the workflow checks the separately retained fresh restore. The protected witness binds the namespace, reader, claim, backup and download. The backup must still be diff --git a/scripts/verify-email-data-recovery/main.go b/scripts/verify-email-data-recovery/main.go index 2563cc74f6..667afbe1b5 100644 --- a/scripts/verify-email-data-recovery/main.go +++ b/scripts/verify-email-data-recovery/main.go @@ -12,7 +12,18 @@ import ( "time" ) +// main selects one independently authorized recovery operation; unknown flags fail closed. func main() { + if len(os.Args) == 2 && os.Args[1] == "--protect-retained-volume" { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + os.Exit(runRetainedVolumeProtection(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, os.Stdout)) + } + if len(os.Args) == 2 && os.Args[1] == "--prepare-reader-inputs" { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + os.Exit(runReaderInputCustody(ctx, os.Getenv, kubectl{}, verifyReaderCustodySource, loadReaderCustodyStore, os.Stdout)) + } if len(os.Args) == 2 && os.Args[1] == "--dispose-retained" { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/scripts/verify-email-data-recovery/reader_custody_entry.go b/scripts/verify-email-data-recovery/reader_custody_entry.go new file mode 100644 index 0000000000..2fb71bf562 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_entry.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "strings" + "time" +) + +var errReaderInputRead = errors.New("reader-input-read") + +type readerInputKubernetes struct{ inner kubernetes } + +// This command has only fixed native reads. Lifecycle operations cannot borrow +// its credential-custody authority to mutate a Pod or persistent object. +func (k readerInputKubernetes) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + if k.inner == nil || len(input) != 0 || (len(args) != 5 && len(args) != 6) || args[0] != "get" || args[3] != "--ignore-not-found=true" || args[4] != "-o=json" { + return nil, errReaderInputRead + } + namespace := "" + if len(args) == 6 { + if !strings.HasPrefix(args[5], "--namespace=") { + return nil, errReaderInputRead + } + namespace = strings.TrimPrefix(args[5], "--namespace=") + } + name, allowed := args[2], false + switch args[1] { + case "Pod": + allowed = (namespace == retainedRecovery.namespace && name == "reader") || (namespace == "private-mail" && name == "private-mail-0") + case "PersistentVolumeClaim": + allowed = (namespace == retainedRecovery.namespace && name == "data") || (namespace == "private-mail" && name == "private-mail-data") + case "PersistentVolume": + allowed = namespace == "" && strings.HasPrefix(name, "pvc-") && kubernetesName.MatchString(name) + case "Namespace", "StorageClass": + allowed = namespace == "" && name == retainedRecovery.namespace + case "CiliumNetworkPolicy": + allowed = namespace == retainedRecovery.namespace && name == "isolation" + case "ConfigMap": + allowed = namespace == retainedRecovery.namespace && name == "datastore" + case "Secret": + allowed = namespace == retainedRecovery.namespace && (name == "storage-key" || name == "test-authentication") + } + if !allowed { + return nil, errReaderInputRead + } + return k.inner.call(ctx, nil, args...) +} + +// authorizeReaderInputCustody requires the confirmed main workflow and distinct independently recorded copy identities. +func authorizeReaderInputCustody(env func(string) string) error { + for key, expected := range map[string]string{ + "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", + "GITHUB_ACTOR": "devantler", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main", + "EMAIL_READER_INPUTS_CONFIRM": "prepare-retained-email-reader-inputs", + } { + if env(key) != expected { + return errors.New("reader-input-dispatch") + } + } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + reader, claim := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + if !recoveryRunPattern.MatchString(env("GITHUB_RUN_ID")) || !recoveryRevisionPattern.MatchString(env("GITHUB_SHA")) || !uuid.MatchString(reader) || !uuid.MatchString(claim) || reader == claim { + return errors.New("reader-input-dispatch") + } + return nil +} + +// runReaderInputCustody captures and rebinds native inputs before create-only custody without granting restart authority. +func runReaderInputCustody(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, loadStore func(func(string) string) (*readerCustodyStore, error), output io.Writer) int { + stage := "dispatch" + fail := func() int { + if output != nil { + _, _ = fmt.Fprintf(output, "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage=%s restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", stage) + } + return 2 + } + if output == nil || verifyMain == nil || loadStore == nil || authorizeReaderInputCustody(env) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_READER_INPUT_CUSTODY=STARTED source_writes=0 restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + stage = "source" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + d := &recoveryDriver{i: retainedRecovery, k: readerInputKubernetes{inner: k}} + var err error + d.source, err = d.readSource(ctx) + if err != nil { + return fail() + } + reader, claim := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + stage = "inputs" + original, err := d.captureCustodyInputs(ctx, reader, claim) + if err != nil { + return fail() + } + stage = "credentials" + store, err := loadStore(env) + if err != nil || store == nil || store.client == nil { + return fail() + } + defer store.client.CloseIdleConnections() + verify := func(ctx context.Context) error { + // Rebind reviewed source before the last native reads. The store calls + // this after its final non-destructive preflight, just before CAS=0. + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return errors.New("custody-source-changed") + } + fresh, err := d.captureCustodyInputs(ctx, reader, claim) + if err != nil || !reflect.DeepEqual(original, fresh) { + return errors.New("custody-inputs-changed") + } + return nil + } + stage = "custody" + if store.ensure(ctx, original, verify) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_READER_INPUT_CUSTODY=STORED source_writes=0 restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + return 0 +} + +// captureCustodyInputs reads the fixed retained graph and excludes the serving claim before freezing credential values. +func (d *recoveryDriver) captureCustodyInputs(ctx context.Context, readerUID, claimUID string) (retainedReaderInputs, error) { + fail := func() (retainedReaderInputs, error) { return retainedReaderInputs{}, errors.New("reader-inputs") } + reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) + if err != nil || !found || text(child(reader, "metadata"), "uid") != readerUID { + return fail() + } + claim, found, err := d.lookup(ctx, "PersistentVolumeClaim", "data", retainedRecovery.namespace) + if err != nil || !found || text(child(claim, "metadata"), "uid") != claimUID || claimUID == d.source.claimUID || text(child(claim, "spec"), "volumeName") == d.source.pv { + return fail() + } + var graph []object + for _, want := range append(recoveryResources(retainedRecovery, "", ""), recoveryReader(retainedRecovery, d.image)) { + m := child(want, "metadata") + actual := reader + switch text(want, "kind") { + case "Pod": + case "PersistentVolumeClaim": + actual = claim + default: + actual, found, err = d.lookup(ctx, text(want, "kind"), text(m, "name"), text(m, "namespace")) + if err != nil || !found { + return fail() + } + } + graph = append(graph, actual) + } + bundle, err := captureRetainedReaderInputs(retainedRecovery, d.image, readerUID, claimUID, d.source.claimUID, graph) + if err != nil || d.sourceRetained(ctx) != nil { + return fail() + } + return bundle, nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_entry_test.go b/scripts/verify-email-data-recovery/reader_custody_entry_test.go new file mode 100644 index 0000000000..7494fb5aa6 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_entry_test.go @@ -0,0 +1,201 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// readerCustodyDispatch supplies the fixed first-attempt workflow identity for custody tests. +func readerCustodyDispatch() map[string]string { + env := confirmedInvocation() + env["GITHUB_ACTIONS"] = "true" + env["GITHUB_WORKFLOW_REF"] = "devantler-tech/platform/.github/workflows/prepare-retained-email-reader-inputs.yaml@refs/heads/main" + env["EMAIL_READER_INPUTS_CONFIRM"] = "prepare-retained-email-reader-inputs" + return env +} + +// readerCustodyNativeFixture separates retained inputs from a distinct serving mailbox graph. +func readerCustodyNativeFixture(t *testing.T) (*cleanupAPI, string, string) { + t.Helper() + t.Chdir("../..") + graph, reader, claim := readerInputFixture(t) + image, err := loadRecoveryImage() + if err != nil { + t.Fatal(err) + } + items(child(graph[7], "spec"), "containers")[0]["image"] = image + a := &cleanupAPI{objects: map[string]object{}} + for _, o := range graph { + m := child(o, "metadata") + a.objects[cleanupKey(text(o, "kind"), text(m, "namespace"), text(m, "name"))] = o + } + pod, pvc, pv := servingFixture() + items(child(pod, "spec"), "containers")[0]["image"] = image + child(pvc, "metadata")["uid"] = "ffffffff-1111-2222-3333-444444444444" + child(pvc, "spec")["volumeName"] = "pvc-serving-volume" + child(pv, "metadata")["name"] = "pvc-serving-volume" + child(child(pv, "spec"), "claimRef")["uid"] = child(pvc, "metadata")["uid"] + child(child(pv, "spec"), "csi")["volumeHandle"] = "pvc-serving-volume" + a.objects[cleanupKey("Pod", "private-mail", "private-mail-0")] = pod + a.objects[cleanupKey("PersistentVolumeClaim", "private-mail", "private-mail-data")] = pvc + a.objects[cleanupKey("PersistentVolume", "", "pvc-serving-volume")] = pv + return a, reader, claim +} + +// TestReaderCustodyDispatchRefusesBeforeAnyAccess proves invalid dispatch never reads credentials or invokes native clients. +func TestReaderCustodyDispatchRefusesBeforeAnyAccess(t *testing.T) { + for _, invalid := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_RUN_ATTEMPT", "GITHUB_ACTOR", "GITHUB_WORKFLOW_REF", "RUNNER_ENVIRONMENT", "GITHUB_SHA", "GITHUB_RUN_ID", "EMAIL_READER_INPUTS_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID"} { + t.Run(invalid, func(t *testing.T) { + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "11111111-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = "22222222-1111-2222-3333-444444444444" + env[invalid] = "invalid" + k := &refusedAPI{} + callbacks := 0 + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, k, + func(context.Context, string) error { callbacks++; return nil }, + func(func(string) string) (*readerCustodyStore, error) { callbacks++; return nil, nil }, &out) + if code != 2 || k.calls != 0 || callbacks != 0 || !strings.Contains(out.String(), "stage=dispatch") { + t.Fatal("foreign dispatch reached credentials, native resources or source execution") + } + }) + } +} + +// TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject checks storage readback and rejects input changes around the write. +func TestReaderCustodyOperatorStoresInputsWithoutChangingAnyKubernetesObject(t *testing.T) { + for _, changeAt := range []int{0, 2, 3} { + t.Run(map[int]string{0: "stable", 2: "changed-before-write", 3: "changed-after-write"}[changeAt], func(t *testing.T) { + a, reader, claim := readerCustodyNativeFixture(t) + before, err := json.Marshal(a.objects) + if err != nil { + t.Fatal(err) + } + posts, sourceChecks := 0, 0 + var saved object + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + var payload object + if json.NewDecoder(r.Body).Decode(&payload) != nil || child(payload, "options")["cas"] != float64(0) { + t.Error("operator escaped create-only custody") + } + saved = child(payload, "data") + _, _ = w.Write([]byte(`{"data":{"version":1}}`)) + return + } + if r.URL.Path == retainedReaderDataPath && saved != nil { + custodyResponse(t, w, saved, 1) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"], env["EMAIL_RETAINED_CLAIM_UID"] = reader, claim + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, a, + func(context.Context, string) error { + sourceChecks++ + if changeAt != 0 && sourceChecks == changeAt { + child(a.objects[cleanupKey("ConfigMap", retainedRecovery.namespace, "datastore")], "data")["config.json"] = `{}` + } + return nil + }, func(func(string) string) (*readerCustodyStore, error) { + return newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + }, &out) + if changeAt != 0 { + expectedPosts := 0 + if changeAt == 3 { + expectedPosts = 1 + } + if code != 2 || posts != expectedPosts || strings.Contains(out.String(), "CUSTODY=STORED") { + t.Fatal("changed native inputs produced a write or a successful custody receipt") + } + return + } + after, err := json.Marshal(a.objects) + if code != 0 || posts != 1 || sourceChecks != 3 || err != nil || !bytes.Equal(before, after) { + t.Fatalf("operator failed its read-only native boundary: code=%d posts=%d checks=%d output=%q", code, posts, sourceChecks, out.String()) + } + if !strings.Contains(out.String(), "restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(out.String(), reader) || strings.Contains(out.String(), "CRYPTO_KEY_VALUE") || strings.Contains(out.String(), "fixture-reader-custody-token") { + t.Fatal("custody receipt exposed inputs or granted lifecycle/recovery authority") + } + }) + } +} + +// TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead verifies rejected requests never reach the underlying client. +func TestReaderCustodyKubernetesCapabilityHasNoMutationOrForeignSecretRead(t *testing.T) { + for _, args := range [][]string{ + {"delete", "Pod", "reader"}, {"create", "-f=-"}, {"patch", "Secret", "storage-key"}, + {"get", "Secret", "openbao-unseal", "--ignore-not-found=true", "-o=json", "--namespace=openbao"}, + {"get", "Pod", "reader", "--ignore-not-found=true", "-o=json", "--namespace=private-mail"}, + {"get", "Secret", "storage-key", "--ignore-not-found=true", "-o=json", "--namespace=" + retainedRecovery.namespace, "--raw=/"}, + } { + k := &refusedAPI{} + _, err := (readerInputKubernetes{inner: k}).call(context.Background(), nil, args...) + if err == nil || k.calls != 0 { + t.Fatal("input custody capability reached a mutation or foreign resource") + } + } + if _, err := (readerInputKubernetes{}).call(context.Background(), nil); !errors.Is(err, errReaderInputRead) { + t.Fatal("missing native reader did not fail closed") + } +} + +// TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess prevents silent operation without source and output evidence. +func TestReaderCustodyLostOutputOrUnverifiedSourceStopsBeforeNativeAccess(t *testing.T) { + for _, failure := range []string{"output", "source"} { + t.Run(failure, func(t *testing.T) { + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "11111111-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = "22222222-1111-2222-3333-444444444444" + k := &refusedAPI{} + var out bytes.Buffer + writer := &observationFailingWriter{base: &out, remaining: 10} + if failure == "output" { + writer.remaining = 0 + } + loaded := 0 + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, k, + func(context.Context, string) error { return errors.New("unverified current main") }, + func(func(string) string) (*readerCustodyStore, error) { loaded++; return nil, nil }, writer) + if code != 2 || k.calls != 0 || loaded != 0 || strings.Contains(out.String(), "CUSTODY=STORED") { + t.Fatal("lost output or source proof reached native credentials") + } + }) + } +} + +// TestReaderCustodyWrongIndependentReaderStopsBeforeKeyAccess checks identity before any retained or service credential is read. +func TestReaderCustodyWrongIndependentReaderStopsBeforeKeyAccess(t *testing.T) { + a, _, claim := readerCustodyNativeFixture(t) + secretReads, loaded := 0, 0 + a.onGet = func(key string) { + if strings.HasPrefix(key, "Secret/") { + secretReads++ + } + } + env := readerCustodyDispatch() + env["EMAIL_RETAINED_READER_UID"] = "eeeeeeee-1111-2222-3333-444444444444" + env["EMAIL_RETAINED_CLAIM_UID"] = claim + var out bytes.Buffer + code := runReaderInputCustody(context.Background(), func(key string) string { return env[key] }, a, + func(context.Context, string) error { return nil }, + func(func(string) string) (*readerCustodyStore, error) { loaded++; return nil, nil }, &out) + if code != 2 || secretReads != 0 || loaded != 0 || !strings.Contains(out.String(), "stage=inputs") { + t.Fatal("a replaced reader reached encryption or store credentials") + } +} diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime.go b/scripts/verify-email-data-recovery/reader_custody_runtime.go new file mode 100644 index 0000000000..5f9ade4821 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_runtime.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "time" +) + +// readReaderCustodyToken reads a bounded regular token from a private directory without exposing failures or credential contents. +func readReaderCustodyToken(directory string) (token string, err error) { + fail := func() (string, error) { return "", errors.New("reader-custody-token") } + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + return fail() + } + info, err = os.Lstat(filepath.Join(directory, "token")) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() < 16 || info.Size() > 256 { + return fail() + } + root, err := os.OpenRoot(directory) + if err != nil { + return fail() + } + defer func() { + if root.Close() != nil { + token, err = fail() + } + }() + file, err := root.Open("token") + if err != nil { + return fail() + } + defer func() { + if file.Close() != nil { + token, err = fail() + } + }() + info, err = file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { + return fail() + } + raw, err := io.ReadAll(io.LimitReader(file, 257)) + if err != nil || !regexp.MustCompile(`^[A-Za-z0-9._-]{16,256}$`).Match(raw) { + return fail() + } + return string(raw), nil +} + +// loadReaderCustodyStore accepts only the workflow's private tmpfs workspace and fixed loopback service tunnel. +func loadReaderCustodyStore(env func(string) string) (*readerCustodyStore, error) { + directory := env("EMAIL_READER_INPUTS_WORK_DIR") + if !regexp.MustCompile(`^/dev/shm/email-reader-inputs\.[A-Za-z0-9]{8}$`).MatchString(directory) || filepath.Clean(directory) != directory { + return nil, errors.New("reader-custody-workspace") + } + token, err := readReaderCustodyToken(directory) + if err != nil { + return nil, errors.New("reader-custody-token") + } + return newReaderCustodyStore("http://127.0.0.1:38200", token) +} + +// verifyReaderCustodySource runs the fixed reviewed source guard with a validated revision and bounded execution time. +func verifyReaderCustodySource(ctx context.Context, revision string) error { + if !recoveryRevisionPattern.MatchString(revision) { + return errors.New("reader-custody-source") + } + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "./scripts/verify-prod-recovery-source.sh", revision) + command.Stdout, command.Stderr = io.Discard, io.Discard + command.WaitDelay = 5 * time.Second + if command.Run() != nil { + return errors.New("reader-custody-source") + } + return nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_runtime_test.go b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go new file mode 100644 index 0000000000..6496e98c2e --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_runtime_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestReaderCustodyTokenRequiresPrivateRegularLocalFiles deliberately creates unsafe modes and links to verify refusal without disclosure. +func TestReaderCustodyTokenRequiresPrivateRegularLocalFiles(t *testing.T) { + for _, failure := range []string{"", "public-directory", "public-file", "file-symlink", "directory-symlink", "empty", "oversized", "newline"} { + t.Run(failure, func(t *testing.T) { + dir := t.TempDir() + if os.Chmod(dir, 0700) != nil { + t.Fatal("private fixture directory") + } + value := "fixture-reader-custody-token" + switch failure { + case "empty": + value = "" + case "oversized": + value = strings.Repeat("x", 257) + case "newline": + value += "\n" + } + path := filepath.Join(dir, "token") + if os.WriteFile(path, []byte(value), 0600) != nil { + t.Fatal("private fixture credential") + } + switch failure { + case "public-directory": + if os.Chmod(dir, 0755) != nil { + t.Fatal("fixture mode") + } + case "public-file": + if os.Chmod(path, 0644) != nil { + t.Fatal("fixture mode") + } + case "file-symlink": + if os.Rename(path, path+"-real") != nil || os.Symlink(path+"-real", path) != nil { + t.Fatal("fixture symlink") + } + case "directory-symlink": + link := dir + "-link" + if os.Symlink(dir, link) != nil { + t.Fatal("fixture directory symlink") + } + t.Cleanup(func() { _ = os.Remove(link) }) + dir = link + } + token, err := readReaderCustodyToken(dir) + if failure == "" { + if err != nil || token != value { + t.Fatal("private bounded credential refused") + } + } else if err == nil || token != "" || strings.Contains(err.Error(), value) && value != "" { + t.Fatal("unqualified credential accepted or exposed") + } + }) + } +} + +// TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints checks destination and authentication syntax before any network access. +func TestReaderCustodyTransportRefusesRemoteOrAmbiguousEndpoints(t *testing.T) { + for _, endpoint := range []string{"https://127.0.0.1:38200", "http://example.invalid:38200", "http://localhost:38200", "http://127.0.0.1", "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://user@127.0.0.1:38200", "http://127.0.0.1:38200/path", "http://127.0.0.1:38200?", "http://127.0.0.1:38200#other"} { + if store, err := newReaderCustodyStore(endpoint, "fixture-reader-custody-token"); err == nil || store != nil { + t.Fatal("custody credential could reach an unqualified endpoint") + } + } + if _, err := newReaderCustodyStore("http://127.0.0.1:38200", "token\r\nInjected: value"); err == nil { + t.Fatal("malformed authentication accepted") + } +} + +// TestReaderCustodyCommandUsesTheSeparateProtectedDispatch executes the real CLI with no tools available to expose an incorrect branch. +func TestReaderCustodyCommandUsesTheSeparateProtectedDispatch(t *testing.T) { + if os.Getenv("EMAIL_READER_CUSTODY_CHILD") == "1" { + os.Args = []string{"verify-email-data-recovery", "--prepare-reader-inputs"} + main() + return + } + for _, confirmed := range []bool{false, true} { + command := exec.Command(os.Args[0], "-test.run=^TestReaderCustodyCommandUsesTheSeparateProtectedDispatch$") + command.Env = []string{"EMAIL_READER_CUSTODY_CHILD=1", "PATH=/nonexistent", "GOCOVERDIR=" + t.TempDir()} + stage := "dispatch" + if confirmed { + for key, value := range readerCustodyDispatch() { + command.Env = append(command.Env, key+"="+value) + } + command.Env = append(command.Env, "EMAIL_RETAINED_READER_UID=11111111-1111-2222-3333-444444444444", "EMAIL_RETAINED_CLAIM_UID=22222222-1111-2222-3333-444444444444") + stage = "source" + } + output, err := command.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_READER_INPUT_CUSTODY=UNVERIFIED stage="+stage+" restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + t.Fatalf("custody command lost its distinct dispatch boundary: %q", output) + } + } +} diff --git a/scripts/verify-email-data-recovery/reader_custody_store.go b/scripts/verify-email-data-recovery/reader_custody_store.go new file mode 100644 index 0000000000..2f03dcbd11 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_store.go @@ -0,0 +1,192 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/url" + "regexp" + "strconv" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const retainedReaderDataPath = "/v1/secret/data/infrastructure/email/recovery/retained-reader-inputs" +const retainedReaderMetadataPath = "/v1/secret/metadata/infrastructure/email/recovery/retained-reader-inputs" + +type readerCustodyStore struct { + client *http.Client + endpoint string + token string +} + +// Only the protected workflow's loopback tunnel is accepted. No environment +// proxy, redirect, URL credentials or arbitrary remote destination receives +// the OpenBao token. The caller pins the production tunnel's exact port. +func newReaderCustodyStore(endpoint, token string) (*readerCustodyStore, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "http" || u.Hostname() != "127.0.0.1" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.Opaque != "" || !regexp.MustCompile(`^[A-Za-z0-9._-]{16,256}$`).MatchString(token) { + return nil, errors.New("custody-endpoint") + } + port, err := strconv.Atoi(u.Port()) + if err != nil || port < 1 || port > 65535 { + return nil, errors.New("custody-endpoint") + } + transport := &http.Transport{Proxy: nil, DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, MaxResponseHeaderBytes: 64 << 10} + client := &http.Client{Transport: transport, Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + return &readerCustodyStore{client: client, endpoint: endpoint, token: token}, nil +} + +// readerCustodyDocument binds the complete copy-specific input set to its content digests and fixed record schema. +func readerCustodyDocument(bundle retainedReaderInputs) (object, error) { + fail := func() (object, error) { return nil, errors.New("custody-document") } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + if !uuid.MatchString(bundle.ReaderUID) || !uuid.MatchString(bundle.ClaimUID) || bundle.ReaderUID == bundle.ClaimUID || len(bundle.Inputs) != 8 { + return fail() + } + allowed := map[string]bool{} + for _, want := range append(recoveryResources(retainedRecovery, "", ""), recoveryReader(retainedRecovery, "")) { + m := child(want, "metadata") + allowed[(ownedReference{kind: text(want, "kind"), name: text(m, "name"), namespace: text(m, "namespace")}).key()] = true + } + seen := map[string]bool{} + for _, input := range bundle.Inputs { + ref := ownedReference{kind: input.Kind, name: input.Name, namespace: input.Namespace} + if !allowed[ref.key()] || !uuid.MatchString(input.UID) || seen[input.UID] || input.Values == nil || (input.Kind == "Pod" && input.UID != bundle.ReaderUID) || (input.Kind == "PersistentVolumeClaim" && input.UID != bundle.ClaimUID) { + return fail() + } + delete(allowed, ref.key()) + seen[input.UID] = true + raw, err := json.Marshal(input.Values) + if err != nil { + return fail() + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != input.Digest { + return fail() + } + } + raw, err := json.Marshal(bundle) + if err != nil || len(raw) > 512<<10 { + return fail() + } + var value object + if json.Unmarshal(raw, &value) != nil { + return fail() + } + return object{"schema": "email-reader-inputs/v1", "trial_run": retainedRecovery.run, "trial_revision": retainedRecovery.revision, "bundle": value}, nil +} + +// ensure creates the one fixed record with CAS=0, or verifies an identical +// original version. It has no update, delete, metadata-write or retry-write +// path. verify runs after the final API preflight, immediately before POST, +// then again after readback. A stored record alone authorizes no lifecycle. +func (s *readerCustodyStore) ensure(ctx context.Context, bundle retainedReaderInputs, verify func(context.Context) error) error { + fail := func() error { return errors.New("reader-custody") } + expected, err := readerCustodyDocument(bundle) + if err != nil || verify == nil || s == nil || s.client == nil || ctx.Err() != nil { + return fail() + } + mounts, status, err := s.request(ctx, http.MethodGet, "/v1/sys/mounts", nil) + mount := child(child(mounts, "data"), "secret/") + if err != nil || status != http.StatusOK || text(mount, "type") != "kv" || text(child(mount, "options"), "version") != "2" { + return fail() + } + found, err := s.matches(ctx, expected) + if err != nil { + return fail() + } + if !found { + // A soft-deleted value is not a new key. Preserve its metadata and + // refuse before any write, rather than trying to replace its history. + _, status, err = s.request(ctx, http.MethodGet, retainedReaderMetadataPath, nil) + if err != nil || status != http.StatusNotFound { + return fail() + } + } + if verify(ctx) != nil { + return fail() + } + if !found { + payload, err := json.Marshal(object{"options": object{"cas": 0}, "data": expected}) + if err != nil { + return fail() + } + _, status, err = s.request(ctx, http.MethodPost, retainedReaderDataPath, payload) + // A lost or failed response is uncertain and is never retried. A CAS + // conflict can succeed only by reading the identical original record. + if err != nil || (status != http.StatusOK && status != http.StatusBadRequest) { + return fail() + } + } + found, err = s.matches(ctx, expected) + if err != nil || !found || verify(ctx) != nil { + return fail() + } + return nil +} + +// matches accepts only an identical, undeleted original KV2 version; missing data is not recovery proof. +func (s *readerCustodyStore) matches(ctx context.Context, expected object) (bool, error) { + o, status, err := s.request(ctx, http.MethodGet, retainedReaderDataPath, nil) + if err != nil { + return false, errors.New("custody-readback") + } + if status == http.StatusNotFound { + return false, nil + } + data := child(o, "data") + metadata := child(data, "metadata") + deleted, hasDeleted := metadata["deletion_time"].(string) + destroyed, hasDestroyed := metadata["destroyed"].(bool) + _, timeErr := time.Parse(time.RFC3339Nano, text(metadata, "created_time")) + actualBytes, actualErr := json.Marshal(data["data"]) + expectedBytes, expectedErr := json.Marshal(expected) + if status != http.StatusOK || metadata["version"] != float64(1) || !hasDeleted || deleted != "" || !hasDestroyed || destroyed || timeErr != nil || actualErr != nil || expectedErr != nil || !bytes.Equal(actualBytes, expectedBytes) { + return false, errors.New("custody-readback") + } + return true, nil +} + +// request restricts custody HTTP methods and destinations and rejects ambiguous or oversized responses. +func (s *readerCustodyStore) request(ctx context.Context, method, path string, payload []byte) (object, int, error) { + fail := func() (object, int, error) { return nil, 0, errors.New("custody-request") } + if (method != http.MethodGet || (path != "/v1/sys/mounts" && path != retainedReaderDataPath && path != retainedReaderMetadataPath)) && (method != http.MethodPost || path != retainedReaderDataPath) { + return fail() + } + req, err := http.NewRequestWithContext(ctx, method, s.endpoint+path, bytes.NewReader(payload)) + if err != nil { + return fail() + } + req.Header.Set("X-Vault-Token", s.token) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/json") + } + response, err := s.client.Do(req) + if err != nil { + return fail() + } + defer func() { _ = response.Body.Close() }() + raw, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil || len(raw) > 1<<20 { + return fail() + } + if len(bytes.TrimSpace(raw)) == 0 { + return object{}, response.StatusCode, nil + } + if strictjson.Validate(raw, nil) != nil { + return fail() + } + var value object + if json.Unmarshal(raw, &value) != nil || value == nil { + return fail() + } + return value, response.StatusCode, nil +} diff --git a/scripts/verify-email-data-recovery/reader_custody_store_test.go b/scripts/verify-email-data-recovery/reader_custody_store_test.go new file mode 100644 index 0000000000..353bc2cf0f --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_custody_store_test.go @@ -0,0 +1,304 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +// custodyBundle freezes independently witnessed fixture inputs for storage protocol tests. +func custodyBundle(t *testing.T) retainedReaderInputs { + t.Helper() + graph, reader, claim := readerInputFixture(t) + bundle, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil { + t.Fatal(err) + } + return bundle +} + +// custodyResponse models KV2 data with explicit original-version and deletion metadata. +func custodyResponse(t *testing.T, w http.ResponseWriter, data any, version int) { + t.Helper() + if err := json.NewEncoder(w).Encode(object{"data": object{"data": data, "metadata": object{"version": version, "destroyed": false, "deletion_time": "", "created_time": "2026-01-01T00:00:00Z"}}}); err != nil { + t.Error(err) + } +} + +// TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite checks the final native rebind, readback and zero-overwrite repeat. +func TestReaderCustodyCreatesOnlyOnceAndRebindsImmediatelyBeforeWrite(t *testing.T) { + bundle := custodyBundle(t) + var stored object + posts, preflight, rebound := 0, false, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Vault-Token") != "fixture-reader-custody-token" { + t.Error("request lost its private authentication header") + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/sys/mounts": + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + case r.Method == http.MethodGet && r.URL.Path == retainedReaderMetadataPath: + preflight = true + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == retainedReaderDataPath: + if stored == nil { + w.WriteHeader(http.StatusNotFound) + return + } + custodyResponse(t, w, stored, 1) + case r.Method == http.MethodPost && r.URL.Path == retainedReaderDataPath: + posts++ + if !preflight || rebound != 1 { + t.Error("custody mutation preceded final native-input rebinding") + } + var payload object + if json.NewDecoder(r.Body).Decode(&payload) != nil || child(payload, "options")["cas"] != float64(0) || len(child(payload, "options")) != 1 || len(payload) != 2 { + t.Error("write lost its create-only boundary") + } + stored = child(payload, "data") + _, _ = w.Write([]byte(`{"data":{"version":1}}`)) + default: + t.Errorf("unexpected credential operation: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusForbidden) + } + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + verify := func(context.Context) error { + if !preflight { + t.Error("native verification happened before the final non-destructive preflight") + } + rebound++ + return nil + } + if err = store.ensure(context.Background(), bundle, verify); err != nil { + t.Fatalf("valid create-only custody failed: %v", err) + } + if posts != 1 || rebound != 2 || stored["schema"] != "email-reader-inputs/v1" { + t.Fatal("custody did not read back and rebind the exact stored inputs") + } + expected, err := readerCustodyDocument(bundle) + if err != nil || !reflect.DeepEqual(stored, expected) { + t.Fatal("stored credentials differ from the independently captured copy") + } + // A new invocation may verify the same immutable record, never overwrite it. + if err = store.ensure(context.Background(), bundle, verify); err != nil || posts != 1 { + t.Fatal("idempotent verification rewrote the existing custody record") + } +} + +// TestReaderCustodyRejectsChangedInputsAfterFinalPreflight models drift in the last interval before a credential write. +func TestReaderCustodyRejectsChangedInputsAfterFinalPreflight(t *testing.T) { + posts, preflight := 0, false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + posts++ + } + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.URL.Path == retainedReaderMetadataPath { + preflight = true + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { + if preflight { + return errors.New("native inputs changed") + } + return nil + }) + if err == nil || posts != 0 { + t.Fatal("changed native inputs authorized a credential write") + } +} + +// TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords preserves existing history when API evidence cannot establish safe creation. +func TestReaderCustodyRefusesAmbiguousExistingOrUnreadableRecords(t *testing.T) { + for _, failure := range []string{"different", "newer-version", "deleted", "destroyed", "absent-metadata", "unreadable", "duplicate-json", "oversized", "wrong-mount"} { + t.Run(failure, func(t *testing.T) { + posts := 0 + bundle := custodyBundle(t) + document, err := readerCustodyDocument(bundle) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + posts++ + } + if r.URL.Path == "/v1/sys/mounts" { + version := "2" + if failure == "wrong-mount" { + version = "1" + } + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"` + version + `"}}}}`)) + return + } + if r.URL.Path == retainedReaderMetadataPath { + _, _ = w.Write([]byte(`{"data":{"current_version":1}}`)) + return + } + switch failure { + case "deleted": + w.WriteHeader(http.StatusNotFound) + case "destroyed": + _, _ = w.Write([]byte(`{"data":{"data":{},"metadata":{"version":1,"destroyed":true,"deletion_time":""}}}`)) + case "absent-metadata": + _ = json.NewEncoder(w).Encode(object{"data": object{"data": document}}) + case "unreadable": + w.WriteHeader(http.StatusForbidden) + case "duplicate-json": + _, _ = w.Write([]byte(`{"data":{},"data":{}}`)) + case "oversized": + _, _ = w.Write([]byte(strings.Repeat("x", (1<<20)+1))) + case "newer-version": + custodyResponse(t, w, document, 2) + default: + custodyResponse(t, w, object{"schema": "foreign"}, 1) + } + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), bundle, func(context.Context) error { return nil }) + if err == nil || posts != 0 || strings.Contains(err.Error(), "fixture-reader-custody-token") { + t.Fatal("ambiguous custody state caused a write or exposed authentication") + } + }) + } +} + +// TestReaderCustodyNeverFollowsCredentialRedirects uses a second server to detect any forwarded authentication request. +func TestReaderCustodyNeverFollowsCredentialRedirects(t *testing.T) { + forwarded := 0 + other := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { forwarded++ })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + if store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { return nil }) == nil || forwarded != 0 { + t.Fatal("a redirect could relay the custody authentication") + } +} + +// TestReaderCustodyCASConflictCannotOverwriteAnotherRecord distinguishes an identical concurrent winner from foreign custody. +func TestReaderCustodyCASConflictCannotOverwriteAnotherRecord(t *testing.T) { + for _, identical := range []bool{false, true} { + t.Run(map[bool]string{false: "foreign-winner", true: "identical-winner"}[identical], func(t *testing.T) { + bundle := custodyBundle(t) + winner, err := readerCustodyDocument(bundle) + if err != nil { + t.Fatal(err) + } + if !identical { + winner = object{"schema": "another-recovery-point"} + } + posts := 0 + var stored object + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + stored = winner + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"errors":["check-and-set conflict"]}`)) + return + } + if r.URL.Path == retainedReaderDataPath && stored != nil { + custodyResponse(t, w, stored, 1) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), bundle, func(context.Context) error { return nil }) + if (err == nil) != identical || posts != 1 || !reflect.DeepEqual(stored, winner) { + t.Fatal("CAS conflict rewrote a winner or accepted different recovery inputs") + } + }) + } +} + +// TestReaderCustodyLostWriteResponseIsNotRetriedOrReportedStored models a committed request whose response connection is lost. +func TestReaderCustodyLostWriteResponseIsNotRetriedOrReportedStored(t *testing.T) { + posts, postWriteReads := 0, 0 + committed := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sys/mounts" { + _, _ = w.Write([]byte(`{"data":{"secret/":{"type":"kv","options":{"version":"2"}}}}`)) + return + } + if r.Method == http.MethodPost { + posts++ + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("HTTP fixture cannot model a lost committed response") + w.WriteHeader(http.StatusInternalServerError) + close(committed) + return + } + connection, _, err := hijacker.Hijack() + if err != nil { + t.Error(err) + } else { + _ = connection.Close() + } + close(committed) + return + } + if posts != 0 { + postWriteReads++ + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + store, err := newReaderCustodyStore(server.URL, "fixture-reader-custody-token") + if err != nil { + t.Fatal(err) + } + defer store.client.CloseIdleConnections() + err = store.ensure(context.Background(), custodyBundle(t), func(context.Context) error { return nil }) + select { + case <-committed: + case <-time.After(2 * time.Second): + t.Fatal("fixture write never reached its committed-response boundary") + } + if err == nil || posts != 1 || postWriteReads != 0 { + t.Fatal("uncertain write was retried or reclassified as verified") + } +} diff --git a/scripts/verify-email-data-recovery/reader_inputs.go b/scripts/verify-email-data-recovery/reader_inputs.go new file mode 100644 index 0000000000..90eb7e0d79 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_inputs.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "reflect" + "regexp" +) + +type retainedReaderInput struct { + Kind, Name, Namespace, UID, Digest string + Values object +} + +type retainedReaderInputs struct { + ReaderUID, ClaimUID string + Inputs []retainedReaderInput +} + +// This captures custody inputs only. It neither establishes durable custody +// nor authorizes a pause, restart, detach or disposal. In particular it does +// not replace the original backup-validity gate with a credential snapshot. +// Values contain credentials and must stay in the protected private workspace. +func captureRetainedReaderInputs(i invocation, image, readerUID, claimUID, sourceClaimUID string, graph []object) (retainedReaderInputs, error) { + fail := func() (retainedReaderInputs, error) { return retainedReaderInputs{}, errors.New("reader-inputs") } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + imagePin := regexp.MustCompile(`^ghcr\.io/devantler-tech/platform-private-mail-bootstrap@sha256:[0-9a-f]{64}$`) + if i != retainedRecovery || !imagePin.MatchString(image) || !uuid.MatchString(readerUID) || !uuid.MatchString(claimUID) || !uuid.MatchString(sourceClaimUID) || readerUID == claimUID || claimUID == sourceClaimUID { + return fail() + } + wanted := append(recoveryResources(i, "", ""), recoveryReader(i, image)) + if len(graph) != len(wanted) { + return fail() + } + actualByName := map[string]object{} + seenUID := map[string]bool{} + for _, actual := range graph { + m := child(actual, "metadata") + uid := text(m, "uid") + ref := ownedReference{kind: text(actual, "kind"), name: text(m, "name"), namespace: text(m, "namespace")} + if !uuid.MatchString(uid) || seenUID[uid] || !regexp.MustCompile(`^[1-9][0-9]{0,31}$`).MatchString(text(m, "resourceVersion")) || m["deletionTimestamp"] != nil || actualByName[ref.key()] != nil || !i.permitsMutation(ref.kind, ref.namespace, ref.name) { + return fail() + } + if owners, exists := m["ownerReferences"]; exists && owners != nil { + list, ok := owners.([]any) + if !ok || len(list) != 0 { + return fail() + } + } + actualByName[ref.key()] = actual + seenUID[uid] = true + } + bundle := retainedReaderInputs{ReaderUID: readerUID, ClaimUID: claimUID} + for _, want := range wanted { + m := child(want, "metadata") + ref := ownedReference{kind: text(want, "kind"), name: text(m, "name"), namespace: text(m, "namespace")} + actual := actualByName[ref.key()] + if actual == nil || text(actual, "apiVersion") != text(want, "apiVersion") { + return fail() + } + am := child(actual, "metadata") + for key, value := range child(m, "labels") { + if child(am, "labels")[key] != value { + return fail() + } + } + switch ref.kind { + case "Pod": + if text(am, "uid") != readerUID || text(child(actual, "status"), "phase") != "Running" || !admittedRecoveryObject(want, actual) || child(actual, "spec")["terminationGracePeriodSeconds"] != float64(30) { + return fail() + } + ready := false + for _, condition := range items(child(actual, "status"), "conditions") { + if text(condition, "type") == "Ready" && text(condition, "status") == "True" { + ready = true + } + } + if !ready { + return fail() + } + case "PersistentVolumeClaim": + spec := child(actual, "spec") + if text(am, "uid") != claimUID || text(child(actual, "status"), "phase") != "Bound" || !kubernetesName.MatchString(text(spec, "volumeName")) || text(spec, "storageClassName") != i.namespace || !reflect.DeepEqual(spec["accessModes"], []any{"ReadWriteOnce"}) || text(child(child(spec, "resources"), "requests"), "storage") != "2Gi" { + return fail() + } + case "Secret": + if !retainedSecret(actual, ref.name) || actual["stringData"] != nil { + return fail() + } + case "ConfigMap": + if !reflect.DeepEqual(actual["data"], want["data"]) || len(child(actual, "binaryData")) != 0 { + return fail() + } + case "StorageClass": + for _, key := range []string{"provisioner", "reclaimPolicy", "volumeBindingMode", "parameters"} { + if !reflect.DeepEqual(actual[key], want[key]) { + return fail() + } + } + case "CiliumNetworkPolicy": + if !reflect.DeepEqual(actual["spec"], want["spec"]) { + return fail() + } + } + // Marshal/unmarshal creates a private snapshot: later API reads cannot + // mutate values already accepted for the conditional custody write. + canonical, err := json.Marshal(retainedMutableFields(actual)) + if err != nil { + return fail() + } + var frozen object + if json.Unmarshal(canonical, &frozen) != nil { + return fail() + } + digest := sha256.Sum256(canonical) + bundle.Inputs = append(bundle.Inputs, retainedReaderInput{Kind: ref.kind, Name: ref.name, Namespace: ref.namespace, UID: text(am, "uid"), Digest: hex.EncodeToString(digest[:]), Values: frozen}) + } + return bundle, nil +} diff --git a/scripts/verify-email-data-recovery/reader_inputs_test.go b/scripts/verify-email-data-recovery/reader_inputs_test.go new file mode 100644 index 0000000000..dc7f3b2d0b --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_inputs_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" +) + +// The independent witnesses identify the existing copy; labels alone must +// never select which credentials are saved for a later restart. +func readerInputFixture(t *testing.T) ([]object, string, string) { + t.Helper() + graph := append(recoveryResources(retainedRecovery, strings.Repeat("a", 64), "restore:"+strings.Repeat("b", 64)), recoveryReader(retainedRecovery, fixtureImage)) + var readerUID, claimUID string + for index, o := range graph { + m := child(o, "metadata") + m["uid"] = fmt.Sprintf("%08x-1111-2222-3333-444444444444", index+1) + m["resourceVersion"] = fmt.Sprint(index + 100) + if text(o, "kind") == "Secret" { + data := object{} + for k, v := range child(o, "stringData") { + value, ok := v.(string) + if !ok { + t.Fatal("fixture secret value is not a string") + } + data[k] = base64.StdEncoding.EncodeToString([]byte(value)) + } + o["data"] = data + delete(o, "stringData") + } + if text(o, "kind") == "Pod" { + readerUID = text(m, "uid") + o["status"] = object{"phase": "Running", "conditions": []any{object{"type": "Ready", "status": "True"}}} + } + if text(o, "kind") == "PersistentVolumeClaim" { + claimUID = text(m, "uid") + child(o, "spec")["volumeName"] = "pvc-existing-recovery" + o["status"] = object{"phase": "Bound"} + } + } + return graph, readerUID, claimUID +} + +// TestReaderInputBundleKeepsTheActualPerCopyCredentials verifies complete credentials, stable ordering and independence from later mutations. +func TestReaderInputBundleKeepsTheActualPerCopyCredentials(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + bundle, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil { + t.Fatalf("independently bound retained inputs refused: %v", err) + } + if bundle.ReaderUID != reader || bundle.ClaimUID != claim || len(bundle.Inputs) != 8 { + t.Fatal("input custody lost the independent reader or complete restart inputs") + } + secrets := 0 + for _, input := range bundle.Inputs { + if len(input.Digest) != 64 || input.UID == "" { + t.Fatal("restart input lacks a frozen identity/content binding") + } + if input.Kind == "Secret" { + secrets++ + if !reflect.DeepEqual(child(input.Values, "data"), child(graph[map[string]int{"storage-key": 3, "test-authentication": 4}[input.Name]], "data")) { + t.Fatal("operational credentials substituted for this copy's inputs") + } + } + } + if secrets != 2 { + t.Fatal("both retained encryption and recovery authentication are required") + } + graph[0], graph[7] = graph[7], graph[0] + reordered, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, "ffffffff-1111-2222-3333-444444444444", graph) + if err != nil || !reflect.DeepEqual(bundle, reordered) { + t.Fatal("discovery order changed the retained-input identity") + } + before, err := json.Marshal(bundle) + if err != nil { + t.Fatal(err) + } + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = "changed-after-capture" + after, err := json.Marshal(bundle) + if err != nil || string(before) != string(after) { + t.Fatal("later live mutations rewrote the captured custody inputs") + } +} + +// TestReaderInputBundleBindsValidChangedKeyBytes requires a content binding to change when the retained encryption key changes. +func TestReaderInputBundleBindsValidChangedKeyBytes(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + source := "ffffffff-1111-2222-3333-444444444444" + before, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, source, graph) + if err != nil { + t.Fatal(err) + } + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = base64.StdEncoding.EncodeToString([]byte(strings.Repeat("c", 64))) + after, err := captureRetainedReaderInputs(retainedRecovery, fixtureImage, reader, claim, source, graph) + if err != nil { + t.Fatal(err) + } + if before.Inputs[3].Digest == after.Inputs[3].Digest { + t.Fatal("a different valid encryption key inherited the original content binding") + } +} + +// TestReaderInputBundleRefusesBorrowedReplacedOrIncompleteInputs exercises foreign identity, missing input and configuration substitutions. +func TestReaderInputBundleRefusesBorrowedReplacedOrIncompleteInputs(t *testing.T) { + for _, failure := range []string{"reader", "claim", "source", "missing", "duplicate", "key", "authentication", "configuration", "storage-class", "executable", "not-ready", "terminating", "controller", "namespace", "identity", "resource-version", "labels", "foreign-target"} { + t.Run(failure, func(t *testing.T) { + graph, reader, claim := readerInputFixture(t) + source := "ffffffff-1111-2222-3333-444444444444" + target := retainedRecovery + switch failure { + case "reader": + reader = "eeeeeeee-1111-2222-3333-444444444444" + case "claim": + claim = "eeeeeeee-1111-2222-3333-444444444444" + case "source": + source = claim + case "missing": + graph = graph[:len(graph)-1] + case "duplicate": + graph = append(graph, graph[0]) + case "key": + child(graph[3], "data")["CRYPTO_KEY_VALUE"] = "not-base64" + case "authentication": + child(graph[4], "data")["credential"] = base64.StdEncoding.EncodeToString([]byte("another-account")) + case "configuration": + child(graph[5], "data")["config.json"] = `{}` + case "storage-class": + child(graph[2], "parameters")["csi.storage.k8s.io/node-stage-secret-namespace"] = "private-mail" + case "executable": + items(child(graph[7], "spec"), "containers")[0]["command"] = []any{"unreviewed"} + case "not-ready": + graph[7]["status"] = object{"phase": "Pending"} + case "terminating": + child(graph[3], "metadata")["deletionTimestamp"] = "2026-01-01T00:00:00Z" + case "controller": + child(graph[7], "metadata")["ownerReferences"] = []any{object{"controller": true}} + case "namespace": + child(graph[3], "metadata")["namespace"] = "private-mail" + case "identity": + child(graph[4], "metadata")["uid"] = child(graph[3], "metadata")["uid"] + case "resource-version": + delete(child(graph[3], "metadata"), "resourceVersion") + case "labels": + child(child(graph[3], "metadata"), "labels")[recoveryLabel] = "another-run" + case "foreign-target": + target.run = "12345678" + } + if _, err := captureRetainedReaderInputs(target, fixtureImage, reader, claim, source, graph); err == nil { + t.Fatal("unqualified input bundle could enter recovery custody") + } + }) + } +} diff --git a/scripts/verify-email-data-recovery/reader_volume_entry_test.go b/scripts/verify-email-data-recovery/reader_volume_entry_test.go new file mode 100644 index 0000000000..863d56de02 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_entry_test.go @@ -0,0 +1,230 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "reflect" + "strings" + "testing" + + jsonpatch "gopkg.in/evanphx/json-patch.v4" +) + +type readerVolumeAPI struct { + inner *cleanupAPI + name string + patches int + beforePatch, afterPatch func() + lostResponse bool +} + +func (a *readerVolumeAPI) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + if args[0] != "patch" { + if len(args) > 1 && args[1] == "Secret" { + return nil, errors.New("secret-read-forbidden") + } + return a.inner.call(ctx, input, args...) + } + a.patches++ + if !reflect.DeepEqual(args, []string{"patch", "PersistentVolume", a.name, "--type=json", "--patch-file=/dev/stdin", "-o=json"}) { + return nil, errors.New("patch-capability") + } + if a.beforePatch != nil { + a.beforePatch() + } + key := cleanupKey("PersistentVolume", "", a.name) + raw, err := json.Marshal(a.inner.objects[key]) + if err != nil { + return nil, err + } + patch, err := jsonpatch.DecodePatch(input) + if err != nil { + return nil, err + } + updated, err := patch.Apply(raw) + if err != nil { + return nil, err + } + var o object + if json.Unmarshal(updated, &o) != nil { + return nil, errors.New("patch-result") + } + child(o, "metadata")["resourceVersion"] = "501" + a.inner.objects[key] = o + if a.afterPatch != nil { + a.afterPatch() + } + if a.lostResponse { + return nil, errors.New("uncertain-patch-response") + } + return json.Marshal(a.inner.objects[key]) +} + +// readerVolumeDispatch supplies the separate confirmation and workflow identity for volume protection tests. +func readerVolumeDispatch() map[string]string { + env := readerCustodyDispatch() + delete(env, "EMAIL_READER_INPUTS_CONFIRM") + env["GITHUB_WORKFLOW_REF"] = "devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main" + env["EMAIL_RETAINED_VOLUME_CONFIRM"] = "protect-retained-email-volume" + return env +} + +// readerVolumeNativeFixture models a restored binding with independently witnessed reader, claim and volume identities. +func readerVolumeNativeFixture(t *testing.T) (*readerVolumeAPI, map[string]string) { + t.Helper() + a, _, _ := readerCustodyNativeFixture(t) + reader, claim, pv, _, witness := retainedVolumeFixture(t) + image, err := loadRecoveryImage() + if err != nil { + t.Fatal(err) + } + items(child(reader, "spec"), "containers")[0]["image"] = image + for _, o := range []object{reader, claim, pv} { + m := child(o, "metadata") + a.objects[cleanupKey(text(o, "kind"), text(m, "namespace"), text(m, "name"))] = o + } + env := readerVolumeDispatch() + env["EMAIL_RETAINED_READER_UID"], env["EMAIL_RETAINED_CLAIM_UID"], env["EMAIL_RETAINED_PV_UID"] = witness.ReaderUID, witness.ClaimUID, witness.PVUID + return &readerVolumeAPI{inner: a, name: text(child(pv, "metadata"), "name")}, env +} + +// TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent compares the complete graph across two invocations. +func TestRetainedVolumeProtectionChangesOnlyTheBoundReclaimPolicyAndIsIdempotent(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + before, err := json.Marshal(a.inner.objects) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + var out bytes.Buffer + checks := 0 + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { checks++; return nil }, &out) + if code != 0 || a.patches != 1 || checks < 2 || !strings.Contains(out.String(), "EMAIL_RETAINED_VOLUME=PROTECTED") || !strings.Contains(out.String(), "reader=RETAINED restart=NOT_AUTHORIZED") { + t.Fatalf("fixed metadata protection was not verified: code=%d patches=%d checks=%d output=%q", code, a.patches, checks, out.String()) + } + if strings.Contains(out.String(), env["EMAIL_RETAINED_PV_UID"]) || strings.Contains(out.String(), a.name) { + t.Fatal("private identity reached output") + } + } + var original map[string]object + if json.Unmarshal(before, &original) != nil { + t.Fatal("fixture") + } + key := cleanupKey("PersistentVolume", "", a.name) + child(original[key], "spec")["persistentVolumeReclaimPolicy"] = "Retain" + child(original[key], "metadata")["resourceVersion"] = "501" + if !reflect.DeepEqual(original, a.inner.objects) { + t.Fatal("protection changed data, readers, keys, configuration or another storage field") + } +} + +// TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess verifies every admission field before native reads or writes. +func TestRetainedVolumeProtectionRejectsForeignDispatchBeforeAccess(t *testing.T) { + for _, field := range []string{"GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "GITHUB_WORKFLOW_REF", "GITHUB_RUN_ID", "GITHUB_SHA", "EMAIL_RETAINED_VOLUME_CONFIRM", "EMAIL_RETAINED_READER_UID", "EMAIL_RETAINED_CLAIM_UID", "EMAIL_RETAINED_PV_UID"} { + t.Run(field, func(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + env[field] = "foreign" + calls := 0 + a.inner.onGet = func(string) { calls++ } + var out bytes.Buffer + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { calls++; return nil }, &out) + if code != 2 || calls != 0 || a.patches != 0 || !strings.Contains(out.String(), "stage=dispatch") { + t.Fatal("foreign dispatch reached native state or source execution") + } + }) + } +} + +// TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation models identity races, lost responses and changed readback. +func TestRetainedVolumeProtectionNeverRetriesOrCertifiesAnUncertainMutation(t *testing.T) { + for _, failure := range []string{"stale-version", "replacement", "lost-response", "changed-readback", "changed-before-write", "changed-after-write"} { + t.Run(failure, func(t *testing.T) { + a, env := readerVolumeNativeFixture(t) + key := cleanupKey("PersistentVolume", "", a.name) + switch failure { + case "stale-version": + a.beforePatch = func() { child(a.inner.objects[key], "metadata")["resourceVersion"] = "502" } + case "replacement": + a.beforePatch = func() { child(a.inner.objects[key], "metadata")["uid"] = "replaced" } + case "lost-response": + a.lostResponse = true + case "changed-readback": + a.afterPatch = func() { child(child(a.inner.objects[key], "spec"), "csi")["volumeHandle"] = "foreign" } + } + checks := 0 + var out bytes.Buffer + code := runRetainedVolumeProtection(context.Background(), func(k string) string { return env[k] }, a, func(context.Context, string) error { + checks++ + if failure == "changed-before-write" && checks == 2 { + child(a.inner.objects[cleanupKey("PersistentVolumeClaim", retainedRecovery.namespace, "data")], "metadata")["uid"] = "changed" + } + if failure == "changed-after-write" && checks == 3 { + return errors.New("source-changed") + } + return nil + }, &out) + wantPatches := 1 + if failure == "changed-before-write" { + wantPatches = 0 + } + if code != 2 || a.patches != wantPatches || strings.Contains(out.String(), "VOLUME=PROTECTED") { + t.Fatalf("uncertain protection was retried or certified: code=%d patches=%d output=%q", code, a.patches, out.String()) + } + }) + } +} + +// TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions refuses secret reads and any unqualified mutation payload. +func TestRetainedVolumeCapabilityCannotBorrowCustodyOrLifecyclePermissions(t *testing.T) { + for _, args := range [][]string{ + {"get", "Secret", "storage-key", "--ignore-not-found=true", "-o=json", "--namespace=" + retainedRecovery.namespace}, + {"delete", "Pod", "reader"}, {"create", "-f=-"}, + {"patch", "PersistentVolumeClaim", "data", "--type=json", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-foreign", "--type=json", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-bound", "--type=merge", "--patch-file=/dev/stdin", "-o=json"}, + {"patch", "PersistentVolume", "pvc-bound", "--type=json", "--patch-file=/dev/stdin", "-o=json", "--subresource=status"}, + } { + inner := &refusedAPI{} + native := &retainedVolumeKubernetes{inner: inner, name: "pvc-bound", patch: []byte("approved-patch")} + if _, err := native.call(context.Background(), []byte("approved-patch"), args...); err == nil || inner.calls != 0 { + t.Fatal("metadata protection borrowed secret, reader, claim or arbitrary patch authority") + } + } + for _, input := range [][]byte{nil, []byte("foreign-patch")} { + inner := &refusedAPI{} + native := &retainedVolumeKubernetes{inner: inner, name: "pvc-bound", patch: []byte("approved-patch")} + if _, err := native.call(context.Background(), input, "patch", "PersistentVolume", "pvc-bound", "--type=json", "--patch-file=/dev/stdin", "-o=json"); err == nil || inner.calls != 0 { + t.Fatal("unqualified patch payload reached Kubernetes") + } + } +} + +// TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch executes the CLI to distinguish admission from source verification. +func TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch(t *testing.T) { + if os.Getenv("EMAIL_RETAINED_VOLUME_CHILD") == "1" { + os.Args = []string{"verify-email-data-recovery", "--protect-retained-volume"} + main() + return + } + for _, confirmed := range []bool{false, true} { + command := exec.Command(os.Args[0], "-test.run=^TestRetainedVolumeCommandUsesItsSeparateProtectedDispatch$") + command.Env = []string{"EMAIL_RETAINED_VOLUME_CHILD=1", "PATH=/nonexistent", "GOCOVERDIR=" + t.TempDir()} + stage := "dispatch" + if confirmed { + for key, value := range readerVolumeDispatch() { + command.Env = append(command.Env, key+"="+value) + } + command.Env = append(command.Env, "EMAIL_RETAINED_READER_UID=11111111-1111-2222-3333-444444444444", "EMAIL_RETAINED_CLAIM_UID=22222222-1111-2222-3333-444444444444", "EMAIL_RETAINED_PV_UID=33333333-1111-2222-3333-444444444444") + stage = "source" + } + output, err := command.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 || !strings.Contains(string(output), "EMAIL_RETAINED_VOLUME=UNVERIFIED stage="+stage+" reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED") || strings.Contains(string(output), "EMAIL_DATA_RECOVERY=PASS") { + t.Fatalf("volume command lost its separate dispatch boundary: %q", output) + } + } +} diff --git a/scripts/verify-email-data-recovery/reader_volume_retention.go b/scripts/verify-email-data-recovery/reader_volume_retention.go new file mode 100644 index 0000000000..2194d09dad --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_retention.go @@ -0,0 +1,251 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +type retainedVolumeWitness struct{ ReaderUID, ClaimUID, PVUID string } + +// runRetainedVolumeProtection rebinds the restored graph, fences the Retain-only patch and verifies actual readback. +func runRetainedVolumeProtection(ctx context.Context, env func(string) string, k kubernetes, verifyMain func(context.Context, string) error, output io.Writer) int { + stage := "dispatch" + fail := func() int { + if output != nil { + _, _ = fmt.Fprintf(output, "EMAIL_RETAINED_VOLUME=UNVERIFIED stage=%s reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", stage) + } + return 2 + } + witness := retainedVolumeWitness{env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID"), env("EMAIL_RETAINED_PV_UID")} + if output == nil || verifyMain == nil || authorizeRetainedVolumeProtection(env, witness) != nil { + return fail() + } + if _, err := fmt.Fprintln(output, "EMAIL_RETAINED_VOLUME=STARTED source_writes=0 reader=UNVERIFIED restart=NOT_AUTHORIZED recovery=UNVERIFIED"); err != nil { + return 2 + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + stage = "source" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + native := &retainedVolumeKubernetes{inner: k} + d := &recoveryDriver{i: retainedRecovery, k: native} + var err error + d.source, err = d.readSource(ctx) + if err != nil { + return fail() + } + stage = "binding" + original, err := d.readRetainedVolume(ctx, witness) + if err != nil || verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + fresh, err := d.readRetainedVolume(ctx, witness) + if err != nil || !sameRetainedVolumeInputs(original, fresh) || d.sourceRetained(ctx) != nil { + return fail() + } + expected, err := cloneRetainedVolumeObjects(fresh) + if err != nil { + return fail() + } + child(expected[2], "spec")["persistentVolumeReclaimPolicy"] = "Retain" + state := "EXISTING" + if text(child(fresh[2], "spec"), "persistentVolumeReclaimPolicy") == "Delete" { + m := child(fresh[2], "metadata") + payload, err := json.Marshal([]object{ + {"op": "test", "path": "/metadata/uid", "value": witness.PVUID}, + {"op": "test", "path": "/metadata/name", "value": text(m, "name")}, + {"op": "test", "path": "/metadata/resourceVersion", "value": text(m, "resourceVersion")}, + {"op": "test", "path": "/spec", "value": child(fresh[2], "spec")}, + {"op": "replace", "path": "/spec/persistentVolumeReclaimPolicy", "value": "Retain"}, + }) + if err != nil { + return fail() + } + stage = "mutation" + native.name, native.patch = text(m, "name"), payload + _, err = native.call(ctx, payload, "patch", "PersistentVolume", native.name, "--type=json", "--patch-file=/dev/stdin", "-o=json") + native.patch = nil + // An uncertain response never retries or reclassifies a mutation as + // success. No rollback can turn an already-protected PV back to Delete. + if err != nil { + return fail() + } + state = "UPDATED" + } + stage = "readback" + if verifyMain(ctx, env("GITHUB_SHA")) != nil { + return fail() + } + after, err := d.readRetainedVolume(ctx, witness) + if err != nil || !sameRetainedVolumeInputs(expected, after) || d.sourceRetained(ctx) != nil { + return fail() + } + if _, err := fmt.Fprintf(output, "EMAIL_RETAINED_VOLUME=PROTECTED state=%s source_writes=0 reader=RETAINED restart=NOT_AUTHORIZED recovery=UNVERIFIED\n", state); err != nil { + return 2 + } + return 0 +} + +// authorizeRetainedVolumeProtection admits only the confirmed first-attempt main workflow with three distinct native witnesses. +func authorizeRetainedVolumeProtection(env func(string) string, witness retainedVolumeWitness) error { + for key, want := range map[string]string{ + "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", + "GITHUB_ACTOR": "devantler", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/protect-retained-email-volume.yaml@refs/heads/main", + "EMAIL_RETAINED_VOLUME_CONFIRM": "protect-retained-email-volume", + } { + if env(key) != want { + return errors.New("retained-volume-dispatch") + } + } + if !recoveryRunPattern.MatchString(env("GITHUB_RUN_ID")) || !recoveryRevisionPattern.MatchString(env("GITHUB_SHA")) || !validRetainedVolumeWitness(witness) { + return errors.New("retained-volume-dispatch") + } + return nil +} + +// This operation reads no Secret and cannot stop a reader or retire storage. +// Its only mutation is the exact JSON patch produced after final binding. +type retainedVolumeKubernetes struct { + inner kubernetes + name string + patch []byte +} + +func (k *retainedVolumeKubernetes) call(ctx context.Context, input []byte, args ...string) ([]byte, error) { + fail := func() ([]byte, error) { return nil, errors.New("retained-volume-capability") } + if k.inner == nil || len(args) < 2 { + return fail() + } + var raw []byte + var err error + if args[0] == "get" && (args[1] == "Pod" || args[1] == "PersistentVolumeClaim" || args[1] == "PersistentVolume") { + raw, err = (readerInputKubernetes{inner: k.inner}).call(ctx, input, args...) + } else if len(k.patch) != 0 && bytes.Equal(input, k.patch) && reflect.DeepEqual(args, []string{"patch", "PersistentVolume", k.name, "--type=json", "--patch-file=/dev/stdin", "-o=json"}) { + raw, err = k.inner.call(ctx, input, args...) + } else { + return fail() + } + if err != nil || len(raw) > 4<<20 || (len(bytes.TrimSpace(raw)) != 0 && strictjson.Validate(raw, nil) != nil) { + return fail() + } + return raw, nil +} + +// readRetainedVolume reads and qualifies the independently witnessed restored binding while excluding serving storage. +func (d *recoveryDriver) readRetainedVolume(ctx context.Context, witness retainedVolumeWitness) ([]object, error) { + fail := func() ([]object, error) { return nil, errors.New("retained-volume-read") } + reader, found, err := d.lookup(ctx, "Pod", "reader", retainedRecovery.namespace) + if err != nil || !found || text(child(reader, "metadata"), "uid") != witness.ReaderUID { + return fail() + } + claim, found, err := d.lookup(ctx, "PersistentVolumeClaim", "data", retainedRecovery.namespace) + if err != nil || !found || text(child(claim, "metadata"), "uid") != witness.ClaimUID { + return fail() + } + name := text(child(claim, "spec"), "volumeName") + if !kubernetesName.MatchString(name) || name == d.source.pv { + return fail() + } + volume, found, err := d.lookup(ctx, "PersistentVolume", name, "") + if err != nil || !found || qualifyRetainedVolume(reader, claim, volume, d.source, d.image, witness) != nil { + return fail() + } + return []object{reader, claim, volume}, nil +} + +// cloneRetainedVolumeObjects takes an independent JSON snapshot for comparison after protection. +func cloneRetainedVolumeObjects(objects []object) ([]object, error) { + raw, err := json.Marshal(objects) + var cloned []object + if err != nil || json.Unmarshal(raw, &cloned) != nil { + return nil, errors.New("retained-volume-snapshot") + } + return cloned, nil +} + +// sameRetainedVolumeInputs compares stable definitions and annotations while allowing API resource versions to advance. +func sameRetainedVolumeInputs(before, after []object) bool { + if len(before) != 3 || len(after) != 3 { + return false + } + for index := range before { + if !reflect.DeepEqual(retainedMutableFields(before[index]), retainedMutableFields(after[index])) || !reflect.DeepEqual(child(child(before[index], "metadata"), "annotations"), child(child(after[index], "metadata"), "annotations")) { + return false + } + } + return true +} + +// validRetainedVolumeWitness requires distinct well-formed reader, claim and volume identities. +func validRetainedVolumeWitness(witness retainedVolumeWitness) bool { + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + seen := map[string]bool{} + for _, uid := range []string{witness.ReaderUID, witness.ClaimUID, witness.PVUID} { + if !uuid.MatchString(uid) || seen[uid] { + return false + } + seen[uid] = true + } + return true +} + +// qualifyRetainedVolume verifies the ready restored reader and encrypted binding without assuming an original provisioning name. +func qualifyRetainedVolume(reader, claim, volume object, source sourceIdentity, image string, witness retainedVolumeWitness) error { + fail := func() error { return errors.New("retained-volume-binding") } + if !validRetainedVolumeWitness(witness) { + return fail() + } + name := text(child(claim, "spec"), "volumeName") + if source.podUID == "" || source.claimUID == "" || source.pv == "" || source.pvUID == "" || source.podUID == witness.ReaderUID || source.claimUID == witness.ClaimUID || source.pvUID == witness.PVUID || source.pv == name || !regexp.MustCompile(`^pvc-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`).MatchString(name) { + return fail() + } + for _, target := range []struct { + actual object + kind, name, namespace, uid string + }{{reader, "Pod", "reader", retainedRecovery.namespace, witness.ReaderUID}, {claim, "PersistentVolumeClaim", "data", retainedRecovery.namespace, witness.ClaimUID}, {volume, "PersistentVolume", name, "", witness.PVUID}} { + m := child(target.actual, "metadata") + if text(target.actual, "apiVersion") != "v1" || text(target.actual, "kind") != target.kind || text(m, "name") != target.name || text(m, "namespace") != target.namespace || text(m, "uid") != target.uid || !regexp.MustCompile(`^[1-9][0-9]{0,31}$`).MatchString(text(m, "resourceVersion")) || m["deletionTimestamp"] != nil { + return fail() + } + if owners := m["ownerReferences"]; owners != nil { + list, ok := owners.([]any) + if !ok || len(list) != 0 { + return fail() + } + } + if target.namespace != "" && text(child(m, "labels"), recoveryLabel) != retainedRecovery.run { + return fail() + } + } + if !admittedRecoveryObject(recoveryReader(retainedRecovery, image), reader) || child(reader, "spec")["terminationGracePeriodSeconds"] != float64(30) || text(child(reader, "status"), "phase") != "Running" { + return fail() + } + ready := false + for _, condition := range items(child(reader, "status"), "conditions") { + ready = ready || text(condition, "type") == "Ready" && text(condition, "status") == "True" + } + claimSpec := child(claim, "spec") + if !ready || text(child(claim, "status"), "phase") != "Bound" || text(claimSpec, "storageClassName") != retainedRecovery.namespace || !reflect.DeepEqual(claimSpec["accessModes"], []any{"ReadWriteOnce"}) || text(child(child(claimSpec, "resources"), "requests"), "storage") != "2Gi" { + return fail() + } + spec := child(volume, "spec") + ref, csi := child(spec, "claimRef"), child(spec, "csi") + key := child(csi, "nodeStageSecretRef") + policy := text(spec, "persistentVolumeReclaimPolicy") + if text(child(volume, "status"), "phase") != "Bound" || text(spec, "storageClassName") != retainedRecovery.namespace || text(ref, "apiVersion") != "v1" || text(ref, "kind") != "PersistentVolumeClaim" || text(ref, "name") != "data" || text(ref, "namespace") != retainedRecovery.namespace || text(ref, "uid") != witness.ClaimUID || text(csi, "driver") != "driver.longhorn.io" || text(csi, "volumeHandle") != name || text(child(csi, "volumeAttributes"), "encrypted") != "true" || text(key, "name") != "storage-key" || text(key, "namespace") != retainedRecovery.namespace || policy != "Delete" && policy != "Retain" { + return fail() + } + return nil +} diff --git a/scripts/verify-email-data-recovery/reader_volume_retention_test.go b/scripts/verify-email-data-recovery/reader_volume_retention_test.go new file mode 100644 index 0000000000..bf87500550 --- /dev/null +++ b/scripts/verify-email-data-recovery/reader_volume_retention_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/json" + "testing" +) + +// retainedVolumeFixture models a restored claim whose UID differs from the volume's original provisioning identity. +func retainedVolumeFixture(t *testing.T) (object, object, object, sourceIdentity, retainedVolumeWitness) { + t.Helper() + graph, readerUID, claimUID := readerInputFixture(t) + reader, claim := graph[7], graph[6] + name := "pvc-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + child(claim, "spec")["volumeName"] = name + witness := retainedVolumeWitness{readerUID, claimUID, "99999999-1111-2222-3333-444444444444"} + volume := object{ + "apiVersion": "v1", "kind": "PersistentVolume", + "metadata": object{"name": name, "uid": witness.PVUID, "resourceVersion": "500"}, + "spec": object{ + "claimRef": object{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "name": "data", "namespace": retainedRecovery.namespace, "uid": claimUID}, + "storageClassName": retainedRecovery.namespace, "persistentVolumeReclaimPolicy": "Delete", + "csi": object{"driver": "driver.longhorn.io", "volumeHandle": name, "volumeAttributes": object{"encrypted": "true"}, "nodeStageSecretRef": object{"name": "storage-key", "namespace": retainedRecovery.namespace}}, + }, + "status": object{"phase": "Bound"}, + } + source := sourceIdentity{podUID: "serving-pod", claimUID: "ffffffff-1111-2222-3333-444444444444", pv: "pvc-serving-volume", pvUID: "serving-pv"} + return reader, claim, volume, source, witness +} + +// TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt verifies both permitted policies without mutating the graph. +func TestRetainedVolumeQualificationAcceptsTheRestoredBindingWithoutChangingIt(t *testing.T) { + for _, policy := range []string{"Delete", "Retain"} { + t.Run(policy, func(t *testing.T) { + reader, claim, volume, source, witness := retainedVolumeFixture(t) + child(volume, "spec")["persistentVolumeReclaimPolicy"] = policy + before, _ := json.Marshal([]object{reader, claim, volume}) + if err := qualifyRetainedVolume(reader, claim, volume, source, fixtureImage, witness); err != nil { + t.Fatalf("the independently bound restored PV was refused: %v", err) + } + after, _ := json.Marshal([]object{reader, claim, volume}) + if string(before) != string(after) || text(child(volume, "metadata"), "name") == "pvc-"+witness.ClaimUID { + t.Fatal("qualification modified inputs or relied on the original-provisioning UID formula") + } + }) + } +} + +// TestRetainedVolumeQualificationRejectsChangedOrForeignBindings rejects drift in each independent identity and storage boundary. +func TestRetainedVolumeQualificationRejectsChangedOrForeignBindings(t *testing.T) { + for _, change := range []string{"reader-witness", "claim-witness", "pv-witness", "same-witness", "reader-uid", "claim-uid", "pv-uid", "reader-version", "pv-version", "pv-terminating", "claim-terminating", "reader-terminating", "reader-command", "reader-not-ready", "reader-owner", "source-claim", "source-volume", "source-pv-uid", "missing-source", "claim-namespace", "claim-unbound", "pv-name", "pv-kind", "pv-unbound", "claim-reference", "foreign-reference", "driver", "handle", "unencrypted", "key-reference", "foreign-key-namespace", "unknown-policy", "storage-class"} { + t.Run(change, func(t *testing.T) { + reader, claim, volume, source, witness := retainedVolumeFixture(t) + spec := child(volume, "spec") + switch change { + case "reader-witness": + witness.ReaderUID = "" + case "claim-witness": + witness.ClaimUID = "foreign" + case "pv-witness": + witness.PVUID = "" + case "same-witness": + witness.PVUID = witness.ClaimUID + case "reader-uid": + child(reader, "metadata")["uid"] = "replacement" + case "claim-uid": + child(claim, "metadata")["uid"] = "replacement" + case "pv-uid": + child(volume, "metadata")["uid"] = "replacement" + case "reader-version": + delete(child(reader, "metadata"), "resourceVersion") + case "pv-version": + child(volume, "metadata")["resourceVersion"] = "invalid" + case "pv-terminating": + child(volume, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "claim-terminating": + child(claim, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "reader-terminating": + child(reader, "metadata")["deletionTimestamp"] = "2026-11-11T00:00:00Z" + case "reader-command": + items(child(reader, "spec"), "containers")[0]["command"] = []any{"foreign"} + case "reader-not-ready": + reader["status"] = object{"phase": "Pending"} + case "reader-owner": + child(reader, "metadata")["ownerReferences"] = []any{object{"uid": "foreign"}} + case "source-claim": + source.claimUID = witness.ClaimUID + case "source-volume": + source.pv = text(child(volume, "metadata"), "name") + case "source-pv-uid": + source.pvUID = witness.PVUID + case "missing-source": + source = sourceIdentity{} + case "claim-namespace": + child(claim, "metadata")["namespace"] = "private-mail" + case "claim-unbound": + child(claim, "status")["phase"] = "Pending" + case "pv-name": + child(volume, "metadata")["name"] = "pvc-foreign" + case "pv-kind": + volume["kind"] = "Secret" + case "pv-unbound": + child(volume, "status")["phase"] = "Released" + case "claim-reference": + child(spec, "claimRef")["uid"] = source.claimUID + case "foreign-reference": + child(spec, "claimRef")["namespace"] = "private-mail" + case "driver": + child(spec, "csi")["driver"] = "foreign.invalid" + case "handle": + child(spec, "csi")["volumeHandle"] = source.pv + case "unencrypted": + child(child(spec, "csi"), "volumeAttributes")["encrypted"] = "false" + case "key-reference": + child(child(spec, "csi"), "nodeStageSecretRef")["name"] = "foreign" + case "foreign-key-namespace": + child(child(spec, "csi"), "nodeStageSecretRef")["namespace"] = "private-mail" + case "unknown-policy": + spec["persistentVolumeReclaimPolicy"] = "Recycle" + case "storage-class": + spec["storageClassName"] = "foreign" + } + if qualifyRetainedVolume(reader, claim, volume, source, fixtureImage, witness) == nil { + t.Fatal("changed or foreign restored binding was accepted") + } + }) + } +} diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 55cd26df7e..cfd8fef81d 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -11,7 +11,23 @@ import ( // The actual workflow guard must reject branch, actor and rerun changes before // checkout or credentials; metadata checks preserve the fixed disposal handoff. func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/dispose-retained-email-copy.yaml") + checkRetainedWorkflow(t, "dispose-retained-email-copy.yaml", "dispose", "dispose-retained-email-copy", "--dispose-retained", "EMAIL_RETAINED_COPY_CONFIRM", retainedDispatch()) +} + +// TestReaderInputCustodyWorkflowMainGuardAndCustody exercises the authored guard before credentials are restored. +func TestReaderInputCustodyWorkflowMainGuardAndCustody(t *testing.T) { + checkRetainedWorkflow(t, "prepare-retained-email-reader-inputs.yaml", "prepare", "prepare-retained-email-reader-inputs", "--prepare-reader-inputs", "EMAIL_READER_INPUTS_CONFIRM", readerCustodyDispatch()) +} + +// TestRetainedVolumeProtectionWorkflowMainGuardAndCustody checks the metadata-only workflow and independent volume witness. +func TestRetainedVolumeProtectionWorkflowMainGuardAndCustody(t *testing.T) { + checkRetainedWorkflow(t, "protect-retained-email-volume.yaml", "protect", "protect-retained-email-volume", "--protect-retained-volume", "EMAIL_RETAINED_VOLUME_CONFIRM", readerVolumeDispatch()) +} + +// checkRetainedWorkflow reads only filenames supplied by the fixed workflow tests and executes each authored admission guard. +func checkRetainedWorkflow(t *testing.T, filename, jobName, confirm, flag, confirmEnv string, env map[string]string) { + t.Helper() + data, err := os.ReadFile("../../.github/workflows/" + filename) if err != nil { t.Fatal(err) } @@ -36,14 +52,17 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { if yaml.Unmarshal(data, &w) != nil || len(w.On) != 1 || w.On["workflow_dispatch"] == nil || len(w.Permissions) != 0 || len(w.Jobs) != 1 || w.Concurrency.Group != "prod-deploy" || w.Concurrency.Cancel { t.Fatal("workflow widened its trigger, permission or concurrency boundary") } - job := w.Jobs["dispose"] + job := w.Jobs[jobName] if job.Environment != "prod" || job.Timeout > 25 || job.Timeout < 3 || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) != 6 { t.Fatal("observation custody changed") } guard := job.Steps[0] - env := retainedDispatch() - env["CONFIRM"] = "dispose-retained-email-copy" - for _, key := range append([]string{""}, "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "CONFIRM") { + env["CONFIRM"] = confirm + guardKeys := []string{"", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_WORKFLOW_REF", "GITHUB_ACTOR", "GITHUB_RUN_ATTEMPT", "RUNNER_ENVIRONMENT", "CONFIRM"} + if jobName != "dispose" { + guardKeys = append(guardKeys, "GITHUB_ACTIONS") + } + for _, key := range guardKeys { t.Run("guard/"+key, func(t *testing.T) { cmd := exec.Command("/bin/bash", "--noprofile", "--norc", "-c", guard.Run+"\nprintf DISPOSAL_ADMITTED\n") for k, v := range env { @@ -72,7 +91,10 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { } } final := job.Steps[5] - if final.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || final.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || final.Env["EMAIL_RETAINED_COPY_CONFIRM"] != "${{ inputs.confirm }}" || !strings.HasSuffix(strings.TrimSpace(final.Run), `"$RUNNER_TEMP/verify-email-data-recovery" --dispose-retained`) { + if jobName == "protect" && (final.Env["EMAIL_RETAINED_PV_UID"] != "${{ secrets.EMAIL_RETAINED_PV_UID }}" || strings.Contains(final.Run, "get secret") || strings.Contains(final.Run, "port-forward")) { + t.Fatal("metadata protection lost its independent volume witness or gained credential export") + } + if final.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || final.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || final.Env[confirmEnv] != "${{ inputs.confirm }}" || !strings.HasSuffix(strings.TrimSpace(final.Run), `"$RUNNER_TEMP/verify-email-data-recovery" `+flag) { t.Fatal("workflow invoked recovery writes or lost credential binding") } if final.Env["EMAIL_RETAINED_READER_UID"] != "${{ secrets.EMAIL_RETAINED_READER_UID }}" || final.Env["EMAIL_RETAINED_CLAIM_UID"] != "${{ secrets.EMAIL_RETAINED_CLAIM_UID }}" || !strings.Contains(final.Run, `./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA"`) {