Prune Sibyl images that no deployment uses

docs/design/overview.md

@@ -553,6 +553,7 @@ A bundle is a directory:
 - A Tailscale auth key goes from the System API to `sibyl-hostd` through `/run/sibyl-secret`, in memory. The request in the disk queue names only the key file. `sibyl-hostd` reads it without following a symlink, removes it, and gives `tailscale` the key in a file in its private `/tmp`, not on the command line.
 - SSH accepts keys only. Root login is off.
 - `bootc` and Podman auto-update are off by default. Updates are explicit, from the TUI, the web console, the System API, or `bootc upgrade`.
+- `sibyl-image-prune.timer` removes each `ghcr.io/diffsec/sibyl-*` tag that no deployment lists in `/usr/share/sibyl/image-tag`, 15 minutes after boot and daily. The booted, rollback and staged deployments keep their images, so a rollback works offline. The script then runs `podman image prune`, which removes only untagged images that no container uses.
 - Every Quadlet container drops all capabilities, sets `NoNewPrivileges=true` and `ReadOnly=true`, and gets a tmpfs for `/tmp`. Writable state goes to named volumes only.
 - `sibyl-hostd` refuses to change or delete an account in `wheel`. Pool creation refuses a disk with active swap, as it refuses a mounted disk.
 

docs/guides/updates.md

@@ -84,6 +84,12 @@ sudo bootc rollback
 sudo systemctl reboot
 ```
 
+## Old images
+
+Each update downloads a new tag of every Sibyl service image. 15 minutes after each boot, and then daily, `sibyl-image-prune.timer` removes the Sibyl images that no deployment uses. It keeps the images of the running version, the rollback version and a staged version, so a rollback needs no download. It also removes untagged images that no container uses. It does not remove other images, such as those of the stacks that Arcane runs.
+
+To see what it removed: `journalctl -u sibyl-image-prune`.
+
 ## Update offline
 
 A host with no internet updates from a bundle on a USB drive. A bundle is a signed directory that holds the host image and every service image for one version and one variant.

internal/hostctl/prune_test.go

@@ -0,0 +1,130 @@
+package hostctl
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// fakePrunePodman is a podman that lists $FAKE_IMAGES, records each rmi and
+// prune in $FAKE_LOG, and refuses to remove the refs in $FAKE_INUSE.
+const fakePrunePodman = `#!/bin/bash
+case "$1" in
+images) cat "$FAKE_IMAGES" ;;
+rmi)
+    if grep -qxF "$2" "$FAKE_INUSE"; then exit 2; fi
+    echo "rmi $2" >>"$FAKE_LOG" ;;
+image) echo "prune" >>"$FAKE_LOG" ;;
+*) exit 1 ;;
+esac
+`
+
+func runPrune(t *testing.T, tags, images, inUse []string, args ...string) (removed []string, pruned bool, out string) {
+	t.Helper()
+	dir := t.TempDir()
+	deploy := filepath.Join(dir, "ostree")
+	for i, tag := range tags {
+		d := filepath.Join(deploy, "default", "deploy", strings.Repeat("a", i+1)+".0", "usr", "share", "sibyl")
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(d, "image-tag"), []byte(tag+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	bin := filepath.Join(dir, "bin")
+	if err := os.MkdirAll(bin, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(bin, "podman"), []byte(fakePrunePodman), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	write := func(name string, lines []string) string {
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		return p
+	}
+	log := filepath.Join(dir, "log")
+	cmd := exec.Command("bash", append([]string{"../../os/rootfs/usr/libexec/sibyl/image-prune"}, args...)...)
+	cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "SIBYL_PRUNE_AS_SELF=1",
+		"SIBYL_PRUNE_DEPLOY="+deploy, "FAKE_IMAGES="+write("images", images),
+		"FAKE_INUSE="+write("inuse", inUse), "FAKE_LOG="+log)
+	b, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("image-prune: %v\n%s", err, b)
+	}
+	raw, _ := os.ReadFile(log)
+	for l := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
+		switch {
+		case strings.HasPrefix(l, "rmi "):
+			removed = append(removed, strings.TrimPrefix(l, "rmi "))
+		case l == "prune":
+			pruned = true
+		}
+	}
+	return removed, pruned, string(b)
+}
+
+func TestImagePrune(t *testing.T) {
+	images := []string{
+		"ghcr.io/diffsec/sibyl-caddy:sha-new",
+		"ghcr.io/diffsec/sibyl-llama-swap:sha-new-hip",
+		"ghcr.io/diffsec/sibyl-llama-swap:sha-new-vulkan",
+		"ghcr.io/diffsec/sibyl-caddy:sha-old",
+		"ghcr.io/diffsec/sibyl-llama-swap:sha-old-hip",
+		"ghcr.io/diffsec/sibyl-caddy:sha-dead",
+		"ghcr.io/diffsec/sibyl-llama-swap:sha-dead-hip",
+		"ghcr.io/diffsec/sibyl-router:sha-dead",
+		"ghcr.io/diffsec/sibyl-caddy:sha-newer",
+		"ghcr.io/diffsec/sibyl-caddy:<none>",
+		"docker.io/library/busybox:1.37",
+		"ghcr.io/other/sibyl-caddy:sha-dead",
+		"localhost/sibyl-test:sha-dead",
+	}
+	removed, pruned, out := runPrune(t, []string{"sha-new", "sha-old"}, images, []string{"ghcr.io/diffsec/sibyl-router:sha-dead"})
+	want := []string{
+		"ghcr.io/diffsec/sibyl-caddy:sha-dead",
+		"ghcr.io/diffsec/sibyl-llama-swap:sha-dead-hip",
+		"ghcr.io/diffsec/sibyl-caddy:sha-newer",
+	}
+	if !slices.Equal(removed, want) {
+		t.Errorf("removed %q, want %q\n%s", removed, want, out)
+	}
+	if !pruned {
+		t.Errorf("no image prune after the removals\n%s", out)
+	}
+	if !strings.Contains(out, "kept ghcr.io/diffsec/sibyl-router:sha-dead: a container uses it") {
+		t.Errorf("the image in use is not reported:\n%s", out)
+	}
+}
+
+func TestImagePruneNoDeployments(t *testing.T) {
+	removed, pruned, out := runPrune(t, nil, []string{"ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil)
+	if len(removed) > 0 || pruned {
+		t.Errorf("removed %q, pruned %v with no deployment tag\n%s", removed, pruned, out)
+	}
+}
+
+func TestImagePruneBadTag(t *testing.T) {
+	// A tag file with a glob must not keep every image or match by pattern.
+	removed, _, out := runPrune(t, []string{"*"}, []string{"ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil)
+	if len(removed) > 0 || !strings.Contains(out, "removing nothing") {
+		t.Errorf("removed %q with only a bad tag\n%s", removed, out)
+	}
+}
+
+func TestImagePruneDryRun(t *testing.T) {
+	removed, pruned, out := runPrune(t, []string{"sha-new"},
+		[]string{"ghcr.io/diffsec/sibyl-caddy:sha-new", "ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil, "--dry-run")
+	if len(removed) > 0 || pruned {
+		t.Errorf("the dry run removed %q, pruned %v", removed, pruned)
+	}
+	if !strings.Contains(out, "would remove ghcr.io/diffsec/sibyl-caddy:sha-dead") || strings.Contains(out, "sibyl-caddy:sha-new\n") {
+		t.Errorf("the dry run does not list the right image:\n%s", out)
+	}
+}

os/Containerfile

@@ -71,7 +71,7 @@ RUN echo "${SIBYL_VERSION}" > /usr/share/sibyl/version \
     && systemctl enable nftables.service sibyl-selinux.service sibyl-firstboot.service sibyl-operator-export.service \
         sibyl-hostd.path sibyl-hostd-auth.path sibyl-hostd-state.timer sibyl-tls.timer sibyl-provision.service \
         sibyl-diag-boot.service sibyl-diag-watchdog.service sibyl-diag-metrics.service sibyl-diag-upload.timer \
-        sibyl-auto-update.timer \
+        sibyl-auto-update.timer sibyl-image-prune.timer \
     && systemctl --global enable podman.socket podman-restart.service sibyl-docker-shim.socket sibyl-agent.path sibyl-status.timer \
     && systemctl disable bootc-fetch-apply-updates.timer \
     && mkdir -p /etc/containers/systemd/users/1001/llama-swap.container.d \

os/rootfs/usr/lib/systemd/system/sibyl-image-prune.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Remove Sibyl images that no deployment uses
+Documentation=file:///usr/libexec/sibyl/image-prune
+After=user@1001.service
+Wants=user@1001.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/sibyl/image-prune
+Nice=19
+IOSchedulingClass=idle

os/rootfs/usr/lib/systemd/system/sibyl-image-prune.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Remove old Sibyl images after boot, and daily
+
+[Timer]
+# An update changes the deployments only at a reboot. The daily run covers
+# a removal that failed because a container still used the image.
+OnBootSec=15min
+OnUnitActiveSec=1d
+
+[Install]
+WantedBy=timers.target

os/rootfs/usr/libexec/sibyl/image-prune

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Remove the Sibyl images that no bootc deployment uses.
+#
+# Each update pulls a new tag of every Sibyl image, and Podman keeps the old
+# tags. This script keeps the tags of the booted, rollback and staged
+# deployments, so a rollback still has its images without a download. It
+# removes every other ghcr.io/diffsec/sibyl-* tag, then the untagged images
+# that no container uses. It does not touch other images, such as those of
+# the stacks that Arcane runs. Podman refuses to remove an image that a
+# container uses, so such an image stays.
+#
+# sibyl-image-prune.timer runs this 15 minutes after boot and then daily.
+# With --dry-run, it lists what it would remove and changes nothing.
+set -euo pipefail
+
+DRY=
+if [[ "${1:-}" == --dry-run ]]; then
+    DRY=1
+fi
+
+DEPLOY=${SIBYL_PRUNE_DEPLOY:-/ostree/deploy}
+REPO=ghcr.io/diffsec/sibyl-
+
+log() { echo "sibyl-image-prune: $*"; }
+
+# as_sibyl runs Podman as the sibyl user, as auto-update does.
+as_sibyl() {
+    if [[ -n "${SIBYL_PRUNE_AS_SELF:-}" ]]; then
+        "$@"
+        return
+    fi
+    setpriv --reuid=1001 --regid=1001 --init-groups -- env HOME="$(getent passwd sibyl | cut -d: -f6)" \
+        XDG_RUNTIME_DIR=/run/user/1001 "$@"
+}
+
+keep=()
+shopt -s nullglob
+for f in "$DEPLOY"/*/deploy/*/usr/share/sibyl/image-tag; do
+    t=$(<"$f")
+    if [[ "$t" =~ ^[A-Za-z0-9_.-]+$ && " ${keep[*]} " != *" $t "* ]]; then
+        keep+=("$t")
+    fi
+done
+if ((${#keep[@]} == 0)); then
+    log "found no deployment's image tag, so removing nothing"
+    exit 0
+fi
+log "keeping the tags of the deployments: ${keep[*]}"
+
+refs=$(as_sibyl podman images --format '{{.Repository}}:{{.Tag}}')
+removed=0
+while read -r ref; do
+    [[ "$ref" == "$REPO"*:* ]] || continue
+    tag=${ref##*:}
+    [[ "$tag" != "<none>" ]] || continue
+    for k in "${keep[@]}"; do
+        # A tag of the deployment, or one of its backend variants (-hip).
+        if [[ "$tag" == "$k" || "$tag" == "$k"-* ]]; then
+            continue 2
+        fi
+    done
+    if [[ -n "$DRY" ]]; then
+        log "would remove $ref"
+        removed=$((removed + 1))
+    elif as_sibyl podman rmi "$ref" >/dev/null 2>&1; then
+        log "removed $ref"
+        removed=$((removed + 1))
+    else
+        log "kept $ref: a container uses it"
+    fi
+done <<<"$refs"
+
+if [[ -n "$DRY" ]]; then
+    log "would remove $removed old Sibyl image tag(s), then the untagged images"
+    exit 0
+fi
+as_sibyl podman image prune -f >/dev/null
+log "removed $removed old Sibyl image tag(s)"