From 622f14fb9bd6de836c35e757dcaa7d9ba5768399 Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:59:07 +1000 Subject: [PATCH] Prune Sibyl images that no deployment uses Each update pulls a new tag of every Sibyl image, and Podman kept the old ones: 179 images and 42 GB reclaimable on the test host. image-prune keeps the tags in /usr/share/sibyl/image-tag of every bootc deployment, so a rollback still has its images, and removes every other ghcr.io/diffsec/sibyl-* tag, then untagged images that no container uses. Images of other stacks stay. sibyl-image-prune.timer runs it 15 minutes after boot and daily. Co-Authored-By: Claude Opus 5.5 --- docs/design/overview.md | 1 + docs/guides/updates.md | 6 + internal/hostctl/prune_test.go | 130 ++++++++++++++++++ os/Containerfile | 2 +- .../systemd/system/sibyl-image-prune.service | 11 ++ .../systemd/system/sibyl-image-prune.timer | 11 ++ os/rootfs/usr/libexec/sibyl/image-prune | 78 +++++++++++ 7 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 internal/hostctl/prune_test.go create mode 100644 os/rootfs/usr/lib/systemd/system/sibyl-image-prune.service create mode 100644 os/rootfs/usr/lib/systemd/system/sibyl-image-prune.timer create mode 100755 os/rootfs/usr/libexec/sibyl/image-prune diff --git a/docs/design/overview.md b/docs/design/overview.md index 273372d..532aed1 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -553,6 +553,7 @@ A bundle is a directory: - A Tailscale auth key goes from the System API to `sibyl-hostd` through `/run/sibyl-secret`, in memory. The request in the disk queue names only the key file. `sibyl-hostd` reads it without following a symlink, removes it, and gives `tailscale` the key in a file in its private `/tmp`, not on the command line. - SSH accepts keys only. Root login is off. - `bootc` and Podman auto-update are off by default. Updates are explicit, from the TUI, the web console, the System API, or `bootc upgrade`. +- `sibyl-image-prune.timer` removes each `ghcr.io/diffsec/sibyl-*` tag that no deployment lists in `/usr/share/sibyl/image-tag`, 15 minutes after boot and daily. The booted, rollback and staged deployments keep their images, so a rollback works offline. The script then runs `podman image prune`, which removes only untagged images that no container uses. - Every Quadlet container drops all capabilities, sets `NoNewPrivileges=true` and `ReadOnly=true`, and gets a tmpfs for `/tmp`. Writable state goes to named volumes only. - `sibyl-hostd` refuses to change or delete an account in `wheel`. Pool creation refuses a disk with active swap, as it refuses a mounted disk. diff --git a/docs/guides/updates.md b/docs/guides/updates.md index 88461cb..45935aa 100644 --- a/docs/guides/updates.md +++ b/docs/guides/updates.md @@ -84,6 +84,12 @@ sudo bootc rollback sudo systemctl reboot ``` +## Old images + +Each update downloads a new tag of every Sibyl service image. 15 minutes after each boot, and then daily, `sibyl-image-prune.timer` removes the Sibyl images that no deployment uses. It keeps the images of the running version, the rollback version and a staged version, so a rollback needs no download. It also removes untagged images that no container uses. It does not remove other images, such as those of the stacks that Arcane runs. + +To see what it removed: `journalctl -u sibyl-image-prune`. + ## Update offline A host with no internet updates from a bundle on a USB drive. A bundle is a signed directory that holds the host image and every service image for one version and one variant. diff --git a/internal/hostctl/prune_test.go b/internal/hostctl/prune_test.go new file mode 100644 index 0000000..faf1be7 --- /dev/null +++ b/internal/hostctl/prune_test.go @@ -0,0 +1,130 @@ +package hostctl + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" +) + +// fakePrunePodman is a podman that lists $FAKE_IMAGES, records each rmi and +// prune in $FAKE_LOG, and refuses to remove the refs in $FAKE_INUSE. +const fakePrunePodman = `#!/bin/bash +case "$1" in +images) cat "$FAKE_IMAGES" ;; +rmi) + if grep -qxF "$2" "$FAKE_INUSE"; then exit 2; fi + echo "rmi $2" >>"$FAKE_LOG" ;; +image) echo "prune" >>"$FAKE_LOG" ;; +*) exit 1 ;; +esac +` + +func runPrune(t *testing.T, tags, images, inUse []string, args ...string) (removed []string, pruned bool, out string) { + t.Helper() + dir := t.TempDir() + deploy := filepath.Join(dir, "ostree") + for i, tag := range tags { + d := filepath.Join(deploy, "default", "deploy", strings.Repeat("a", i+1)+".0", "usr", "share", "sibyl") + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "image-tag"), []byte(tag+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "podman"), []byte(fakePrunePodman), 0o755); err != nil { + t.Fatal(err) + } + write := func(name string, lines []string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return p + } + log := filepath.Join(dir, "log") + cmd := exec.Command("bash", append([]string{"../../os/rootfs/usr/libexec/sibyl/image-prune"}, args...)...) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "SIBYL_PRUNE_AS_SELF=1", + "SIBYL_PRUNE_DEPLOY="+deploy, "FAKE_IMAGES="+write("images", images), + "FAKE_INUSE="+write("inuse", inUse), "FAKE_LOG="+log) + b, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("image-prune: %v\n%s", err, b) + } + raw, _ := os.ReadFile(log) + for l := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { + switch { + case strings.HasPrefix(l, "rmi "): + removed = append(removed, strings.TrimPrefix(l, "rmi ")) + case l == "prune": + pruned = true + } + } + return removed, pruned, string(b) +} + +func TestImagePrune(t *testing.T) { + images := []string{ + "ghcr.io/diffsec/sibyl-caddy:sha-new", + "ghcr.io/diffsec/sibyl-llama-swap:sha-new-hip", + "ghcr.io/diffsec/sibyl-llama-swap:sha-new-vulkan", + "ghcr.io/diffsec/sibyl-caddy:sha-old", + "ghcr.io/diffsec/sibyl-llama-swap:sha-old-hip", + "ghcr.io/diffsec/sibyl-caddy:sha-dead", + "ghcr.io/diffsec/sibyl-llama-swap:sha-dead-hip", + "ghcr.io/diffsec/sibyl-router:sha-dead", + "ghcr.io/diffsec/sibyl-caddy:sha-newer", + "ghcr.io/diffsec/sibyl-caddy:", + "docker.io/library/busybox:1.37", + "ghcr.io/other/sibyl-caddy:sha-dead", + "localhost/sibyl-test:sha-dead", + } + removed, pruned, out := runPrune(t, []string{"sha-new", "sha-old"}, images, []string{"ghcr.io/diffsec/sibyl-router:sha-dead"}) + want := []string{ + "ghcr.io/diffsec/sibyl-caddy:sha-dead", + "ghcr.io/diffsec/sibyl-llama-swap:sha-dead-hip", + "ghcr.io/diffsec/sibyl-caddy:sha-newer", + } + if !slices.Equal(removed, want) { + t.Errorf("removed %q, want %q\n%s", removed, want, out) + } + if !pruned { + t.Errorf("no image prune after the removals\n%s", out) + } + if !strings.Contains(out, "kept ghcr.io/diffsec/sibyl-router:sha-dead: a container uses it") { + t.Errorf("the image in use is not reported:\n%s", out) + } +} + +func TestImagePruneNoDeployments(t *testing.T) { + removed, pruned, out := runPrune(t, nil, []string{"ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil) + if len(removed) > 0 || pruned { + t.Errorf("removed %q, pruned %v with no deployment tag\n%s", removed, pruned, out) + } +} + +func TestImagePruneBadTag(t *testing.T) { + // A tag file with a glob must not keep every image or match by pattern. + removed, _, out := runPrune(t, []string{"*"}, []string{"ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil) + if len(removed) > 0 || !strings.Contains(out, "removing nothing") { + t.Errorf("removed %q with only a bad tag\n%s", removed, out) + } +} + +func TestImagePruneDryRun(t *testing.T) { + removed, pruned, out := runPrune(t, []string{"sha-new"}, + []string{"ghcr.io/diffsec/sibyl-caddy:sha-new", "ghcr.io/diffsec/sibyl-caddy:sha-dead"}, nil, "--dry-run") + if len(removed) > 0 || pruned { + t.Errorf("the dry run removed %q, pruned %v", removed, pruned) + } + if !strings.Contains(out, "would remove ghcr.io/diffsec/sibyl-caddy:sha-dead") || strings.Contains(out, "sibyl-caddy:sha-new\n") { + t.Errorf("the dry run does not list the right image:\n%s", out) + } +} diff --git a/os/Containerfile b/os/Containerfile index 730f75a..3427fc6 100644 --- a/os/Containerfile +++ b/os/Containerfile @@ -71,7 +71,7 @@ RUN echo "${SIBYL_VERSION}" > /usr/share/sibyl/version \ && systemctl enable nftables.service sibyl-selinux.service sibyl-firstboot.service sibyl-operator-export.service \ sibyl-hostd.path sibyl-hostd-auth.path sibyl-hostd-state.timer sibyl-tls.timer sibyl-provision.service \ sibyl-diag-boot.service sibyl-diag-watchdog.service sibyl-diag-metrics.service sibyl-diag-upload.timer \ - sibyl-auto-update.timer \ + sibyl-auto-update.timer sibyl-image-prune.timer \ && systemctl --global enable podman.socket podman-restart.service sibyl-docker-shim.socket sibyl-agent.path sibyl-status.timer \ && systemctl disable bootc-fetch-apply-updates.timer \ && mkdir -p /etc/containers/systemd/users/1001/llama-swap.container.d \ diff --git a/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.service b/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.service new file mode 100644 index 0000000..3e98c19 --- /dev/null +++ b/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.service @@ -0,0 +1,11 @@ +[Unit] +Description=Remove Sibyl images that no deployment uses +Documentation=file:///usr/libexec/sibyl/image-prune +After=user@1001.service +Wants=user@1001.service + +[Service] +Type=oneshot +ExecStart=/usr/libexec/sibyl/image-prune +Nice=19 +IOSchedulingClass=idle diff --git a/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.timer b/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.timer new file mode 100644 index 0000000..e4f76e9 --- /dev/null +++ b/os/rootfs/usr/lib/systemd/system/sibyl-image-prune.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Remove old Sibyl images after boot, and daily + +[Timer] +# An update changes the deployments only at a reboot. The daily run covers +# a removal that failed because a container still used the image. +OnBootSec=15min +OnUnitActiveSec=1d + +[Install] +WantedBy=timers.target diff --git a/os/rootfs/usr/libexec/sibyl/image-prune b/os/rootfs/usr/libexec/sibyl/image-prune new file mode 100755 index 0000000..ae1bfaf --- /dev/null +++ b/os/rootfs/usr/libexec/sibyl/image-prune @@ -0,0 +1,78 @@ +#!/bin/bash +# Remove the Sibyl images that no bootc deployment uses. +# +# Each update pulls a new tag of every Sibyl image, and Podman keeps the old +# tags. This script keeps the tags of the booted, rollback and staged +# deployments, so a rollback still has its images without a download. It +# removes every other ghcr.io/diffsec/sibyl-* tag, then the untagged images +# that no container uses. It does not touch other images, such as those of +# the stacks that Arcane runs. Podman refuses to remove an image that a +# container uses, so such an image stays. +# +# sibyl-image-prune.timer runs this 15 minutes after boot and then daily. +# With --dry-run, it lists what it would remove and changes nothing. +set -euo pipefail + +DRY= +if [[ "${1:-}" == --dry-run ]]; then + DRY=1 +fi + +DEPLOY=${SIBYL_PRUNE_DEPLOY:-/ostree/deploy} +REPO=ghcr.io/diffsec/sibyl- + +log() { echo "sibyl-image-prune: $*"; } + +# as_sibyl runs Podman as the sibyl user, as auto-update does. +as_sibyl() { + if [[ -n "${SIBYL_PRUNE_AS_SELF:-}" ]]; then + "$@" + return + fi + setpriv --reuid=1001 --regid=1001 --init-groups -- env HOME="$(getent passwd sibyl | cut -d: -f6)" \ + XDG_RUNTIME_DIR=/run/user/1001 "$@" +} + +keep=() +shopt -s nullglob +for f in "$DEPLOY"/*/deploy/*/usr/share/sibyl/image-tag; do + t=$(<"$f") + if [[ "$t" =~ ^[A-Za-z0-9_.-]+$ && " ${keep[*]} " != *" $t "* ]]; then + keep+=("$t") + fi +done +if ((${#keep[@]} == 0)); then + log "found no deployment's image tag, so removing nothing" + exit 0 +fi +log "keeping the tags of the deployments: ${keep[*]}" + +refs=$(as_sibyl podman images --format '{{.Repository}}:{{.Tag}}') +removed=0 +while read -r ref; do + [[ "$ref" == "$REPO"*:* ]] || continue + tag=${ref##*:} + [[ "$tag" != "" ]] || continue + for k in "${keep[@]}"; do + # A tag of the deployment, or one of its backend variants (-hip). + if [[ "$tag" == "$k" || "$tag" == "$k"-* ]]; then + continue 2 + fi + done + if [[ -n "$DRY" ]]; then + log "would remove $ref" + removed=$((removed + 1)) + elif as_sibyl podman rmi "$ref" >/dev/null 2>&1; then + log "removed $ref" + removed=$((removed + 1)) + else + log "kept $ref: a container uses it" + fi +done <<<"$refs" + +if [[ -n "$DRY" ]]; then + log "would remove $removed old Sibyl image tag(s), then the untagged images" + exit 0 +fi +as_sibyl podman image prune -f >/dev/null +log "removed $removed old Sibyl image tag(s)"