From 559a0d304ccf0c7f7d54f68ec52f7a417653bf7d Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:28:04 +1000 Subject: [PATCH 1/2] Set iommu=pt on the amd variant DMA from the GPUs then skips IOMMU address translation, which AMD's ROCm docs recommend for multi-GPU hosts. The argument goes in /usr/lib/bootc/kargs.d/50-sibyl-amd.toml, as nvidia's kargs do. The cost is that a rogue PCIe or Thunderbolt device can read all memory. Co-Authored-By: Claude Opus 5.5 --- docs/design/overview.md | 2 +- os/Containerfile | 6 ++++-- os/variants/sibyl-amd-kargs.toml | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 os/variants/sibyl-amd-kargs.toml diff --git a/docs/design/overview.md b/docs/design/overview.md index 532aed1..b437c11 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -100,7 +100,7 @@ Caddy sends `/v1/*` to the model router. The router checks each client's key and - The container signature policy in `/etc/containers/policy.json`. - Host-only tools that cannot run in a container: GPU detection, the installer and disk management. -The AMD variant needs no extra kernel modules. `amdgpu` is in the Fedora kernel and is signed for Secure Boot. The host needs only the firmware (`amd-gpu-firmware`) and device access. The ROCm userspace is in the HIP image only. +The AMD variant needs no extra kernel modules. `amdgpu` is in the Fedora kernel and is signed for Secure Boot. The host needs only the firmware (`amd-gpu-firmware`) and device access. The ROCm userspace is in the HIP image only. The image sets the kernel argument `iommu=pt` in `/usr/lib/bootc/kargs.d/50-sibyl-amd.toml`. DMA from the GPUs then skips IOMMU address translation, which AMD's ROCm docs recommend for multi-GPU hosts. The cost is that a rogue PCIe or Thunderbolt device can read all memory. The Intel variant also uses in-kernel drivers (`i915` and `xe`) and only adds the firmware (`intel-gpu-firmware`). It adds `intel-compute-runtime` to the host for device nodes. Its inference image is the Vulkan one, with Mesa ANV. SYCL was named as a second choice and is not built. diff --git a/os/Containerfile b/os/Containerfile index 3427fc6..359505c 100644 --- a/os/Containerfile +++ b/os/Containerfile @@ -44,13 +44,15 @@ RUN grep -hEv '^\s*(#|$)' /tmp/packages/common.txt "/tmp/packages/${GPU_VARIANT} && dnf clean all \ && rm -rf /tmp/packages /run/dnf /var/cache/* /var/lib/dnf /var/log/dnf* -# Variant-specific build steps. Only nvidia has any today: it builds and -# signs the NVIDIA kernel modules (os/variants/nvidia.sh). +# Variant-specific build steps. nvidia builds and signs the NVIDIA kernel +# modules (os/variants/nvidia.sh). amd sets iommu=pt. RUN --mount=type=bind,from=variants,target=/variants \ --mount=type=secret,id=mok_key \ if [ "${GPU_VARIANT}" = nvidia ]; then \ bash /variants/nvidia.sh \ && install -D -m 0644 /variants/sibyl-nvidia-kargs.toml /usr/lib/bootc/kargs.d/50-sibyl-nvidia.toml; \ + elif [ "${GPU_VARIANT}" = amd ]; then \ + install -D -m 0644 /variants/sibyl-amd-kargs.toml /usr/lib/bootc/kargs.d/50-sibyl-amd.toml; \ fi COPY os/rootfs/ / diff --git a/os/variants/sibyl-amd-kargs.toml b/os/variants/sibyl-amd-kargs.toml new file mode 100644 index 0000000..9dfe680 --- /dev/null +++ b/os/variants/sibyl-amd-kargs.toml @@ -0,0 +1,6 @@ +# IOMMU passthrough for the devices that the host owns. DMA from the GPUs +# then skips address translation, which AMD's ROCm docs recommend for +# multi-GPU hosts. The IOMMU stays on for devices given to a VM. The cost: +# a rogue PCIe or Thunderbolt device can read all memory. +kargs = ["iommu=pt"] +match-architectures = ["x86_64"] From 5f0ac6f5699de03c29b8c4f3213e9683b82dacfa Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:32:45 +1000 Subject: [PATCH 2/2] Bump the fedora-bootc base image digest Co-Authored-By: Claude Opus 5.5 --- os/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os/Containerfile b/os/Containerfile index 359505c..bce3e9b 100644 --- a/os/Containerfile +++ b/os/Containerfile @@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/sibyl-hostd ./cmd/ FROM scratch AS variants COPY os/variants/ / -FROM quay.io/fedora/fedora-bootc:44@sha256:6bda175e89575aa3dfb6b0b4b8342978cb7de8d6109042f0338560eb3fe82c15 +FROM quay.io/fedora/fedora-bootc:44@sha256:06f956b9fe717679760a9fbcd669fc0f9b3f7731244a5734db7970fafa3f681d ARG GPU_VARIANT=amd ARG SIBYL_VERSION=dev