Set iommu=pt on the amd variant

docs/design/overview.md

@@ -100,7 +100,7 @@ Caddy sends `/v1/*` to the model router. The router checks each client's key and
 - The container signature policy in `/etc/containers/policy.json`.
 - Host-only tools that cannot run in a container: GPU detection, the installer and disk management.
 
-The AMD variant needs no extra kernel modules. `amdgpu` is in the Fedora kernel and is signed for Secure Boot. The host needs only the firmware (`amd-gpu-firmware`) and device access. The ROCm userspace is in the HIP image only.
+The AMD variant needs no extra kernel modules. `amdgpu` is in the Fedora kernel and is signed for Secure Boot. The host needs only the firmware (`amd-gpu-firmware`) and device access. The ROCm userspace is in the HIP image only. The image sets the kernel argument `iommu=pt` in `/usr/lib/bootc/kargs.d/50-sibyl-amd.toml`. DMA from the GPUs then skips IOMMU address translation, which AMD's ROCm docs recommend for multi-GPU hosts. The cost is that a rogue PCIe or Thunderbolt device can read all memory.
 
 The Intel variant also uses in-kernel drivers (`i915` and `xe`) and only adds the firmware (`intel-gpu-firmware`). It adds `intel-compute-runtime` to the host for device nodes. Its inference image is the Vulkan one, with Mesa ANV. SYCL was named as a second choice and is not built.
 

os/Containerfile

@@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/sibyl-hostd ./cmd/
 FROM scratch AS variants
 COPY os/variants/ /
 
-FROM quay.io/fedora/fedora-bootc:44@sha256:6bda175e89575aa3dfb6b0b4b8342978cb7de8d6109042f0338560eb3fe82c15
+FROM quay.io/fedora/fedora-bootc:44@sha256:06f956b9fe717679760a9fbcd669fc0f9b3f7731244a5734db7970fafa3f681d
 
 ARG GPU_VARIANT=amd
 ARG SIBYL_VERSION=dev
@@ -44,13 +44,15 @@ RUN grep -hEv '^\s*(#|$)' /tmp/packages/common.txt "/tmp/packages/${GPU_VARIANT}
     && dnf clean all \
     && rm -rf /tmp/packages /run/dnf /var/cache/* /var/lib/dnf /var/log/dnf*
 
-# Variant-specific build steps. Only nvidia has any today: it builds and
-# signs the NVIDIA kernel modules (os/variants/nvidia.sh).
+# Variant-specific build steps. nvidia builds and signs the NVIDIA kernel
+# modules (os/variants/nvidia.sh). amd sets iommu=pt.
 RUN --mount=type=bind,from=variants,target=/variants \
     --mount=type=secret,id=mok_key \
     if [ "${GPU_VARIANT}" = nvidia ]; then \
         bash /variants/nvidia.sh \
         && install -D -m 0644 /variants/sibyl-nvidia-kargs.toml /usr/lib/bootc/kargs.d/50-sibyl-nvidia.toml; \
+    elif [ "${GPU_VARIANT}" = amd ]; then \
+        install -D -m 0644 /variants/sibyl-amd-kargs.toml /usr/lib/bootc/kargs.d/50-sibyl-amd.toml; \
     fi
 
 COPY os/rootfs/ /

os/variants/sibyl-amd-kargs.toml

@@ -0,0 +1,6 @@
+# IOMMU passthrough for the devices that the host owns. DMA from the GPUs
+# then skips address translation, which AMD's ROCm docs recommend for
+# multi-GPU hosts. The IOMMU stays on for devices given to a VM. The cost:
+# a rogue PCIe or Thunderbolt device can read all memory.
+kargs = ["iommu=pt"]
+match-architectures = ["x86_64"]