Add native EROFS OCI full-bundle producer

README.md

@@ -65,3 +65,48 @@ or
 ```
 nix build .#custom-bundle-squashfs
 ```
+
+To build the full bundle (including historical modules) as a native EROFS OCI image:
+
+```
+nix build .#bundle-oci
+```
+
+The result is an OCI image layout directory with the reference `bundle`, suitable
+for copying with an OCI-layout-aware tool such as
+`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`.
+The output also provides a copy helper that keeps the exact image in its Nix
+closure and preserves the manifest digest:
+
+```
+nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest
+```
+
+The destination can use any Skopeo-supported transport, including `oci:`.
+Additional arguments are passed to `skopeo copy`. The helper trusts the locally
+built source (`--insecure-policy`); registry TLS verification remains enabled.
+For a clean Git source, the manifest records the full flake revision, source URL
+`https://github.com/replit/nixmodules`, and output name `bundle-oci` in OCI
+annotations. The manifest creation annotation and config `created` field use
+the flake's last-modified timestamp in UTC, never the build's wall clock.
+Local sources without a clean revision omit the revision annotation; sources
+without a last-modified timestamp omit creation metadata. Publish from a
+revision-pinned Git flake when complete provenance is required.
+
+It contains one uncompressed `application/vnd.oci.image.layer.v1.erofs` layer,
+not a tar layer. The consumer must support native EROFS layers; ordinary
+tar-only image unpackers cannot use it. Its diffID equals the layer blob digest.
+
+The filesystem is generated directly from the same pinned full bundle used by
+`disk-script`: its complete store closure and `/etc/nixmodules` metadata, with
+unchanged store paths, file contents, modes and symlink targets. All guest
+UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and
+per-module OCI outputs are unchanged. There is no disk conversion or dependency
+update. Uncompressed EROFS does not by itself guarantee DAX sharing; that also
+depends on the consumer's backing storage and mount configuration.
+
+A small fixture exercises the producer without building the full bundle:
+
+```
+nix build .#bundle-oci-check
+```

pkgs/bundle-oci/check.nix

@@ -0,0 +1,48 @@
+{ pkgs }:
+
+let
+  dependency = pkgs.runCommand "bundle-oci-dependency" { } ''
+    mkdir -p "$out"
+    printf 'dependency\n' > "$out/data"
+    ln "$out/data" "$out/hardlink"
+  '';
+  module = pkgs.runCommand "bundle-oci-module" { } ''
+    mkdir -p "$out/bin"
+    printf '%s\n' '${dependency}' > "$out/module.json"
+    printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool"
+    chmod 755 "$out/bin/tool"
+    ln -s ../module.json "$out/bin/relative"
+    ln -s '${dependency}/data' "$out/absolute"
+  '';
+  bundle = (pkgs.callPackage ../bundle {
+    self = { modules.fixture = module; };
+  }) { };
+  image = pkgs.callPackage ./. {
+    inherit bundle;
+    revision = "0123456789abcdef0123456789abcdef01234567";
+    sourceTimestamp = 1700000000;
+  };
+  closure = pkgs.closureInfo { rootPaths = [ bundle ]; };
+in
+pkgs.runCommand "bundle-oci-check"
+{
+  nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ];
+}
+  ''
+    python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py}
+    ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \
+      --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \
+      --digestfile "$PWD/copied.digest"
+    python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY'
+    import json, pathlib, sys
+    original, copied, digest_file = map(pathlib.Path, sys.argv[1:])
+    descriptor = json.loads((original / "index.json").read_text())["manifests"][0]
+    copied_descriptor = json.loads((copied / "index.json").read_text())["manifests"][0]
+    assert copied_descriptor["digest"] == descriptor["digest"]
+    assert digest_file.read_text().strip() == descriptor["digest"]
+    for blob in (original / "blobs/sha256").iterdir():
+        assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes()
+    print("copyTo preserved manifest digest and every OCI blob")
+    PY
+    touch "$out"
+  ''

pkgs/bundle-oci/check.py

@@ -0,0 +1,109 @@
+import hashlib
+import importlib.util
+import json
+import os
+import pathlib
+import re
+import stat
+import shutil
+import subprocess
+import sys
+
+image, bundle, closure, dependency, layout_script = map(pathlib.Path, sys.argv[1:])
+assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"}
+
+
+def blob(descriptor):
+    path = image / "blobs" / "sha256" / descriptor["digest"].removeprefix("sha256:")
+    assert path.stat().st_size == descriptor["size"]
+    assert "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() == descriptor["digest"]
+    return path
+
+
+index = json.loads((image / "index.json").read_text())
+assert index["schemaVersion"] == 2 and len(index["manifests"]) == 1
+manifest = json.loads(blob(index["manifests"][0]).read_text())
+assert manifest["schemaVersion"] == 2
+assert manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json"
+assert manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json"
+config = json.loads(blob(manifest["config"]).read_text())
+assert config["architecture"] == "amd64" and config["os"] == "linux"
+assert config["created"] == "2023-11-14T22:13:20Z"
+assert manifest["annotations"] == {
+    "org.opencontainers.image.source": "https://github.com/replit/nixmodules",
+    "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567",
+    "org.opencontainers.image.created": config["created"],
+    "dev.replit.nixmodules.flake-output": "bundle-oci",
+}
+assert len(manifest["layers"]) == 1
+layer = manifest["layers"][0]
+assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs"
+assert config["rootfs"] == {"type": "layers", "diff_ids": [layer["digest"]]}
+erofs = blob(layer)
+spec = importlib.util.spec_from_file_location("layout", layout_script)
+layout = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(layout)
+repeated = pathlib.Path("repeated")
+(repeated / "blobs/sha256").mkdir(parents=True)
+shutil.copyfile(erofs, "repeated.erofs")
+layout.write_layout(pathlib.Path("repeated.erofs"), repeated, "x86_64", {
+    "revision": "0123456789abcdef0123456789abcdef01234567",
+    "sourceTimestamp": 1700000000,
+})
+for path in image.rglob("*"):
+    if path.is_file():
+        assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes()
+local = pathlib.Path("local")
+(local / "blobs/sha256").mkdir(parents=True)
+shutil.copyfile(erofs, "local.erofs")
+layout.write_layout(pathlib.Path("local.erofs"), local, "x86_64", {
+    "revision": None, "sourceTimestamp": None,
+})
+local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0]
+local_manifest = json.loads(
+    (local / "blobs/sha256" / local_descriptor["digest"].split(":")[1]).read_text()
+)
+assert "org.opencontainers.image.revision" not in local_manifest["annotations"]
+assert "org.opencontainers.image.created" not in local_manifest["annotations"]
+local_config = json.loads(
+    (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text()
+)
+assert "created" not in local_config
+subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True)
+root = pathlib.Path("extracted")
+paths = closure.read_text().splitlines()
+assert str(dependency) in paths
+assert {p.name for p in (root / "nix/store").iterdir()} == {
+    pathlib.Path(p).name for p in paths
+}
+
+
+def compare(source, target):
+    before, after = source.lstat(), target.lstat()
+    assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target
+    assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target
+    guest = "/" + str(target.relative_to(root))
+    info = subprocess.check_output(
+        ["dump.erofs", f"--path={guest}", str(erofs)], text=True
+    )
+    assert re.search(r"Uid:\s*11000\b", info), info
+    assert re.search(r"Gid:\s*11000\b", info), info
+    if source.is_symlink():
+        assert os.readlink(source) == os.readlink(target), target
+    elif source.is_dir():
+        assert {p.name for p in source.iterdir()} == {p.name for p in target.iterdir()}
+        for child in source.iterdir():
+            compare(child, target / child.name)
+    else:
+        assert source.read_bytes() == target.read_bytes(), target
+
+
+for path in paths:
+    compare(pathlib.Path(path), root / path.lstrip("/"))
+for metadata in (bundle / "etc/nixmodules").iterdir():
+    compare(metadata, root / "etc/nixmodules" / metadata.name)
+packed_dependency = root / str(dependency).lstrip("/")
+assert (packed_dependency / "data").stat().st_ino == (
+    packed_dependency / "hardlink"
+).stat().st_ino
+print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified")

pkgs/bundle-oci/default.nix

@@ -0,0 +1,55 @@
+{ runCommand
+, lib
+, stdenv
+, bundle
+, closureInfo
+, erofs-utils
+, python3
+, writeShellApplication
+, skopeo
+, revision ? null
+, sourceTimestamp ? null
+}:
+
+let
+  bundleClosure = closureInfo { rootPaths = [ bundle ]; };
+  metadata = builtins.toJSON { inherit revision sourceTimestamp; };
+  image = runCommand "nixmodules-bundle-oci"
+    {
+      nativeBuildInputs = [ erofs-utils python3 ];
+      inherit bundle bundleClosure;
+      architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name;
+      __structuredAttrs = true;
+      # The embedded closure must not retain references to the builder's store.
+      unsafeDiscardReferences.out = true;
+      passthru.copyTo = writeShellApplication {
+        name = "copy-nixmodules-bundle-oci";
+        runtimeInputs = [ skopeo ];
+        text = ''
+          if [ "$#" -lt 1 ]; then
+            echo "usage: copy-nixmodules-bundle-oci <transport:destination> [skopeo copy options]" >&2
+            exit 1
+          fi
+          destination="$1"
+          shift
+          exec skopeo --insecure-policy copy --preserve-digests \
+            "oci:${image}:bundle" "$destination" "$@"
+        '';
+      };
+    }
+    ''
+      mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256"
+      cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/
+      while IFS= read -r path; do
+        cp -a --reflink=auto "$path" root/nix/store/
+      done < "$bundleClosure/store-paths"
+      chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules
+
+      # Match the legacy disk's guest ownership. No compression: consumers may
+      # mount this blob directly; compression must not imply DAX page sharing.
+      mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \
+        --force-uid=11000 --force-gid=11000 layer.erofs root
+      python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata}
+    '';
+in
+image

pkgs/bundle-oci/layout.py

@@ -0,0 +1,80 @@
+import hashlib
+import datetime
+import json
+import pathlib
+import shutil
+import sys
+
+
+def write_layout(layer, output, architecture, metadata):
+    architectures = {"x86_64": "amd64", "aarch64": "arm64"}
+    architecture = architectures[architecture]
+    blobs = output / "blobs" / "sha256"
+    annotations = {
+        "org.opencontainers.image.source": "https://github.com/replit/nixmodules",
+        "dev.replit.nixmodules.flake-output": "bundle-oci",
+    }
+    revision = metadata["revision"]
+    if revision is not None:
+        if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision):
+            raise ValueError("source revision must be a full lowercase Git SHA")
+        annotations["org.opencontainers.image.revision"] = revision
+    created = {}
+    if metadata["sourceTimestamp"] is not None:
+        timestamp = datetime.datetime.fromtimestamp(
+            metadata["sourceTimestamp"], datetime.timezone.utc
+        ).strftime("%Y-%m-%dT%H:%M:%SZ")
+        annotations["org.opencontainers.image.created"] = timestamp
+        created["created"] = timestamp
+
+    def descriptor(path, media_type):
+        digest = hashlib.sha256()
+        with path.open("rb") as source:
+            for chunk in iter(lambda: source.read(1024 * 1024), b""):
+                digest.update(chunk)
+        return {
+            "mediaType": media_type,
+            "digest": "sha256:" + digest.hexdigest(),
+            "size": path.stat().st_size,
+        }
+
+    def store_json(value, media_type):
+        data = json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
+        path = blobs / hashlib.sha256(data).hexdigest()
+        path.write_bytes(data)
+        return descriptor(path, media_type)
+
+    layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.erofs")
+    shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1])
+    config = store_json(
+        {
+            **created,
+            "architecture": architecture,
+            "os": "linux",
+            "config": {},
+            "rootfs": {"type": "layers", "diff_ids": [layer_descriptor["digest"]]},
+        },
+        "application/vnd.oci.image.config.v1+json",
+    )
+    manifest = store_json(
+        {
+            "schemaVersion": 2,
+            "mediaType": "application/vnd.oci.image.manifest.v1+json",
+            "config": config,
+            "layers": [layer_descriptor],
+            "annotations": annotations,
+        },
+        "application/vnd.oci.image.manifest.v1+json",
+    )
+    manifest["annotations"] = {"org.opencontainers.image.ref.name": "bundle"}
+    (output / "index.json").write_text(
+        json.dumps({"schemaVersion": 2, "manifests": [manifest]}) + "\n"
+    )
+    (output / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}\n')
+
+
+if __name__ == "__main__":
+    write_layout(
+        pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]),
+        sys.argv[3], json.loads(sys.argv[4]),
+    )

pkgs/default.nix

@@ -45,6 +45,14 @@ rec {
 
   bundle = bundle-fn { };
 
+  bundle-oci = pkgs.callPackage ./bundle-oci {
+    inherit bundle;
+    revision = self.rev or null;
+    sourceTimestamp = self.lastModified or null;
+  };
+
+  bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { };
+
   custom-bundle = bundle-fn {
     moduleIds = dev-module-ids;
   };
@@ -59,7 +67,7 @@ rec {
   bundle-image-tarball = pkgs.callPackage ./bundle-image-tarball { inherit bundle-image revstring; };
 
   disk-script = pkgs.callPackage ./disk-script {
-    bundle = bundle-fn { };
+    inherit bundle;
   };
 
   disk-script-dev = pkgs.callPackage ./disk-script-dev {

scripts/ci_check.sh

@@ -18,3 +18,5 @@ nix eval "${NIX_FLAGS[@]}" .#modules --json
 nix develop "${NIX_FLAGS[@]}" --command echo Hello, world
 
 nix eval .#bundle
+
+nix build .#bundle-oci-check --no-link