From 80b62dfba70b179b2b41f92e0d58c9dca3686a52 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 5 Oct 2026 20:23:09 +0000 Subject: [PATCH 1/3] Add native EROFS OCI producer for the complete module bundle Assisted-by: Replit --- README.md | 27 ++++++++++++++ pkgs/bundle-oci/check.nix | 30 ++++++++++++++++ pkgs/bundle-oci/check.py | 71 +++++++++++++++++++++++++++++++++++++ pkgs/bundle-oci/default.nix | 34 ++++++++++++++++++ pkgs/bundle-oci/layout.py | 58 ++++++++++++++++++++++++++++++ pkgs/default.nix | 6 +++- 6 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 pkgs/bundle-oci/check.nix create mode 100644 pkgs/bundle-oci/check.py create mode 100644 pkgs/bundle-oci/default.nix create mode 100644 pkgs/bundle-oci/layout.py diff --git a/README.md b/README.md index dd593d33..911385d2 100644 --- a/README.md +++ b/README.md @@ -65,3 +65,30 @@ or ``` nix build .#custom-bundle-squashfs ``` + +To build the full bundle (including historical modules) as a native EROFS OCI image: + +``` +nix build .#bundle-oci +``` + +The result is an OCI image layout directory with the reference `bundle`, suitable +for copying with an OCI-layout-aware tool such as +`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`. +It contains one uncompressed `application/vnd.oci.image.layer.v1.erofs` layer, +not a tar layer. The consumer must support native EROFS layers; ordinary +tar-only image unpackers cannot use it. Its diffID equals the layer blob digest. + +The filesystem is generated directly from the same pinned full bundle used by +`disk-script`: its complete store closure and `/etc/nixmodules` metadata, with +unchanged store paths, file contents, modes and symlink targets. All guest +UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and +per-module OCI outputs are unchanged. There is no disk conversion or dependency +update. Uncompressed EROFS does not by itself guarantee DAX sharing; that also +depends on the consumer's backing storage and mount configuration. + +A small fixture exercises the producer without building the full bundle: + +``` +nix build .#bundle-oci-check +``` diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix new file mode 100644 index 00000000..d099ffc9 --- /dev/null +++ b/pkgs/bundle-oci/check.nix @@ -0,0 +1,30 @@ +{ pkgs }: + +let + dependency = pkgs.runCommand "bundle-oci-dependency" { } '' + mkdir -p "$out" + printf 'dependency\n' > "$out/data" + ln "$out/data" "$out/hardlink" + ''; + module = pkgs.runCommand "bundle-oci-module" { } '' + mkdir -p "$out/bin" + printf '%s\n' '${dependency}' > "$out/module.json" + printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" + chmod 755 "$out/bin/tool" + ln -s ../module.json "$out/bin/relative" + ln -s '${dependency}/data' "$out/absolute" + ''; + bundle = (pkgs.callPackage ../bundle { + self = { modules.fixture = module; }; + }) { }; + image = pkgs.callPackage ./. { inherit bundle; }; + closure = pkgs.closureInfo { rootPaths = [ bundle ]; }; +in +pkgs.runCommand "bundle-oci-check" +{ + nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ]; +} +'' + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} + touch "$out" +'' diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py new file mode 100644 index 00000000..bf095f9a --- /dev/null +++ b/pkgs/bundle-oci/check.py @@ -0,0 +1,71 @@ +import hashlib +import json +import os +import pathlib +import re +import stat +import subprocess +import sys + +image, bundle, closure, dependency = map(pathlib.Path, sys.argv[1:]) +assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} + + +def blob(descriptor): + path = image / "blobs" / "sha256" / descriptor["digest"].removeprefix("sha256:") + assert path.stat().st_size == descriptor["size"] + assert "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() == descriptor["digest"] + return path + + +index = json.loads((image / "index.json").read_text()) +assert index["schemaVersion"] == 2 and len(index["manifests"]) == 1 +manifest = json.loads(blob(index["manifests"][0]).read_text()) +assert manifest["schemaVersion"] == 2 +assert manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json" +assert manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" +config = json.loads(blob(manifest["config"]).read_text()) +assert config["architecture"] == "amd64" and config["os"] == "linux" +assert len(manifest["layers"]) == 1 +layer = manifest["layers"][0] +assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs" +assert config["rootfs"] == {"type": "layers", "diff_ids": [layer["digest"]]} +erofs = blob(layer) +subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True) +root = pathlib.Path("extracted") +paths = closure.read_text().splitlines() +assert str(dependency) in paths +assert {p.name for p in (root / "nix/store").iterdir()} == { + pathlib.Path(p).name for p in paths +} + + +def compare(source, target): + before, after = source.lstat(), target.lstat() + assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target + assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target + guest = "/" + str(target.relative_to(root)) + info = subprocess.check_output( + ["dump.erofs", f"--path={guest}", str(erofs)], text=True + ) + assert re.search(r"UID:\s*11000", info), info + assert re.search(r"GID:\s*11000", info), info + if source.is_symlink(): + assert os.readlink(source) == os.readlink(target), target + elif source.is_dir(): + assert {p.name for p in source.iterdir()} == {p.name for p in target.iterdir()} + for child in source.iterdir(): + compare(child, target / child.name) + else: + assert source.read_bytes() == target.read_bytes(), target + + +for path in paths: + compare(pathlib.Path(path), root / path.lstrip("/")) +for metadata in (bundle / "etc/nixmodules").iterdir(): + compare(metadata, root / "etc/nixmodules" / metadata.name) +packed_dependency = root / str(dependency).lstrip("/") +assert (packed_dependency / "data").stat().st_ino == ( + packed_dependency / "hardlink" +).stat().st_ino +print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified") diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix new file mode 100644 index 00000000..958a0e5c --- /dev/null +++ b/pkgs/bundle-oci/default.nix @@ -0,0 +1,34 @@ +{ runCommand +, lib +, stdenv +, bundle +, closureInfo +, erofs-utils +, python3 +}: + +let + bundleClosure = closureInfo { rootPaths = [ bundle ]; }; +in +runCommand "nixmodules-bundle-oci" +{ + nativeBuildInputs = [ erofs-utils python3 ]; + inherit bundle bundleClosure; + architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; + # The embedded closure must not retain references to the builder's store. + unsafeDiscardReferences.out = true; +} +'' + mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" + cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ + while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ + done < "$bundleClosure/store-paths" + chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules + + # Match the legacy disk's guest ownership. No compression: consumers may + # mount this blob directly; compression must not imply DAX page sharing. + mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ + --force-uid=11000 --force-gid=11000 layer.erofs root + python3 ${./layout.py} layer.erofs "$out" "$architecture" +'' diff --git a/pkgs/bundle-oci/layout.py b/pkgs/bundle-oci/layout.py new file mode 100644 index 00000000..74981c4e --- /dev/null +++ b/pkgs/bundle-oci/layout.py @@ -0,0 +1,58 @@ +import hashlib +import json +import pathlib +import shutil +import sys + + +def write_layout(layer, output, architecture): + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + architecture = architectures[architecture] + blobs = output / "blobs" / "sha256" + + def descriptor(path, media_type): + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return { + "mediaType": media_type, + "digest": "sha256:" + digest.hexdigest(), + "size": path.stat().st_size, + } + + def store_json(value, media_type): + data = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + path = blobs / hashlib.sha256(data).hexdigest() + path.write_bytes(data) + return descriptor(path, media_type) + + layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.erofs") + shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1]) + config = store_json( + { + "architecture": architecture, + "os": "linux", + "config": {}, + "rootfs": {"type": "layers", "diff_ids": [layer_descriptor["digest"]]}, + }, + "application/vnd.oci.image.config.v1+json", + ) + manifest = store_json( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config, + "layers": [layer_descriptor], + }, + "application/vnd.oci.image.manifest.v1+json", + ) + manifest["annotations"] = {"org.opencontainers.image.ref.name": "bundle"} + (output / "index.json").write_text( + json.dumps({"schemaVersion": 2, "manifests": [manifest]}) + "\n" + ) + (output / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}\n') + + +if __name__ == "__main__": + write_layout(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3]) diff --git a/pkgs/default.nix b/pkgs/default.nix index 63bc30e8..167c6e3f 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -45,6 +45,10 @@ rec { bundle = bundle-fn { }; + bundle-oci = pkgs.callPackage ./bundle-oci { inherit bundle; }; + + bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { }; + custom-bundle = bundle-fn { moduleIds = dev-module-ids; }; @@ -59,7 +63,7 @@ rec { bundle-image-tarball = pkgs.callPackage ./bundle-image-tarball { inherit bundle-image revstring; }; disk-script = pkgs.callPackage ./disk-script { - bundle = bundle-fn { }; + inherit bundle; }; disk-script-dev = pkgs.callPackage ./disk-script-dev { From 3e9505ecc8370bb1b490b9060541535f144ded88 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 5 Oct 2026 20:34:02 +0000 Subject: [PATCH 2/3] Validate native EROFS fixture and enable it in CI Assisted-by: Replit --- pkgs/bundle-oci/check.nix | 8 ++++---- pkgs/bundle-oci/check.py | 4 ++-- pkgs/bundle-oci/default.nix | 27 ++++++++++++++------------- scripts/ci_check.sh | 2 ++ 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index d099ffc9..420ed4cc 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -24,7 +24,7 @@ pkgs.runCommand "bundle-oci-check" { nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ]; } -'' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} - touch "$out" -'' + '' + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} + touch "$out" + '' diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index bf095f9a..4c051f09 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -48,8 +48,8 @@ def compare(source, target): info = subprocess.check_output( ["dump.erofs", f"--path={guest}", str(erofs)], text=True ) - assert re.search(r"UID:\s*11000", info), info - assert re.search(r"GID:\s*11000", info), info + assert re.search(r"Uid:\s*11000\b", info), info + assert re.search(r"Gid:\s*11000\b", info), info if source.is_symlink(): assert os.readlink(source) == os.readlink(target), target elif source.is_dir(): diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index 958a0e5c..355a8a90 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -15,20 +15,21 @@ runCommand "nixmodules-bundle-oci" nativeBuildInputs = [ erofs-utils python3 ]; inherit bundle bundleClosure; architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; + __structuredAttrs = true; # The embedded closure must not retain references to the builder's store. unsafeDiscardReferences.out = true; } -'' - mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" - cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ - while IFS= read -r path; do - cp -a --reflink=auto "$path" root/nix/store/ - done < "$bundleClosure/store-paths" - chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules + '' + mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" + cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ + while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ + done < "$bundleClosure/store-paths" + chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - # Match the legacy disk's guest ownership. No compression: consumers may - # mount this blob directly; compression must not imply DAX page sharing. - mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ - --force-uid=11000 --force-gid=11000 layer.erofs root - python3 ${./layout.py} layer.erofs "$out" "$architecture" -'' + # Match the legacy disk's guest ownership. No compression: consumers may + # mount this blob directly; compression must not imply DAX page sharing. + mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ + --force-uid=11000 --force-gid=11000 layer.erofs root + python3 ${./layout.py} layer.erofs "$out" "$architecture" + '' diff --git a/scripts/ci_check.sh b/scripts/ci_check.sh index 83bac663..b964c939 100755 --- a/scripts/ci_check.sh +++ b/scripts/ci_check.sh @@ -18,3 +18,5 @@ nix eval "${NIX_FLAGS[@]}" .#modules --json nix develop "${NIX_FLAGS[@]}" --command echo Hello, world nix eval .#bundle + +nix build .#bundle-oci-check --no-link From 8900856a030cf9bcab0f1f18d53080e22a4a844e Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 6 Oct 2026 00:32:19 +0000 Subject: [PATCH 3/3] Embed source provenance and provide digest-preserving image copy helper Assisted-by: Replit --- README.md | 18 ++++++++++ pkgs/bundle-oci/check.nix | 22 +++++++++++-- pkgs/bundle-oci/check.py | 40 +++++++++++++++++++++- pkgs/bundle-oci/default.nix | 66 ++++++++++++++++++++++++------------- pkgs/bundle-oci/layout.py | 26 +++++++++++++-- pkgs/default.nix | 6 +++- 6 files changed, 149 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 911385d2..8cf7a992 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,24 @@ nix build .#bundle-oci The result is an OCI image layout directory with the reference `bundle`, suitable for copying with an OCI-layout-aware tool such as `skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`. +The output also provides a copy helper that keeps the exact image in its Nix +closure and preserves the manifest digest: + +``` +nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest +``` + +The destination can use any Skopeo-supported transport, including `oci:`. +Additional arguments are passed to `skopeo copy`. The helper trusts the locally +built source (`--insecure-policy`); registry TLS verification remains enabled. +For a clean Git source, the manifest records the full flake revision, source URL +`https://github.com/replit/nixmodules`, and output name `bundle-oci` in OCI +annotations. The manifest creation annotation and config `created` field use +the flake's last-modified timestamp in UTC, never the build's wall clock. +Local sources without a clean revision omit the revision annotation; sources +without a last-modified timestamp omit creation metadata. Publish from a +revision-pinned Git flake when complete provenance is required. + It contains one uncompressed `application/vnd.oci.image.layer.v1.erofs` layer, not a tar layer. The consumer must support native EROFS layers; ordinary tar-only image unpackers cannot use it. Its diffID equals the layer blob digest. diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index 420ed4cc..eec9ab62 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -17,7 +17,11 @@ let bundle = (pkgs.callPackage ../bundle { self = { modules.fixture = module; }; }) { }; - image = pkgs.callPackage ./. { inherit bundle; }; + image = pkgs.callPackage ./. { + inherit bundle; + revision = "0123456789abcdef0123456789abcdef01234567"; + sourceTimestamp = 1700000000; + }; closure = pkgs.closureInfo { rootPaths = [ bundle ]; }; in pkgs.runCommand "bundle-oci-check" @@ -25,6 +29,20 @@ pkgs.runCommand "bundle-oci-check" nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ]; } '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} + ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \ + --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ + --digestfile "$PWD/copied.digest" + python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY' + import json, pathlib, sys + original, copied, digest_file = map(pathlib.Path, sys.argv[1:]) + descriptor = json.loads((original / "index.json").read_text())["manifests"][0] + copied_descriptor = json.loads((copied / "index.json").read_text())["manifests"][0] + assert copied_descriptor["digest"] == descriptor["digest"] + assert digest_file.read_text().strip() == descriptor["digest"] + for blob in (original / "blobs/sha256").iterdir(): + assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes() + print("copyTo preserved manifest digest and every OCI blob") + PY touch "$out" '' diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index 4c051f09..b332fb73 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -1,13 +1,15 @@ import hashlib +import importlib.util import json import os import pathlib import re import stat +import shutil import subprocess import sys -image, bundle, closure, dependency = map(pathlib.Path, sys.argv[1:]) +image, bundle, closure, dependency, layout_script = map(pathlib.Path, sys.argv[1:]) assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} @@ -26,11 +28,47 @@ def blob(descriptor): assert manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" config = json.loads(blob(manifest["config"]).read_text()) assert config["architecture"] == "amd64" and config["os"] == "linux" +assert config["created"] == "2023-11-14T22:13:20Z" +assert manifest["annotations"] == { + "org.opencontainers.image.source": "https://github.com/replit/nixmodules", + "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567", + "org.opencontainers.image.created": config["created"], + "dev.replit.nixmodules.flake-output": "bundle-oci", +} assert len(manifest["layers"]) == 1 layer = manifest["layers"][0] assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs" assert config["rootfs"] == {"type": "layers", "diff_ids": [layer["digest"]]} erofs = blob(layer) +spec = importlib.util.spec_from_file_location("layout", layout_script) +layout = importlib.util.module_from_spec(spec) +spec.loader.exec_module(layout) +repeated = pathlib.Path("repeated") +(repeated / "blobs/sha256").mkdir(parents=True) +shutil.copyfile(erofs, "repeated.erofs") +layout.write_layout(pathlib.Path("repeated.erofs"), repeated, "x86_64", { + "revision": "0123456789abcdef0123456789abcdef01234567", + "sourceTimestamp": 1700000000, +}) +for path in image.rglob("*"): + if path.is_file(): + assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes() +local = pathlib.Path("local") +(local / "blobs/sha256").mkdir(parents=True) +shutil.copyfile(erofs, "local.erofs") +layout.write_layout(pathlib.Path("local.erofs"), local, "x86_64", { + "revision": None, "sourceTimestamp": None, +}) +local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0] +local_manifest = json.loads( + (local / "blobs/sha256" / local_descriptor["digest"].split(":")[1]).read_text() +) +assert "org.opencontainers.image.revision" not in local_manifest["annotations"] +assert "org.opencontainers.image.created" not in local_manifest["annotations"] +local_config = json.loads( + (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text() +) +assert "created" not in local_config subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True) root = pathlib.Path("extracted") paths = closure.read_text().splitlines() diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index 355a8a90..3693a580 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -5,31 +5,51 @@ , closureInfo , erofs-utils , python3 +, writeShellApplication +, skopeo +, revision ? null +, sourceTimestamp ? null }: let bundleClosure = closureInfo { rootPaths = [ bundle ]; }; -in -runCommand "nixmodules-bundle-oci" -{ - nativeBuildInputs = [ erofs-utils python3 ]; - inherit bundle bundleClosure; - architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; - __structuredAttrs = true; - # The embedded closure must not retain references to the builder's store. - unsafeDiscardReferences.out = true; -} - '' - mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" - cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ - while IFS= read -r path; do - cp -a --reflink=auto "$path" root/nix/store/ - done < "$bundleClosure/store-paths" - chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules + metadata = builtins.toJSON { inherit revision sourceTimestamp; }; + image = runCommand "nixmodules-bundle-oci" + { + nativeBuildInputs = [ erofs-utils python3 ]; + inherit bundle bundleClosure; + architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; + __structuredAttrs = true; + # The embedded closure must not retain references to the builder's store. + unsafeDiscardReferences.out = true; + passthru.copyTo = writeShellApplication { + name = "copy-nixmodules-bundle-oci"; + runtimeInputs = [ skopeo ]; + text = '' + if [ "$#" -lt 1 ]; then + echo "usage: copy-nixmodules-bundle-oci [skopeo copy options]" >&2 + exit 1 + fi + destination="$1" + shift + exec skopeo --insecure-policy copy --preserve-digests \ + "oci:${image}:bundle" "$destination" "$@" + ''; + }; + } + '' + mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" + cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ + while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ + done < "$bundleClosure/store-paths" + chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - # Match the legacy disk's guest ownership. No compression: consumers may - # mount this blob directly; compression must not imply DAX page sharing. - mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ - --force-uid=11000 --force-gid=11000 layer.erofs root - python3 ${./layout.py} layer.erofs "$out" "$architecture" - '' + # Match the legacy disk's guest ownership. No compression: consumers may + # mount this blob directly; compression must not imply DAX page sharing. + mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ + --force-uid=11000 --force-gid=11000 layer.erofs root + python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata} + ''; +in +image diff --git a/pkgs/bundle-oci/layout.py b/pkgs/bundle-oci/layout.py index 74981c4e..c2fe2c23 100644 --- a/pkgs/bundle-oci/layout.py +++ b/pkgs/bundle-oci/layout.py @@ -1,14 +1,31 @@ import hashlib +import datetime import json import pathlib import shutil import sys -def write_layout(layer, output, architecture): +def write_layout(layer, output, architecture, metadata): architectures = {"x86_64": "amd64", "aarch64": "arm64"} architecture = architectures[architecture] blobs = output / "blobs" / "sha256" + annotations = { + "org.opencontainers.image.source": "https://github.com/replit/nixmodules", + "dev.replit.nixmodules.flake-output": "bundle-oci", + } + revision = metadata["revision"] + if revision is not None: + if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision): + raise ValueError("source revision must be a full lowercase Git SHA") + annotations["org.opencontainers.image.revision"] = revision + created = {} + if metadata["sourceTimestamp"] is not None: + timestamp = datetime.datetime.fromtimestamp( + metadata["sourceTimestamp"], datetime.timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + annotations["org.opencontainers.image.created"] = timestamp + created["created"] = timestamp def descriptor(path, media_type): digest = hashlib.sha256() @@ -31,6 +48,7 @@ def store_json(value, media_type): shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1]) config = store_json( { + **created, "architecture": architecture, "os": "linux", "config": {}, @@ -44,6 +62,7 @@ def store_json(value, media_type): "mediaType": "application/vnd.oci.image.manifest.v1+json", "config": config, "layers": [layer_descriptor], + "annotations": annotations, }, "application/vnd.oci.image.manifest.v1+json", ) @@ -55,4 +74,7 @@ def store_json(value, media_type): if __name__ == "__main__": - write_layout(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3]) + write_layout( + pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), + sys.argv[3], json.loads(sys.argv[4]), + ) diff --git a/pkgs/default.nix b/pkgs/default.nix index 167c6e3f..ecf1dd5a 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -45,7 +45,11 @@ rec { bundle = bundle-fn { }; - bundle-oci = pkgs.callPackage ./bundle-oci { inherit bundle; }; + bundle-oci = pkgs.callPackage ./bundle-oci { + inherit bundle; + revision = self.rev or null; + sourceTimestamp = self.lastModified or null; + }; bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { };