From 9d5d73ecd9cca888624471c1bccfd1476dff0830 Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 24 Sep 2026 20:33:28 +0000 Subject: [PATCH 1/3] Include Nix store registration in disk images Assisted-by: Replit --- flake.nix | 6 + pkgs/bundle-image/builder.sh | 5 +- pkgs/bundle-image/default.nix | 4 +- pkgs/default.nix | 16 +- .../disk-image-registration-check/default.nix | 171 ++++++++++++++++++ pkgs/disk-script-dev/default.nix | 10 +- pkgs/disk-script/default.nix | 10 +- pkgs/store-registration/default.nix | 19 ++ 8 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 pkgs/disk-image-registration-check/default.nix create mode 100644 pkgs/store-registration/default.nix diff --git a/flake.nix b/flake.nix index e9e46a2e..5a4c1aee 100644 --- a/flake.nix +++ b/flake.nix @@ -109,6 +109,12 @@ packages.x86_64-linux = import ./pkgs { inherit pkgs self; }; + checks.x86_64-linux.disk-image-registration = pkgs.callPackage ./pkgs/disk-image-registration-check { + productionImage = self.packages.x86_64-linux.bundle-image; + productionBundle = self.packages.x86_64-linux.bundle; + developmentImage = self.packages.x86_64-linux.bundle-squashfs; + developmentBundle = self.packages.x86_64-linux.custom-bundle; + }; devShells.x86_64-linux.default = pkgs.mkShell { packages = with pkgs; [ python310 diff --git a/pkgs/bundle-image/builder.sh b/pkgs/bundle-image/builder.sh index 4db9ae4a..d5ea64a4 100644 --- a/pkgs/bundle-image/builder.sh +++ b/pkgs/bundle-image/builder.sh @@ -7,11 +7,12 @@ out="${outputs[out]}" mkdir "$out" root="$PWD/root" -mkdir -p "$root/nix/store" "$root/etc/nixmodules" +mkdir -p "$root/nix/store" "$root/etc/nixmodules" "$root/nix-lower-registration/v1" cp --archive --reflink=auto "${env["bundle"]}/etc/nixmodules/"* "$root/etc/nixmodules" +cp --archive --reflink=auto "${env["storeRegistration"]}/." "$root/nix-lower-registration/v1/" -xargs -I % cp -a --reflink=auto % "$root/nix/store/" < "${env[diskClosureInfo]}"/store-paths +xargs -I % cp -a --reflink=auto % "$root/nix/store/" < "${env["storeRegistration"]}"/store-paths diskImage=$out/${env[diskName]} diff --git a/pkgs/bundle-image/default.nix b/pkgs/bundle-image/default.nix index 5a43bf69..41c53141 100644 --- a/pkgs/bundle-image/default.nix +++ b/pkgs/bundle-image/default.nix @@ -2,10 +2,10 @@ , bash , lib , bundle +, storeRegistration , revstring , coreutils , findutils -, closureInfo , squashfsTools , fetchFromGitHub , pkgs @@ -30,6 +30,6 @@ derivation { findutils squashfsTools ]; - diskClosureInfo = closureInfo { rootPaths = [ bundle ]; }; + inherit storeRegistration; }; } diff --git a/pkgs/default.nix b/pkgs/default.nix index 63bc30e8..3484ae28 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -28,13 +28,23 @@ let bundle-fn = pkgs.callPackage ./bundle { inherit self; }; + store-registration-fn = pkgs.callPackage ./store-registration { }; + + store-registration = bundle: store-registration-fn { + rootPaths = [ bundle ]; + }; + bundle-squashfs-fn = { moduleIds ? null , diskName ? "disk.raw" , }: - pkgs.callPackage ./bundle-image { + let bundle = bundle-fn { inherit moduleIds; }; + in + pkgs.callPackage ./bundle-image { + inherit bundle; + storeRegistration = store-registration bundle; inherit revstring diskName; }; @@ -60,12 +70,16 @@ rec { disk-script = pkgs.callPackage ./disk-script { bundle = bundle-fn { }; + storeRegistration = store-registration (bundle-fn { }); }; disk-script-dev = pkgs.callPackage ./disk-script-dev { bundle = bundle-fn { moduleIds = dev-module-ids; }; + storeRegistration = store-registration (bundle-fn { + moduleIds = dev-module-ids; + }); }; # For dev use: builds the shared Nixmodules disk diff --git a/pkgs/disk-image-registration-check/default.nix b/pkgs/disk-image-registration-check/default.nix new file mode 100644 index 00000000..dbbc472d --- /dev/null +++ b/pkgs/disk-image-registration-check/default.nix @@ -0,0 +1,171 @@ +{ coreutils +, diffutils +, findutils +, gawk +, nix +, runCommand +, squashfsTools +, productionImage +, productionBundle +, developmentImage +, developmentBundle +}: + +runCommand "nixmodules-disk-image-registration-check" { + nativeBuildInputs = [ + coreutils + diffutils + findutils + gawk + nix + squashfsTools + ]; +} '' + validate_metadata() { + local root="$1" + local artifact="$2" + local name="$3" + local expected="$TMPDIR/expected-$name" + local registration_index="$TMPDIR/registration-index-$name" + local image_paths="$TMPDIR/image-paths-$name" + local registered_paths="$TMPDIR/registered-paths-$name" + + test -f "$artifact/format-version" || return 1 + test "$(cat "$artifact/format-version")" = 1 || return 1 + test -f "$artifact/registration" || return 1 + test -f "$artifact/store-paths" || return 1 + test -f "$artifact/SHA256SUMS" || return 1 + (cd "$artifact" && sha256sum -c SHA256SUMS) || return 1 + + LC_ALL=C sort -u "$artifact/store-paths" > "$expected" + find "$root/nix/store" -mindepth 1 -maxdepth 1 -printf '/nix/store/%f\n' | LC_ALL=C sort > "$image_paths" + cmp "$expected" "$image_paths" || return 1 + while IFS= read -r path; do + test -e "$root$path" || return 1 + done < "$expected" + + gawk ' + state == 0 { entry = $0; print entry; state = 1; next } + state == 1 { state = 2; next } + state == 2 { state = 3; next } + state == 3 { state = 4; next } + state == 4 { + if ($0 !~ /^[0-9]+$/) exit 1 + remaining = $0 + 0 + state = remaining == 0 ? 0 : 5 + next + } + state == 5 { + print entry "\t" $0 + remaining-- + if (remaining == 0) state = 0 + } + END { if (state != 0) exit 1 } + ' "$artifact/registration" > "$registration_index" || return 1 + awk -F '\t' 'NR == FNR { paths[$0] = 1; next } NF == 1 { if (!($0 in paths)) exit 1; next } !($1 in paths) || !($2 in paths) { exit 1 }' \ + "$expected" "$registration_index" || return 1 + awk -F '\t' 'NF == 1 { print }' "$registration_index" | LC_ALL=C sort > "$registered_paths" + cmp "$expected" "$registered_paths" || return 1 + } + + check_image() { + local image="$1" + local bundle="$2" + local name="$3" + local root="$TMPDIR/root-$name" + local scratch="$TMPDIR/store-$name" + local store="local?root=$scratch" + local artifact="$root/nix-lower-registration/v1" + local testdir="$TMPDIR/metadata-tests-$name" + + unsquashfs -no-progress -d "$root" "$image" >/dev/null + validate_metadata "$root" "$artifact" "$name" + mkdir -p "$testdir" + + cp -a "$artifact" "$testdir/artifact-missing-version" + chmod -R u+w "$testdir/artifact-missing-version" + rm "$testdir/artifact-missing-version/format-version" + if validate_metadata "$root" "$testdir/artifact-missing-version" "$name-missing-version" >/dev/null 2>&1; then + echo "missing format version was not rejected" >&2 + exit 1 + fi + if test -f "$scratch/nix/var/nix/db"; then + echo "invalid metadata created a scratch database" >&2 + exit 1 + fi + + cp -a "$artifact" "$testdir/artifact-unknown-version" + chmod -R u+w "$testdir/artifact-unknown-version" + printf '999\n' > "$testdir/artifact-unknown-version/format-version" + (cd "$testdir/artifact-unknown-version" && sha256sum format-version registration store-paths > SHA256SUMS) + if validate_metadata "$root" "$testdir/artifact-unknown-version" "$name-unknown-version" >/dev/null 2>&1; then + echo "unknown format version was not rejected" >&2 + exit 1 + fi + + cp -a "$artifact" "$testdir/artifact-digest-mismatch" + chmod -R u+w "$testdir/artifact-digest-mismatch" + printf '\n' >> "$testdir/artifact-digest-mismatch/registration" + if validate_metadata "$root" "$testdir/artifact-digest-mismatch" "$name-digest-mismatch" >/dev/null 2>&1; then + echo "registration tampering passed the integrity check" >&2 + exit 1 + fi + + cp -a "$artifact" "$testdir/artifact-nonclosed" + chmod -R u+w "$testdir/artifact-nonclosed" + gawk ' + state == 0 { state = 1; next } + state == 1 { state = 2; next } + state == 2 { state = 3; next } + state == 3 { state = 4; next } + state == 4 { remaining = $0 + 0; state = remaining == 0 ? 0 : 5; next } + state == 5 { + if (!changed) { + print "/nix/store/not-in-registration" + changed = 1 + } else print + remaining-- + if (remaining == 0) state = 0 + } + END { if (!changed || state != 0) exit 1 } + ' "$artifact/registration" > "$testdir/artifact-nonclosed/registration" + (cd "$testdir/artifact-nonclosed" && sha256sum format-version registration store-paths > SHA256SUMS) + if validate_metadata "$root" "$testdir/artifact-nonclosed" "$name-nonclosed" >/dev/null 2>&1; then + echo "non-closed registration reference was not rejected" >&2 + exit 1 + fi + + cp -R "$root" "$testdir/root-missing-path" + rm -rf "$testdir/root-missing-path/nix/store/$(basename "$bundle")" + if validate_metadata "$testdir/root-missing-path" "$testdir/root-missing-path/nix-lower-registration/v1" "$name-missing-path" >/dev/null 2>&1; then + echo "missing image path was not rejected" >&2 + exit 1 + fi + + mkdir -p "$scratch/nix/store" + cp -R "$root/nix/store/." "$scratch/nix/store/" + nix-store --store "$store" --query --valid-paths > "$TMPDIR/valid-before-$name" + test ! -s "$TMPDIR/valid-before-$name" + nix-store --store "$store" --load-db < "$artifact/registration" + nix-store --store "$store" --query --valid-paths | LC_ALL=C sort > "$TMPDIR/registered-$name" + cmp "$TMPDIR/expected-$name" "$TMPDIR/registered-$name" + nix-store --store "$store" --verify-path "$bundle" + nix-store --store "$store" --query --references "$bundle" > "$TMPDIR/references-$name" + test -s "$TMPDIR/references-$name" + while IFS= read -r path; do + grep -Fxq "$path" "$TMPDIR/expected-$name" + done < "$TMPDIR/references-$name" + + file="$(find "$scratch$bundle" -type f -print -quit)" + chmod u+w "$file" + printf tampered >> "$file" + if nix-store --store "$store" --verify-path "$bundle" >/dev/null 2>&1; then + echo "store content tampering passed Nix path verification" >&2 + exit 1 + fi + } + + check_image ${productionImage}/disk.raw ${productionBundle} production + check_image ${developmentImage}/disk.sqsh ${developmentBundle} development + touch "$out" +'' diff --git a/pkgs/disk-script-dev/default.nix b/pkgs/disk-script-dev/default.nix index 11507bae..137ee9f3 100644 --- a/pkgs/disk-script-dev/default.nix +++ b/pkgs/disk-script-dev/default.nix @@ -3,13 +3,10 @@ , squashfsTools , coreutils , findutils -, closureInfo +, storeRegistration , }: -let - diskClosureInfo = closureInfo { rootPaths = [ bundle ]; }; -in writeShellApplication { name = "disk-script"; runtimeInputs = [ @@ -27,12 +24,13 @@ writeShellApplication { diskImage="$TMP_DIR/disk.sqsh" ( - mkdir -p "$root/nix/store" "$root/etc/nixmodules" + mkdir -p "$root/nix/store" "$root/etc/nixmodules" "$root/nix-lower-registration/v1" cp --archive --reflink=auto "${bundle}/etc/nixmodules/"* "$root/etc/nixmodules" + cp --archive --reflink=auto "${storeRegistration}/." "$root/nix-lower-registration/v1/" SECONDS=0 - xargs -P "$(nproc)" cp -a --reflink=auto -t "$root/nix/store/" < "${diskClosureInfo}/store-paths" + xargs -P "$(nproc)" cp -a --reflink=auto -t "$root/nix/store/" < "${storeRegistration}/store-paths" echo "xargs copy took $SECONDS seconds" >&2 echo "making squashfs..." diff --git a/pkgs/disk-script/default.nix b/pkgs/disk-script/default.nix index ccb495e5..d30d8611 100644 --- a/pkgs/disk-script/default.nix +++ b/pkgs/disk-script/default.nix @@ -5,14 +5,11 @@ , pigz , coreutils , findutils -, closureInfo , pv +, storeRegistration , }: -let - diskClosureInfo = closureInfo { rootPaths = [ bundle ]; }; -in writeShellApplication { name = "disk-script"; runtimeInputs = [ @@ -35,12 +32,13 @@ writeShellApplication { tarball="$TMP_DIR/disk.raw.tar.gz" ( - mkdir -p "$root/nix/store" "$root/etc/nixmodules" + mkdir -p "$root/nix/store" "$root/etc/nixmodules" "$root/nix-lower-registration/v1" cp --archive --reflink=auto "${bundle}/etc/nixmodules/"* "$root/etc/nixmodules" + cp --archive --reflink=auto "${storeRegistration}/." "$root/nix-lower-registration/v1/" SECONDS=0 - xargs -P "$(nproc)" cp -a --reflink=auto -t "$root/nix/store/" < "${diskClosureInfo}/store-paths" + xargs -P "$(nproc)" cp -a --reflink=auto -t "$root/nix/store/" < "${storeRegistration}/store-paths" echo "xargs copy took $SECONDS seconds" >&2 echo "making squashfs..." diff --git a/pkgs/store-registration/default.nix b/pkgs/store-registration/default.nix new file mode 100644 index 00000000..c649371d --- /dev/null +++ b/pkgs/store-registration/default.nix @@ -0,0 +1,19 @@ +{ closureInfo +, coreutils +, runCommand +}: +{ rootPaths }: + +let + closure = closureInfo { inherit rootPaths; }; +in +runCommand "nixmodules-store-registration-v1" { + nativeBuildInputs = [ coreutils ]; +} '' + mkdir -p "$out" + printf '1\n' > "$out/format-version" + cp ${closure}/registration "$out/registration" + LC_ALL=C sort -u ${closure}/store-paths > "$out/store-paths" + cd "$out" + sha256sum format-version registration store-paths > SHA256SUMS +'' From 4302239f724a50e2df1229d59c2d6aa8f93eae29 Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 24 Sep 2026 20:49:24 +0000 Subject: [PATCH 2/3] Add lightweight disk registration image fixture Assisted-by: Replit --- flake.nix | 16 +++++++------ .../disk-image-registration-check/default.nix | 13 ++++++---- .../default.nix | 24 +++++++++++++++++++ 3 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 pkgs/disk-image-registration-fixture/default.nix diff --git a/flake.nix b/flake.nix index 5a4c1aee..f20fa994 100644 --- a/flake.nix +++ b/flake.nix @@ -106,15 +106,17 @@ }; formatter.x86_64-linux = pkgs.nixpkgs-fmt; formatter.aarch64-darwin = pkgs-aarch64-darwin.nixpkgs-fmt; - packages.x86_64-linux = import ./pkgs { + packages.x86_64-linux = (import ./pkgs { inherit pkgs self; + }) // { + disk-image-registration-full = pkgs.callPackage ./pkgs/disk-image-registration-check { + productionImage = self.packages.x86_64-linux.bundle-image; + productionBundle = self.packages.x86_64-linux.bundle; + developmentImage = self.packages.x86_64-linux.bundle-squashfs; + developmentBundle = self.packages.x86_64-linux.custom-bundle; + }; }; - checks.x86_64-linux.disk-image-registration = pkgs.callPackage ./pkgs/disk-image-registration-check { - productionImage = self.packages.x86_64-linux.bundle-image; - productionBundle = self.packages.x86_64-linux.bundle; - developmentImage = self.packages.x86_64-linux.bundle-squashfs; - developmentBundle = self.packages.x86_64-linux.custom-bundle; - }; + checks.x86_64-linux.disk-image-registration-fixture = pkgs.callPackage ./pkgs/disk-image-registration-fixture { }; devShells.x86_64-linux.default = pkgs.mkShell { packages = with pkgs; [ python310 diff --git a/pkgs/disk-image-registration-check/default.nix b/pkgs/disk-image-registration-check/default.nix index dbbc472d..66c6d6a9 100644 --- a/pkgs/disk-image-registration-check/default.nix +++ b/pkgs/disk-image-registration-check/default.nix @@ -136,6 +136,7 @@ runCommand "nixmodules-disk-image-registration-check" { fi cp -R "$root" "$testdir/root-missing-path" + chmod -R u+w "$testdir/root-missing-path/nix/store" rm -rf "$testdir/root-missing-path/nix/store/$(basename "$bundle")" if validate_metadata "$testdir/root-missing-path" "$testdir/root-missing-path/nix-lower-registration/v1" "$name-missing-path" >/dev/null 2>&1; then echo "missing image path was not rejected" >&2 @@ -144,16 +145,20 @@ runCommand "nixmodules-disk-image-registration-check" { mkdir -p "$scratch/nix/store" cp -R "$root/nix/store/." "$scratch/nix/store/" - nix-store --store "$store" --query --valid-paths > "$TMPDIR/valid-before-$name" - test ! -s "$TMPDIR/valid-before-$name" + if nix-store --store "$store" --query --references "$bundle" > "$TMPDIR/valid-before-$name" 2>&1; then + echo "scratch store was not empty before registration import" >&2 + exit 1 + fi nix-store --store "$store" --load-db < "$artifact/registration" - nix-store --store "$store" --query --valid-paths | LC_ALL=C sort > "$TMPDIR/registered-$name" - cmp "$TMPDIR/expected-$name" "$TMPDIR/registered-$name" + while IFS= read -r path; do + nix-store --store "$store" --query --references "$path" >/dev/null + done < "$TMPDIR/expected-$name" nix-store --store "$store" --verify-path "$bundle" nix-store --store "$store" --query --references "$bundle" > "$TMPDIR/references-$name" test -s "$TMPDIR/references-$name" while IFS= read -r path; do grep -Fxq "$path" "$TMPDIR/expected-$name" + nix-store --store "$store" --verify-path "$path" done < "$TMPDIR/references-$name" file="$(find "$scratch$bundle" -type f -print -quit)" diff --git a/pkgs/disk-image-registration-fixture/default.nix b/pkgs/disk-image-registration-fixture/default.nix new file mode 100644 index 00000000..963823c8 --- /dev/null +++ b/pkgs/disk-image-registration-fixture/default.nix @@ -0,0 +1,24 @@ +{ pkgs }: + +let + referencedPath = pkgs.writeText "nixmodules-registration-fixture-reference" "fixture reference\n"; + bundle = pkgs.runCommand "nixmodules-registration-fixture-bundle" { + inherit referencedPath; + } '' + mkdir -p "$out/etc/nixmodules" + printf '%s\n' "$referencedPath" > "$out/etc/nixmodules/reference" + ''; + storeRegistration = (pkgs.callPackage ../store-registration { }) { + rootPaths = [ bundle ]; + }; + image = diskName: pkgs.callPackage ../bundle-image { + inherit bundle storeRegistration diskName; + revstring = "registration-fixture"; + }; +in +pkgs.callPackage ../disk-image-registration-check { + productionImage = image "disk.raw"; + productionBundle = bundle; + developmentImage = image "disk.sqsh"; + developmentBundle = bundle; +} From c8b638e3d8afeeb2cade69f9f3c2c5a2c25f5cca Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 24 Sep 2026 21:05:11 +0000 Subject: [PATCH 3/3] Format Nix registration checks Assisted-by: Replit --- pkgs/disk-image-registration-check/default.nix | 3 ++- pkgs/disk-image-registration-fixture/default.nix | 7 ++++--- pkgs/store-registration/default.nix | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkgs/disk-image-registration-check/default.nix b/pkgs/disk-image-registration-check/default.nix index 66c6d6a9..7458cc49 100644 --- a/pkgs/disk-image-registration-check/default.nix +++ b/pkgs/disk-image-registration-check/default.nix @@ -11,7 +11,8 @@ , developmentBundle }: -runCommand "nixmodules-disk-image-registration-check" { +runCommand "nixmodules-disk-image-registration-check" +{ nativeBuildInputs = [ coreutils diffutils diff --git a/pkgs/disk-image-registration-fixture/default.nix b/pkgs/disk-image-registration-fixture/default.nix index 963823c8..4e35b559 100644 --- a/pkgs/disk-image-registration-fixture/default.nix +++ b/pkgs/disk-image-registration-fixture/default.nix @@ -2,9 +2,10 @@ let referencedPath = pkgs.writeText "nixmodules-registration-fixture-reference" "fixture reference\n"; - bundle = pkgs.runCommand "nixmodules-registration-fixture-bundle" { - inherit referencedPath; - } '' + bundle = pkgs.runCommand "nixmodules-registration-fixture-bundle" + { + inherit referencedPath; + } '' mkdir -p "$out/etc/nixmodules" printf '%s\n' "$referencedPath" > "$out/etc/nixmodules/reference" ''; diff --git a/pkgs/store-registration/default.nix b/pkgs/store-registration/default.nix index c649371d..5e1517f5 100644 --- a/pkgs/store-registration/default.nix +++ b/pkgs/store-registration/default.nix @@ -7,7 +7,8 @@ let closure = closureInfo { inherit rootPaths; }; in -runCommand "nixmodules-store-registration-v1" { +runCommand "nixmodules-store-registration-v1" +{ nativeBuildInputs = [ coreutils ]; } '' mkdir -p "$out"