fix(ci): name the failed Fix Exporter verify check and re-read a lagging artifact listing

.github/tests/fixtures/prepare-fixes-runner.sh

@@ -46,31 +46,60 @@ GIT
     fi
     ;;
   verify)
-    [ ! -e .devantler-tech-actions ]
-    [ "$ACTUAL_ARTIFACT" = "$ARTIFACT" ]
-    [ "$ACTUAL_ELIGIBLE" = "$UPLOAD" ]
+    # Name every failed check with its expected and actual values: a bare test
+    # under errexit only reports "exit code 1".
+    expect() {
+      if [ "$2" != "$3" ]; then
+        printf 'FAIL hosted fix exporter (%s): %s: expected %s, got %s\n' \
+          "$SCENARIO" "$1" "${2:-<empty>}" "${3:-<empty>}" >&2
+        exit 1
+      fi
+    }
+    helper=absent
+    [ ! -e .devantler-tech-actions ] || helper=present
+    expect 'helper checkout' absent "$helper"
+    expect 'artifact name' "$ARTIFACT" "$ACTUAL_ARTIFACT"
+    expect 'upload eligibility' "$UPLOAD" "$ACTUAL_ELIGIBLE"
     expected_changed=true
     [ "$SCENARIO" != clean ] || expected_changed=false
-    [ "$ACTUAL_CHANGED" = "$expected_changed" ]
+    expect 'changed output' "$expected_changed" "$ACTUAL_CHANGED"
     if [ "$SCENARIO" = export-failure ]; then
-      [ "$ACTUAL_OUTCOME" = failure ]
-      [ -z "$ACTUAL_MANUAL" ]
+      expect 'step outcome' failure "$ACTUAL_OUTCOME"
+      expect 'manual-required output' '' "$ACTUAL_MANUAL"
     else
-      [ "$ACTUAL_OUTCOME" = success ]
-      [ "$ACTUAL_MANUAL" = "$MANUAL" ]
+      expect 'step outcome' success "$ACTUAL_OUTCOME"
+      expect 'manual-required output' "$MANUAL" "$ACTUAL_MANUAL"
     fi
     expected_artifacts=0
     if [ "$UPLOAD" = true ] && [ "$expected_changed" = true ]; then
       expected_artifacts=1
       cmp "$RUNNER_TEMP/$ARTIFACT.patch" "$RUNNER_TEMP/download/$ARTIFACT.patch"
       git -C "$RUNNER_TEMP/replay" apply --index "$RUNNER_TEMP/download/$ARTIFACT.patch"
-      [ "$(git -C "$RUNNER_TEMP/replay" write-tree)" = "$(cat "$RUNNER_TEMP/expected-tree")" ]
+      expect 'replayed tree' "$(cat "$RUNNER_TEMP/expected-tree")" "$(git -C "$RUNNER_TEMP/replay" write-tree)"
     fi
     # Check real artifact absence as well as successful transfers. A skipped
-    # download alone cannot establish that the upload was suppressed.
-    gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate > "$RUNNER_TEMP/artifacts.json"
-    count="$(jq -s --arg name "$ARTIFACT" '[.[].artifacts[] | select(.name == $name)] | length' "$RUNNER_TEMP/artifacts.json")"
-    [ "$count" = "$expected_artifacts" ]
+    # download alone cannot establish that the upload was suppressed. The run's
+    # artifact listing can lag a finished upload by a few seconds, so re-read it
+    # (bounded) while it shows fewer than expected; the count must still match
+    # exactly, so a missing or extra artifact fails.
+    list_artifacts() {
+      gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate > "$RUNNER_TEMP/artifacts.json"
+    }
+    count_artifacts() {
+      jq -s --arg name "$ARTIFACT" '[.[].artifacts[] | select(.name == $name)] | length' "$RUNNER_TEMP/artifacts.json"
+    }
+    attempts=6
+    delay="${FIXTURE_ARTIFACT_LIST_DELAY:-5}"
+    attempt=1
+    list_artifacts
+    count="$(count_artifacts)"
+    while [ "$count" -lt "$expected_artifacts" ] && [ "$attempt" -lt "$attempts" ]; do
+      sleep "$delay"
+      attempt=$((attempt + 1))
+      list_artifacts
+      count="$(count_artifacts)"
+    done
+    expect 'artifact count' "$expected_artifacts" "$count"
     printf 'PASS hosted fix exporter: %s; artifact count=%s; outcome=%s\n' "$SCENARIO" "$count" "$ACTUAL_OUTCOME"
     ;;
   *) echo 'unknown fixture phase' >&2; exit 1 ;;

.github/tests/test-lint-workflow-fixes.sh

@@ -250,3 +250,86 @@ for ablated in false true; do
   fi
   echo "PASS: native hosted exporter; cleanup ablated=$ablated"
 done
+
+# Exercise the hosted verify phase against a stubbed artifact listing. A failed
+# assertion must name the check with its expected and actual values, and a
+# listing that has not caught up yet must be re-read, without relaxing the exact
+# count: a missing or extra artifact still fails.
+runner_fixture="$root/.github/tests/fixtures/prepare-fixes-runner.sh"
+mkdir -p "$work/verify-bin"
+cat >"$work/verify-bin/gh" <<'GH'
+#!/usr/bin/env bash
+# Answer each artifact listing with the next count from VERIFY_COUNTS.
+calls=$(( $(cat "$VERIFY_STATE/calls" 2>/dev/null || echo 0) + 1 ))
+echo "$calls" >"$VERIFY_STATE/calls"
+read -r -a counts <<<"$VERIFY_COUNTS"
+index=$(( calls <= ${#counts[@]} ? calls - 1 : ${#counts[@]} - 1 ))
+jq -n --arg name "$ARTIFACT" --argjson n "${counts[$index]}" \
+  '{artifacts: ([range($n) | {name: $name}] + [{name: "unrelated"}])}'
+GH
+chmod +x "$work/verify-bin/gh"
+
+# Run verify for one hosted scenario. Remaining arguments are NAME=value overrides
+# of the step outputs. Upload scenarios get a real patch in both transfer paths.
+run_verify() {
+  local name="$1" scenario="$2" upload="$3" counts="$4"
+  shift 4
+  local case_dir="$work/verify-$name"
+  mkdir -p "$case_dir/consumer" "$case_dir/runner/download" "$case_dir/state"
+  local artifact="prepare-fixes-test-$name"
+  (
+    cd "$case_dir/consumer"
+    SCENARIO="$scenario" MANUAL=false RUNNER_TEMP="$case_dir/runner" GITHUB_PATH="$case_dir/path" \
+      bash "$runner_fixture" setup >/dev/null
+    if [[ "$upload" == true && "$scenario" != clean ]]; then
+      git add -- notes.txt new-file.txt binary.bin executable.sh
+      git diff --cached --no-color --binary --full-index HEAD >"$case_dir/runner/$artifact.patch"
+      cp "$case_dir/runner/$artifact.patch" "$case_dir/runner/download/$artifact.patch"
+    fi
+  )
+  local changed=true
+  [[ "$scenario" != clean ]] || changed=false
+  (
+    cd "$case_dir/consumer"
+    env PATH="$work/verify-bin:$PATH" VERIFY_STATE="$case_dir/state" VERIFY_COUNTS="$counts" \
+      SCENARIO="$scenario" MANUAL=false UPLOAD="$upload" ARTIFACT="$artifact" \
+      RUNNER_TEMP="$case_dir/runner" GITHUB_REPOSITORY=example/fixture GITHUB_RUN_ID=1 \
+      FIXTURE_ARTIFACT_LIST_DELAY=0 \
+      ACTUAL_ARTIFACT="$artifact" ACTUAL_ELIGIBLE="$upload" ACTUAL_CHANGED="$changed" \
+      ACTUAL_OUTCOME=success ACTUAL_MANUAL=false "$@" \
+      bash "$runner_fixture" verify
+  ) >"$case_dir/log" 2>&1
+}
+verify_calls() { cat "$work/verify-$1/state/calls" 2>/dev/null || echo 0; }
+
+run_verify pass ordinary true 1 || fail "verify rejected a correct upload: $(cat "$work/verify-pass/log")"
+grep -Fq 'artifact count=1' "$work/verify-pass/log" || fail 'verify did not report the artifact count'
+
+run_verify lagging ordinary true '0 0 1' ||
+  fail "verify did not re-read a listing that had not caught up: $(cat "$work/verify-lagging/log")"
+[[ "$(verify_calls lagging)" == 3 ]] || fail "lagging listing read $(verify_calls lagging) times, want 3"
+
+if run_verify missing ordinary true 0; then fail 'verify accepted a missing artifact'; fi
+grep -Fq 'artifact count: expected 1, got 0' "$work/verify-missing/log" ||
+  fail "missing artifact did not name the count: $(cat "$work/verify-missing/log")"
+[[ "$(verify_calls missing)" -gt 1 ]] || fail 'missing artifact was not re-read before failing'
+
+if run_verify extra ordinary true 2; then fail 'verify accepted an extra artifact'; fi
+grep -Fq 'artifact count: expected 1, got 2' "$work/verify-extra/log" ||
+  fail "extra artifact did not name the count: $(cat "$work/verify-extra/log")"
+[[ "$(verify_calls extra)" == 1 ]] || fail 'an extra artifact must fail without waiting'
+
+run_verify absent read-only false 0 || fail "verify rejected a suppressed upload: $(cat "$work/verify-absent/log")"
+if run_verify unexpected read-only false 1; then fail 'verify accepted an artifact from a suppressed upload'; fi
+grep -Fq 'artifact count: expected 0, got 1' "$work/verify-unexpected/log" ||
+  fail "unexpected artifact did not name the count: $(cat "$work/verify-unexpected/log")"
+
+if run_verify outcome clean true 0 ACTUAL_OUTCOME=failure; then fail 'verify accepted a failed step'; fi
+grep -Fq 'step outcome: expected success, got failure' "$work/verify-outcome/log" ||
+  fail "failed step did not name the outcome: $(cat "$work/verify-outcome/log")"
+
+if run_verify name clean true 0 ACTUAL_ARTIFACT=; then fail 'verify accepted a missing artifact name'; fi
+grep -Fq 'artifact name: expected prepare-fixes-test-name, got <empty>' "$work/verify-name/log" ||
+  fail "missing artifact name was not reported: $(cat "$work/verify-name/log")"
+
+echo 'PASS: hosted verify names failed checks and re-reads a lagging artifact listing'