diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md b/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md index 62ba7ca8..72471a97 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md @@ -12,7 +12,7 @@ fails (maximize). The rescale is the v1 one (`log_anchor`, anchor 14.155 -> 43). - Workspace files, scaffolds and the judge's `tests/` (`test.sh`, `compute_reward.py`): the task authors' copy of the public frontier-swe task `ffmpeg-swscale-rewrite`, identical by sha256 to - the published v1 images. The Rust scaffold `lib.rs`, the Zig `build.zig` fix and + the published v1 images at migration. Subsequent changes are listed below. The Rust scaffold `lib.rs`, the Zig `build.zig` fix and `pytest_shim.sh` come from v1's build recipe, also byte-identical. - `judge/eval.sh`: v1's eval command, wrapped exactly as the v1 harness wrapped it. `judge/run.sh` and `judge/grade.py` are new and write `result.json` with v1's parsing. @@ -61,6 +61,43 @@ Every gate outcome, pass count and summary matches. Speedups are wall-clock rati up to ±25% between runs of the old judge alone; the mean difference is under 0.5 points on 0..100 for every row, well inside that spread. +## Changes + +### 1.0.1 — distinguish verifier failure stages + +Failed submissions now receive a bounded stage description in `result.json`'s summary and +existing failed overall-score detail. The producing verifier branch records the stage; +`grade.py` reads it from `reward.json` and emits only a fixed public message. Source-policy, +build, library-policy, library-loading, baseline, correctness and verifier failures are distinct. +Raw reasons, submitted paths, exceptions and individual workload data are not forwarded. +Unknown, missing, malformed or contradictory diagnostic evidence leaves the original feedback. + +This changes feedback, not the numeric score, validity, number of tests, source-scan rules, +correctness thresholds, benchmark workloads or rescale. It does not certify that a source-policy +rejection was justified. In particular, policy strings inside research notes remain subject to +the existing scan; exclude research notes from the submitted implementation. The legacy eval +script's richer failure-category hints remain broken (Known issue 2); the new stage feedback +uses the structured reward path and does not depend on those hints. + +Same-host regression for 1.0.1 used the published 1.0.0 amd64 judge +`e938bf6f485e` and an image derived from its immutable digest with only the four changed +judge files replaced. Both used 4 CPUs, 8 GiB and no network. No historical solver +submission is required to reproduce these checks. + +| Submission | 1.0.0 raw score | 1.0.1 raw score | New zero-score feedback | +|---|---|---|---| +| empty | 0 | 0 | build | +| research note rejected by the existing scan | 0 | 0 | source policy | +| failing Makefile | 0 | 0 | build | +| library returning a null conversion context | 0 | 0 | correctness | +| original Rust scaffold, repetition 1 | 0.364605 | 0.371320 | unchanged passing result | +| original Rust scaffold, repetition 2 | 0.371796 | 0.360566 | unchanged passing result | + +All zero-score cases remain valid with one failed test. Both scaffold repetitions remain +valid with one passed test and 30/30 correctness; the old/new speed ranges overlap. +These runs check score compatibility and diagnosis delivery, not solver improvement or +statistical performance equivalence. The scorer's speed-measurement code is unchanged. + ## Known issues Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/grade.py b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/grade.py index 996a66ec..3a9d154d 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/grade.py +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/grade.py @@ -6,7 +6,7 @@ The test output is the text between the START/END markers (extract_test_output); in this task the END marker is never printed (see NOTES.md, Known issue 2), so it is the whole output. -Usage: grade.py +Usage: grade.py [reward.json path] """ import json import re @@ -14,6 +14,36 @@ START, END = ">>>>> Start Test Output", ">>>>> End Test Output" +# Task-local stages are recorded by the failing verifier branch. Never infer them +# from build output or forward free-form reasons (paths, workload data, exceptions). +FAILURE_MESSAGES = { + "source_policy": "Source policy check failed before compilation; pixel correctness and speed were not evaluated.", + "build": "The candidate did not build or produce the required shared library; pixel correctness and speed were not evaluated.", + "artifact_policy": "The compiled candidate failed the library policy check; pixel correctness and speed were not evaluated.", + "candidate_load": "The candidate shared library could not be loaded; pixel correctness and speed were not evaluated.", + "baseline": "The verifier baseline was unavailable or failed its integrity check; no candidate quality conclusion is available.", + "correctness": "Pixel correctness checks failed; the performance benchmark was not run.", + "verifier": "The verifier did not complete; no candidate quality conclusion is available.", +} + + +def failure_diagnostic(reward_path: str | None, score: float | None) -> dict: + """Optional bounded feedback; unavailable or contradictory evidence stays absent.""" + if reward_path is None or score != 0: + return {} + try: + with open(reward_path, encoding="utf-8") as f: + reward = json.load(f) + except (OSError, ValueError): + return {} + if not isinstance(reward, dict) or type(reward.get("score")) not in (int, float) or reward["score"] != 0: + return {} + additional = reward.get("additional_data") + stage = additional.get("failure_stage") if isinstance(additional, dict) else None + if not isinstance(stage, str) or stage not in FAILURE_MESSAGES: + return {} + return {"failure_stage": stage, "message": FAILURE_MESSAGES[stage]} + def extract_test_output(raw: str) -> str: start_idx, found = raw.find("\n" + START), False @@ -90,6 +120,13 @@ def main() -> int: summary += f" (+{len(bad) - 10} more)" m = re.search(r"TOTAL_SCORE\s+(inf|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)", out) score = float(m.group(1)) if m else None + diagnostic = failure_diagnostic(sys.argv[3] if len(sys.argv) > 3 else None, score) + if diagnostic: + for test in tests: + if test["name"] == "tests/eval.py::overall_score" and test["status"] == "FAILED": + test.update(diagnostic) + summary += ". " + diagnostic["message"] + break metrics = {} r = re.search(r"Correctness: (\d+)/(\d+) workloads passed", out) if r: diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/run.sh b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/run.sh index 4537a335..5240f2bf 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/run.sh +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/run.sh @@ -7,5 +7,5 @@ cd /home/workspace/swscale-impl /bin/bash /judge/eval.sh > "$OUT" 2>&1 cat "$OUT" mkdir -p /opt/sforge -python3 -I /judge/grade.py "$OUT" /opt/sforge/result.json +python3 -I /judge/grade.py "$OUT" /opt/sforge/result.json /logs/verifier/reward.json rm -f "$OUT" diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/compute_reward.py b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/compute_reward.py index 798196d0..fa696199 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/compute_reward.py +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/compute_reward.py @@ -52,6 +52,7 @@ def parse_args(): parser.add_argument("--total-time-ms", type=int, default=0) parser.add_argument("--oracle", action="store_true") parser.add_argument("--fail", type=str, default=None) + parser.add_argument("--fail-stage", choices=("baseline", "source_policy", "build", "artifact_policy")) return parser.parse_args() @@ -62,6 +63,8 @@ def emit_reward( total_time_ms: int, subscores: list[dict] | None = None, additional_data: dict | None = None, + *, + failure_stage: str | None = None, ) -> None: payload = { "score": score, @@ -73,6 +76,8 @@ def emit_reward( "total_time_ms": total_time_ms, }, } + if failure_stage is not None: + payload["additional_data"]["failure_stage"] = failure_stage out_dir = Path(output_dir) out_dir.mkdir(parents=True, exist_ok=True) with open(out_dir / "reward.json", "w") as f: @@ -577,7 +582,8 @@ def main(): # Quick-fail mode (called from test.sh on integrity failures) if args.fail: - emit_reward(args.output_dir, 0.0, f"HARD FAIL: {args.fail}", args.total_time_ms) + emit_reward(args.output_dir, 0.0, f"HARD FAIL: {args.fail}", args.total_time_ms, + failure_stage=args.fail_stage) return 0 # Load libraries @@ -585,7 +591,8 @@ def main(): candidate_lib = load_swscale_library(args.candidate) except Exception as e: emit_reward( - args.output_dir, 0.0, f"Failed to load candidate: {e}", args.total_time_ms + args.output_dir, 0.0, f"Failed to load candidate: {e}", args.total_time_ms, + failure_stage="candidate_load" ) return 0 @@ -593,7 +600,8 @@ def main(): baseline_lib = load_swscale_library(args.baseline) except Exception as e: emit_reward( - args.output_dir, 0.0, f"Failed to load baseline: {e}", args.total_time_ms + args.output_dir, 0.0, f"Failed to load baseline: {e}", args.total_time_ms, + failure_stage="baseline" ) return 0 @@ -607,7 +615,8 @@ def main(): import traceback traceback.print_exc() - emit_reward(args.output_dir, 0.0, f"Verifier crashed: {e}", args.total_time_ms) + emit_reward(args.output_dir, 0.0, f"Verifier crashed: {e}", args.total_time_ms, + failure_stage="verifier") return 0 @@ -637,6 +646,7 @@ def _run_scoring(args, candidate_lib, baseline_lib, rng): 0.0, f"Correctness failed: {passed}/{total} workloads passed", args.total_time_ms, + failure_stage="correctness", subscores=[ { "name": "correctness", diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/test.sh b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/test.sh index 2dc940b1..115426ba 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/test.sh +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/judge/tests/test.sh @@ -55,7 +55,7 @@ echo "Step 2: Checking baseline integrity..." BASELINE_LIB="/verifier-data/libswscale_baseline.so" if [ ! -f "$BASELINE_LIB" ]; then "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "baseline library missing" \ + --fail "baseline library missing" --fail-stage baseline \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -65,7 +65,7 @@ EXPECTED_HASH=$(cat /verifier-data/baseline_hash.txt 2>/dev/null | tr -d '[:spac ACTUAL_HASH=$(sha256sum "$BASELINE_LIB" | awk '{print $1}') if [ "${EXPECTED_HASH}" != "${ACTUAL_HASH}" ]; then "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "baseline library integrity check failed" \ + --fail "baseline library integrity check failed" --fail-stage baseline \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -112,7 +112,7 @@ done < <(find "${IMPL_DIR}" \ -type f \( -name "*.zig" -o -name "*.rs" -o -name "*.c" -o -name "*.h" -o -name "*.cc" -o -name "*.cpp" -o -name "*.hpp" -o -name "*.inc" -o -name "*.toml" -o -name "*.py" -o -name "*.sh" -o -name "*.json" -o -name "*.txt" -o -name "*.cfg" -o -name "*.ld" -o -name "*.S" -o -name "*.asm" -o -name "Makefile" -o -name "CMakeLists.txt" -o -name "*.mk" -o -name "*.cmake" \) -print0 2>/dev/null) if [ "$SCAN_FAILED" = "true" ]; then "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "source code references verifier internals or wraps FFmpeg" \ + --fail "source code references verifier internals or wraps FFmpeg" --fail-stage source_policy \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -129,7 +129,7 @@ CANDIDATE_LIB="" if [ ! -d "${IMPL_DIR}" ]; then echo "FAIL: implementation directory ${IMPL_DIR} does not exist" "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "implementation directory not found" \ + --fail "implementation directory not found" --fail-stage build \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -201,7 +201,7 @@ fi if [ "$BUILD_OK" != "true" ] || [ -z "$CANDIDATE_LIB" ]; then echo "FAIL: build failed or no libswscale_candidate.so produced" "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "build failed or library not found" \ + --fail "build failed or library not found" --fail-stage build \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -217,7 +217,7 @@ if [ "$IS_ORACLE" != "true" ]; then if ldd "$CANDIDATE_LIB" 2>/dev/null | grep -qiE 'libswscale|libavutil|libavcodec|libavformat|libswresample'; then echo "FAIL: candidate dynamically links against FFmpeg libraries" "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "candidate dynamically links FFmpeg (wrapper detected)" \ + --fail "candidate dynamically links FFmpeg (wrapper detected)" --fail-stage artifact_policy \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -227,7 +227,7 @@ if [ "$IS_ORACLE" != "true" ]; then if nm -D "$CANDIDATE_LIB" 2>/dev/null | grep -qiE '\bsws_init|\bsws_get|\bsws_free|\bsws_scale|\bav_image|\bav_pix_fmt|\bavutil_'; then echo "FAIL: candidate exports/imports FFmpeg symbols (static linking detected)" "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "candidate contains FFmpeg symbols (static link detected)" \ + --fail "candidate contains FFmpeg symbols (static link detected)" --fail-stage artifact_policy \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -240,7 +240,7 @@ if [ "$IS_ORACLE" != "true" ]; then if [ -n "$CANDIDATE_HASH" ] && [ "$CANDIDATE_HASH" = "$BASELINE_HASH_VAL" ]; then echo "FAIL: candidate is identical to baseline (copy detected)" "${PY_RUN[@]}" "${SCRIPT_DIR}/compute_reward.py" \ - --fail "candidate is a copy of the baseline library" \ + --fail "candidate is a copy of the baseline library" --fail-stage artifact_policy \ --total-time-ms "$(( $(python3 -c "import time; print(int(time.time()*1000))") - HARBOR_START_MS ))" \ --output-dir "$VERIFIER_DIR" exit 0 @@ -273,7 +273,7 @@ HARBOR_TOTAL_MS=$(( HARBOR_END_MS - HARBOR_START_MS )) # Fallback: if compute_reward.py crashed without writing reward, emit 0 if [ ! -f "$VERIFIER_DIR/reward.json" ]; then - echo '{"reward": 0.0, "score": 0.0, "subscores": [], "additional_data": {"reason": "verifier crashed"}}' \ + echo '{"reward": 0.0, "score": 0.0, "subscores": [], "additional_data": {"reason": "verifier crashed", "failure_stage": "verifier"}}' \ > "$VERIFIER_DIR/reward.json" echo "0.0" > "$VERIFIER_DIR/reward.txt" fi diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml b/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml index cf690f08..0bc491c8 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml @@ -4,7 +4,7 @@ name = "Ffmpeg Swscale Reimplementation" category = "Systems & Software Engineering" language = "C++" # as in v1 tasks.jsonl; the deliverable is Rust (or Zig), see NOTES.md description = "Reimplement FFmpeg's libswscale pixel-format conversion and scaling library in Rust, handling multiple pixel formats and scaling algorithms. A correctness-passing scaffold is provided; the agent must optimize for speed via SIMD." -version = "1.0.0" +version = "1.0.1" status = "migrated" [environment] diff --git a/tests/test_ffmpeg_feedback.py b/tests/test_ffmpeg_feedback.py new file mode 100644 index 00000000..b0ced5be --- /dev/null +++ b/tests/test_ffmpeg_feedback.py @@ -0,0 +1,123 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise the task's reward -> shim -> grade -> public result boundary.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from sforge.harness.grading import parse_result + +TASK = Path(__file__).parents[1] / "edgebench/tasks/ffmpeg_swscale_reimplementation" +JUDGE = TASK / "judge" + + +def grade(tmp_path, payload, *, log_prefix="", supplied=True): + reward = tmp_path / "reward.json" + reward.write_text(json.dumps(payload)) + output = subprocess.check_output( + ["bash", str(JUDGE / "tests/pytest_shim.sh"), str(reward)], text=True, + ) + log = tmp_path / "eval.log" + log.write_text(log_prefix + output) + result = tmp_path / "result.json" + subprocess.run( + [sys.executable, "-I", str(JUDGE / "grade.py"), str(log), str(result)] + + ([str(reward)] if supplied else []), check=True, capture_output=True, + ) + return parse_result(result.read_text()) + + +@pytest.mark.parametrize("stage", ["source_policy", "build", "artifact_policy", "baseline"]) +def test_producer_failure_reaches_public_result_without_extra_failed_test(tmp_path, stage): + env = dict(os.environ, PYTHONPATH=str(TASK / "environment/workspace"), PYTHONDONTWRITEBYTECODE="1") + subprocess.run([ + sys.executable, str(JUDGE / "tests/compute_reward.py"), + "--fail", "private path /private/hidden-workload: do not expose", + "--fail-stage", stage, "--output-dir", str(tmp_path), + ], env=env, check=True, capture_output=True) + payload = json.loads((tmp_path / "reward.json").read_text()) + old_score, old_valid, old = grade(tmp_path, payload, supplied=False) + score, valid, result = grade(tmp_path, payload) + assert (score, valid) == (old_score, old_valid) == (0.0, True) + assert len(result["details"]) == len(old["details"]) == 1 + assert result["details"][0]["failure_stage"] == stage + assert result["details"][0]["message"] in result["summary"] + assert "private" not in json.dumps(result) + assert result["metrics"] == old["metrics"] + + +def test_candidate_load_failure_has_its_own_stage(tmp_path): + env = dict(os.environ, PYTHONPATH=str(TASK / "environment/workspace"), PYTHONDONTWRITEBYTECODE="1") + subprocess.run([ + sys.executable, str(JUDGE / "tests/compute_reward.py"), + "--candidate", str(tmp_path / "missing.so"), "--output-dir", str(tmp_path), + ], env=env, check=True, capture_output=True) + _, _, result = grade(tmp_path, json.loads((tmp_path / "reward.json").read_text())) + assert result["details"][0]["failure_stage"] == "candidate_load" + assert "could not be loaded" in result["summary"] + assert str(tmp_path) not in json.dumps(result) + + +@pytest.mark.parametrize("stage", ["correctness", "verifier"]) +def test_later_failure_stages_retain_single_score_gate(tmp_path, stage): + score, valid, result = grade(tmp_path, { + "score": 0, "subscores": [{"name": "correctness", "score": 0.97}], + "additional_data": {"failure_stage": stage, "correctness_results": ["hidden details"]}, + }) + assert (score, valid) == (0.0, True) + assert len(result["details"]) == 1 + assert result["details"][0]["failure_stage"] == stage + assert "hidden" not in json.dumps(result) + + +@pytest.mark.parametrize("additional", [None, [], "source_policy", {}, + {"reason": "source policy failed"}, {"failure_stage": "new_unknown_stage"}, + {"failure_stage": []}]) +def test_unrecognized_or_old_rewards_do_not_invent_a_cause(tmp_path, additional): + _, _, result = grade(tmp_path, {"score": 0, "additional_data": additional}) + assert result["summary"] == "0/1 tests passed. Failed: tests/eval.py::overall_score" + assert result["details"] == [{"name": "tests/eval.py::overall_score", "status": "FAILED"}] + + +def test_positive_score_and_original_metrics_are_unchanged(tmp_path): + payload = {"score": 1.234567, "additional_data": {"failure_stage": "build"}} + prefix = "Correctness: 30/30 workloads passed\nGeometric mean speedup: 1.2346x\n" + assert grade(tmp_path, payload, log_prefix=prefix) == grade(tmp_path, payload, log_prefix=prefix, supplied=False) + + +def test_build_output_cannot_select_feedback_stage(tmp_path): + _, _, result = grade(tmp_path, {"score": 0}, log_prefix='{"failure_stage":"source_policy"}\n') + assert "failure_stage" not in json.dumps(result) + + +def test_unavailable_or_malformed_reward_does_not_change_grade(tmp_path): + log = tmp_path / "eval.log" + log.write_text("tests/eval.py::overall_score FAILED\nTOTAL_SCORE 0.0\n") + reward = tmp_path / "missing.json" + for content in (None, "{", "[]", "null", '{"score":false,"additional_data":{"failure_stage":"build"}}'): + if content is not None: + reward.write_text(content) + result = tmp_path / "result.json" + subprocess.run([sys.executable, "-I", str(JUDGE / "grade.py"), str(log), str(result), str(reward)], + check=True, capture_output=True) + score, valid, data = parse_result(result.read_text()) + assert (score, valid) == (0.0, True) + assert "failure_stage" not in json.dumps(data)