From a84ce603e55d0344e8628f441203596777701677 Mon Sep 17 00:00:00 2001 From: findalexli Date: Wed, 4 Nov 2026 00:07:31 -0800 Subject: [PATCH] wesnoth_tactical_ai: let top-level Lua files load --- edgebench/tasks/wesnoth_tactical_ai/NOTES.md | 22 ++++++++++++++++--- .../wesnoth_tactical_ai/judge/eval_wesnoth.py | 2 +- edgebench/tasks/wesnoth_tactical_ai/task.toml | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/edgebench/tasks/wesnoth_tactical_ai/NOTES.md b/edgebench/tasks/wesnoth_tactical_ai/NOTES.md index b79014dc..0f5bc901 100644 --- a/edgebench/tasks/wesnoth_tactical_ai/NOTES.md +++ b/edgebench/tasks/wesnoth_tactical_ai/NOTES.md @@ -13,8 +13,8 @@ efficiency = `min(1, baseline_turns / turns)` averaged over wins. The rescale is - No upstream repository: v1 built everything from Debian packages and two inline scripts in its build recipe. - `environment/workspace/my_custom_ai.cfg` and `judge/eval_wesnoth.py`: byte-identical to v1's - inline scripts and to the published v1 images. `judge/grade.py` is new and writes `result.json` - with v1's scoring logic. + inline scripts and to the published v1 images, apart from the 1.1.0 change below. `judge/grade.py` + is new and writes `result.json` with v1's scoring logic. - `instruction.md`: v1's prompt, verbatim. - Wesnoth `1:1.18.5-1` (`wesnoth-1.18`, `wesnoth-1.18-data`) from the Debian snapshot of 2026-06-05, the day v1 installed it. Every file under `/usr` and every package version matches @@ -53,9 +53,25 @@ are within about two standard errors, and the pooled means over 40 runs per side and 74.1 (new). Together with identical files and packages, this is agreement within v1's own noise. +## Changes + +### 1.1.0 + +Top-level `.lua` files now load. The sandbox header ended in `if io then`, and Wesnoth's Lua raises +on an undefined global, so every such file failed at that line, before reaching the submitted code, +and the grade reported nothing. The test is now `if rawget(_G, "io") then`. The rest of the header +is unchanged and runs as before. + +A submission whose config uses a top-level `.lua` file now runs that code, so its score can change +in either direction. There is no mapping, because the old score never ran it. +A submission with no top-level `.lua` file runs the same code as before; its score varies only as +in Known issue 1. + +Source, Deviations and Regression summary describe 1.0.0 unless they say otherwise. + ## Known issues -Kept as in v1, so scores stay comparable. Fixing any of them changes scores and bumps `version`. +Not fixed. Fixing any of them changes scores and bumps `version`. 1. **Games are not reproducible, so the score is noisy.** The same map and seed do not replay the same game, and the two repeats of a pair often disagree. A ~70-point AI has a run-to-run diff --git a/edgebench/tasks/wesnoth_tactical_ai/judge/eval_wesnoth.py b/edgebench/tasks/wesnoth_tactical_ai/judge/eval_wesnoth.py index 0621204c..10a4410a 100644 --- a/edgebench/tasks/wesnoth_tactical_ai/judge/eval_wesnoth.py +++ b/edgebench/tasks/wesnoth_tactical_ai/judge/eval_wesnoth.py @@ -53,7 +53,7 @@ if os and os.execute then os.execute = function(...) error("SE-Bench: os.execute is forbidden") end end -if io then +if rawget(_G, "io") then io.open = function(...) error("SE-Bench: io.open is forbidden") end end """ diff --git a/edgebench/tasks/wesnoth_tactical_ai/task.toml b/edgebench/tasks/wesnoth_tactical_ai/task.toml index 4d9e52b2..20ce0fe3 100644 --- a/edgebench/tasks/wesnoth_tactical_ai/task.toml +++ b/edgebench/tasks/wesnoth_tactical_ai/task.toml @@ -4,7 +4,7 @@ name = "Wesnoth Tactical Ai" category = "Interactive Games & Simulators" language = "Python" description = "Write tactical AI logic for Battle for Wesnoth that defeats the built-in AI through custom recruitment, focus-fire targeting, terrain exploitation, and village-capture timing across multiple maps." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment]