diff --git a/dokimi/changelog.d/20261117_120000_leftovers-cache-dir-help-exit-2.md b/dokimi/changelog.d/20261117_120000_leftovers-cache-dir-help-exit-2.md new file mode 100644 index 000000000..7a2b451d2 --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_leftovers-cache-dir-help-exit-2.md @@ -0,0 +1,3 @@ +### Fixed + +- **The `repo run`, `eval check` and `tasks generate` help, the repository-evaluation contract and the quickstart list a `--cache-dir` that is not a usable directory among the exit-2 cases**, as the commands have refused it since #828; the help texts of the three commands are pinned by a test. diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index d836bdfe8..6bcea0af0 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -265,7 +265,7 @@ failed check. A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and -`eval check` exit 2 for a source that does not parse or a pinned local path inside a repository, +`eval check` exit 2 for a source that does not parse, a pinned local path inside a repository or a `--cache-dir` that is not a usable directory, but still exit 1 for other input refusals (a missing or malformed manifest, a bad arm; the `✗` lines; a `--manifest` path that leaves the source tree as typed is exit 2, pinned or not, as a source refusal; a link in the tree that leads out of it is exit 1), and `tools preference` exits 1 when every call failed and 3 when `--max-cost-usd` diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 7d9404768..3c0a30399 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -699,7 +699,7 @@ apart, neither failed nor missing. If the probe itself fails, a task with cells is left unrun; a cell that already has a `result.json` keeps it. Both commands exit 2, before any build, on a source that does not parse (a local path that is not a -directory, an unpinned remote) and on a pinned local path inside a repository that is not its root; a +directory, an unpinned remote), on a pinned local path inside a repository that is not its root and on a `--cache-dir` that is not a usable directory; a source that parses but cannot be opened (clone, fetch, revision) exits 1 with a `✗` line. A manifest that is missing or malformed exits 1 too; a `--manifest` path that leaves the source tree is refused as a source and exits 2, for an unpinned local directory and for a pinned source (`@`) alike; a link in the tree that leads out of it, the default @@ -1043,8 +1043,8 @@ as it is, so fetch those refs first, for example whose revision is still missing is listed and left out. The command exits 2, before it reads the threads file, on a malformed source, -an unpinned remote, a local source that is not a directory, or a pinned local path inside a repository -that is not its root (`/sub@`: give the root). It exits 1 when the threads file cannot be read or holds no pull +an unpinned remote, a local source that is not a directory, a pinned local path inside a repository +that is not its root (`/sub@`: give the root), or a `--cache-dir` that is not a usable directory. It exits 1 when the threads file cannot be read or holds no pull request, when no pull request gave a task (written, or kept from before), and on an error that is not one pull request's own: a malformed pull request, a source it cannot open, a manifest that is missing or malformed, the prompt file, or a task it cannot write. Each prints @@ -1140,7 +1140,7 @@ is F1 against `gold`. A gold file with an unknown `answer_key`, an empty error. The command exits 2 without `--index`, with `--references 0 --derives 0`, on -an unpinned source, an index that does not read, or one that maps to no file +an unpinned source, a `--cache-dir` that is not a usable directory, an index that does not read, or one that maps to no file of the tree; 1 when no target gave a task, a task cannot be written, or the source cannot be resolved at the revision. ### 4.8 Authoring tasks from merged pull requests @@ -1249,8 +1249,8 @@ and `headRefOid` it lacks from `origin`; a local repository is read as it is, so The command prints one line per pull request (`written`, `kept` or `skipped`, with the reason or the counts) and a summary. It exits 2, before it reads the pull request file, on a -malformed source, an unpinned remote, a local source that is not a directory, or a pinned local path inside -a repository that is not its root (`/sub@`: give the root). It exits 1 when the pull request file cannot be read or +malformed source, an unpinned remote, a local source that is not a directory, a pinned local path inside +a repository that is not its root (`/sub@`: give the root), or a `--cache-dir` that is not a usable directory. It exits 1 when the pull request file cannot be read or holds no pull request, when no pull request gave a task (written, or kept from before), and on an error that is not one pull request's own: a malformed pull request (a `number` that is not an integer), a source it cannot open, a manifest that is missing or malformed, the prompt file (unreadable or empty), or a task it diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 7c3a04035..a0b1b3c86 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -485,8 +485,8 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit _SOURCE_EXIT = ( - "Exit 2 for a source that does not parse (a path that is not a directory, an unpinned remote) or a " - "pinned local path inside a repository (give its root); exit 1 for a source that cannot be opened " + "Exit 2 for a source that does not parse (a path that is not a directory, an unpinned remote), a " + "pinned local path inside a repository (give its root) or a --cache-dir that is not a usable directory; exit 1 for a source that cannot be opened " "(clone, fetch, revision) and " ) CHECK_EXIT = _SOURCE_EXIT + "for a task that fails the check." diff --git a/dokimi/dokimi/commands/tasks_generate.py b/dokimi/dokimi/commands/tasks_generate.py index 1555ef8a8..ae1543637 100644 --- a/dokimi/dokimi/commands/tasks_generate.py +++ b/dokimi/dokimi/commands/tasks_generate.py @@ -162,8 +162,8 @@ def add_parser(eval_tasks_sub, positive_int) -> None: "from a SCIP code index; --from replay writes one dokimi:review_replay task per merged pull request of a " "threads file; --from prs writes one dokimi:hidden_tests task per merged pull request of a pull request " "file (its description as the prompt, its test files hidden until grading, the rest of its diff as the " - "oracle). Exits 2 on a usage error, among them a malformed source, an unpinned remote or a local source " - "that is not a directory. With graph, also 2 on an index that does not read or matches no file " + "oracle). Exits 2 on a usage error, among them a malformed source, an unpinned remote, a local source " + "that is not a directory or a --cache-dir that is not a usable directory. With graph, also 2 on an index that does not read or matches no file " "of the source, and 1 when no target of the index gave a task, the tasks cannot be written or the source " "cannot be resolved at the revision. With replay or prs, 1 when no pull request gave a task, a task cannot " "be written, the source cannot be opened, or the threads or pull request file, the manifest or the prompt " diff --git a/dokimi/tests/test_cli_help.py b/dokimi/tests/test_cli_help.py index 9918d4884..475e35138 100644 --- a/dokimi/tests/test_cli_help.py +++ b/dokimi/tests/test_cli_help.py @@ -47,3 +47,11 @@ def test_help_renders(path, capsys): out = capsys.readouterr().out assert out.startswith("usage: ") assert " ".join(path) in out.splitlines()[0] + + +@pytest.mark.parametrize("path", [("eval", "check"), ("repo", "run"), ("eval", "tasks")], + ids=["eval-check", "repo-run", "tasks-generate-in-the-tasks-listing"]) +def test_the_exit_2_help_of_a_source_command_names_the_cache_dir_case(path, capsys): + with pytest.raises(SystemExit): + build_parser().parse_args([*path, "--help"]) + assert "a --cache-dir that is not a usable directory" in " ".join(capsys.readouterr().out.split()) diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py index 5a7236e4c..46a51cd5d 100644 --- a/dokimi/tests/test_repo_report_unreadable_cell.py +++ b/dokimi/tests/test_repo_report_unreadable_cell.py @@ -11,7 +11,7 @@ import pytest from dokimi.cli import build_parser -from dokimi.repos.report import _SAFE_MAGNITUDE +from dokimi.repos.report import _SAFE_MAGNITUDE, CellFileError, refuse_mixed_executions from tests.cli_refusal import refusal @@ -450,3 +450,12 @@ def test_a_result_file_with_a_batch_that_is_not_a_string_exits_2(tmp_path, capsy err = capsys.readouterr().err assert f"{batch / 't__a__m__r1' / 'result.json'} has no usable `batch`" in err assert "Traceback" not in err + + +@pytest.mark.parametrize("batch, starts", [("b7", "b7: the cells of arm a, model m ran with"), + ("", "WHERE: the cells of arm a, model m ran with")]) +def test_the_mixed_execution_refusal_starts_with_the_batch_name_or_else_where(batch, starts): + cells = [{"batch": batch, "arm": "a", "model": "m", "execution": {"id": ran}} for ran in ("e1", "e2")] + with pytest.raises(CellFileError) as exc: + refuse_mixed_executions(cells, "WHERE") + assert str(exc.value).startswith(starts)