diff --git a/dokimi/changelog.d/20261117_120000_vlad-perevezentsev_687-low-reruns.md b/dokimi/changelog.d/20261117_120000_vlad-perevezentsev_687-low-reruns.md new file mode 100644 index 000000000..b187477bd --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_vlad-perevezentsev_687-low-reruns.md @@ -0,0 +1,22 @@ + + +### Fixed + +- **`data import docling` reruns replace their own files** — a rerun into the same `--out-dir` + wrote `--2.md` beside the file the first run wrote, so the docs dir filled with + copies. The output directory now keeps `.dokimi-docling.json` (`docling_sources.v1`), which + names each file's source; a rerun replaces that file atomically, and another source with the + same name still gets the next number, also across runs. A file the record does not name and a + link are still never written over. A directory converted before this change gets one more + numbered copy on its first rerun, then stays put (#687). +- **`eval personas discover` keeps a use-case bullet that names its section** — a README bullet + such as `- Web applications` under `## Use cases` was read as a section header and dropped, + and the same bullet outside a use-case section opened one (#687). +- **Docker harness cells: an egress sidecar whose removal failed is no longer taken as gone** — + a `docker rm` of the sidecar that exited non-zero still marked it stopped, so the verifier ran + beside it and teardown did not try the removal again. The cell now errors before the + verifier runs, as for any failed docker step, and teardown retries the removal before it drops + the network (#687). diff --git a/dokimi/docs/ingestion-docling.md b/dokimi/docs/ingestion-docling.md index 45dec8b6f..6ef7d2c3e 100644 --- a/dokimi/docs/ingestion-docling.md +++ b/dokimi/docs/ingestion-docling.md @@ -32,6 +32,18 @@ This converts every supported document under `--input` to a `.md` file in `.pptx`, `.xlsx`, `.html`/`.htm`, and common image formats (`.png`, `.jpg`, `.tiff`, …). +Each source becomes `-.md` (`manual.pdf` → `manual-pdf.md`). The +output directory keeps `.dokimi-docling.json`, a `docling_sources.v1` record of +which source each file came from, so a rerun replaces each source's own file. +A second source with the same name, such as `v2/manual.pdf`, gets the next +numbered name, `manual-pdf-2.md`, in the same run or a later one. A file the +record does not name, a directory and a symbolic link are never written over: +the source gets the next free name instead. A deleted file's name stays with its +source while that source exists, and names that differ only in letter case count as one. A directory at +the record's own name is refused with a message naming it. If the record cannot be read, a +warning names it, the files stay, and the run writes new numbered ones beside +them. + If Docling is not installed, the command exits with an install hint and the rest of the pipeline is unaffected — the dependency is fully optional. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a9b377d5b..3d064538b 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1560,7 +1560,7 @@ "params": [ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, - {"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'report_summary'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'docling_sources'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'report_summary'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--subject", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} ], "path": "dokimi runs list", diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 325463049..00a460aa9 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -19,8 +19,9 @@ path under a tree is), ``repo_hits_judged`` (a judge's labels of one replay cell's credited pairs), ``dokimi_config`` (a repository's subjects and where their evals are), ``reach_matrix`` (the reach conformance matrix), ``security_report`` (unsafe agent behaviour per arm of a run batch), ``personas`` and ``persona_analysis`` (proposed user personas and the GitHub -signals behind them), ``question_panel`` and ``question_quality`` (reviews of a question set) and -``report_qa`` (the Q&A sidecar of ``report eval``). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in +signals behind them), ``question_panel`` and ``question_quality`` (reviews of a question set), +``report_qa`` (the Q&A sidecar of ``report eval``) and ``docling_sources`` (which source each file of a +``data import docling`` output directory came from). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay forward-compatible and self-describing. @@ -103,6 +104,7 @@ "question_panel": "question_panel.v1", "question_quality": "question_quality.v1", "report_qa": "report_qa.v1", + "docling_sources": "docling_sources.v1", "report_summary": "report_summary.v1", } diff --git a/dokimi/dokimi/harnesses/container_agent.py b/dokimi/dokimi/harnesses/container_agent.py index d2764d250..5d45f25d3 100644 --- a/dokimi/dokimi/harnesses/container_agent.py +++ b/dokimi/dokimi/harnesses/container_agent.py @@ -199,7 +199,8 @@ def verifier_sidecar(self, docker: Docker, log, allow: list[str]) -> "ContainerS try: side._start_proxy(docker, log) except BaseException: # started, it holds the upstream proxy env and the network - side._stopped = not docker(["rm", "-f", side.proxy_name], log, check=False).timed_out + removed = docker(["rm", "-f", side.proxy_name], log, check=False) + side._stopped = not removed.timed_out and removed.returncode == 0 raise return side @@ -358,16 +359,21 @@ def stop_proxy(self, docker: Docker, log) -> None: removed = docker(["rm", "-f", self.proxy_name], log, check=False) if removed.timed_out: # still running, with the credentials: not stopped, close() retries raise subprocess.TimeoutExpired(removed.argv, DOCKER_STEP_TIMEOUT) + if removed.returncode != 0: # `rm -f` of a gone container exits 0: this one may still run (#687) + raise RuntimeError(f"docker rm {self.proxy_name} failed (exit {removed.returncode}); see run.log") self._stopped = True def stop_proxy_quietly(self, docker: Docker, log) -> None: - """:meth:`stop_proxy` once nothing runs behind the sidecar: a removal that times out is - logged, not raised, and :meth:`close` retries it.""" + """:meth:`stop_proxy` once nothing runs behind the sidecar: a removal that times out or + fails is logged, not raised, and :meth:`close` retries it.""" try: self.stop_proxy(docker, log) except subprocess.TimeoutExpired as exc: # teardown is best-effort: go on to the network if log is not None: log.write(f"\n[sidecar removal] {self.proxy_name} timed out after {exc.timeout:g}s\n") + except RuntimeError as exc: + if log is not None: + log.write(f"\n[sidecar removal] {exc}\n") def close(self, docker: Docker, log, output_dir: Optional[Path]) -> None: """Stop the sidecars, save the solve's decisions as ``egress.jsonl``, drop the network. diff --git a/dokimi/dokimi/ingest/docling_loader.py b/dokimi/dokimi/ingest/docling_loader.py index a01dbdf8d..81d0ccdfe 100644 --- a/dokimi/dokimi/ingest/docling_loader.py +++ b/dokimi/dokimi/ingest/docling_loader.py @@ -15,9 +15,17 @@ from __future__ import annotations import importlib.util +import itertools +import logging import os from pathlib import Path +from dokimi.artifacts import load_artifact, save_artifact +from dokimi.atomic_write import write_text_atomic +from dokimi.utils import InputError + +logger = logging.getLogger(__name__) + # Document types Docling can convert to Markdown. SUPPORTED_SUFFIXES = frozenset({ ".pdf", ".docx", ".pptx", ".xlsx", ".html", ".htm", @@ -26,6 +34,9 @@ _INSTALL_HINT = "Docling is not installed. Install it with: pip install 'dokimi[ocr]'" +# In the output directory: which source each .md was converted from (schemas/docling_sources.v1.json). +SOURCES_NAME = ".dokimi-docling.json" + def docling_available() -> bool: """True if the optional ``docling`` package is importable.""" @@ -64,23 +75,81 @@ def convert_to_markdown(path) -> str: def materialize_markdown(inputs, out_dir) -> list[Path]: """Convert ``inputs`` to Markdown files under ``out_dir``; return written paths. - Makes a corpus of PDFs/Office/scans benchmark-ready by emitting one ``.md`` - per source document. + Makes a corpus of PDFs/Office/scans benchmark-ready by emitting one ``-.md`` per + source document. A source converted into ``out_dir`` before replaces its own file (#687): the + ``.dokimi-docling.json`` record there names each file's source. Another source with the same + name, a file no record names and a link of any kind are left alone, and the next free + ``--N.md`` is written instead. """ out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) + record = out_dir / SOURCES_NAME + # an entry whose file and source are both gone holds its name no longer; the source is + # found as _source_id built it, lexically, even when out_dir is a link + base = os.path.abspath(out_dir) + owners = {name: source for name, source in _read_owners(record).items() + if os.path.lexists(out_dir / name) or os.path.lexists(os.path.normpath(os.path.join(base, source)))} written: list[Path] = [] for src in inputs: src = Path(src) markdown = convert_to_markdown(src) - # Disambiguate by suffix and a counter so same-named inputs (a.pdf, a.docx, - # or two a.pdf in different dirs) don't overwrite each other. - base = f"{src.stem}-{src.suffix.lstrip('.').lower()}" - dest = out_dir / f"{base}.md" - i = 2 - while os.path.lexists(dest): # a link of any kind is taken: a write would follow it (#545) - dest = out_dir / f"{base}-{i}.md" - i += 1 - dest.write_text(markdown, encoding="utf-8") + source = _source_id(src, out_dir) + dest = _dest(src, out_dir, owners, source) + # Recorded before the write: a file is never left that no record names (a rerun would write a -2). + owners = {name: s for name, s in owners.items() if s != source} | {dest.name: source} + _save_owners(record, owners) + write_text_atomic(dest, markdown) written.append(dest) return written + + +def _source_id(src: Path, out_dir: Path) -> str: + """The source as the record names it: relative to ``out_dir``, so the two can move together. + Links are not followed: a link is a source of its own, whatever it points at.""" + absolute = os.path.abspath(src) + try: + return Path(os.path.relpath(absolute, os.path.abspath(out_dir))).as_posix() + except ValueError: # on another Windows drive there is no relative path + return Path(absolute).as_posix() + + +def _dest(src: Path, out_dir: Path, owners: dict[str, str], source: str) -> Path: + """The file the record gives ``source``, else the first free name: one no other source holds in + the record (letter case aside), even when its file was deleted, and no file or link of any kind + is at (#545).""" + for name, owner in owners.items(): + path = out_dir / name + if owner != source or path.name != name: # a `\` is a separator on Windows: not a name there + continue + if path.is_file() and not path.is_symlink() or not os.path.lexists(path): + return path + base = f"{src.stem}-{src.suffix.lstrip('.').lower()}" + names = itertools.chain([f"{base}.md"], (f"{base}-{i}.md" for i in itertools.count(2))) + held = {name.casefold() for name in owners} # A-pdf.md and a-pdf.md are one file on a case-insensitive disk + return next(out_dir / n for n in names if n.casefold() not in held and not os.path.lexists(out_dir / n)) + + +def _read_owners(record: Path) -> dict[str, str]: + """File name -> source, from the record an earlier run left; empty when there is none to use.""" + if not os.path.lexists(record): + return {} + if record.is_symlink(): + reason = f"{record} is a link" + else: + try: + return {f["name"]: f["source"] for f in load_artifact("docling_sources", record)["files"]} + except InputError as exc: + reason = str(exc) + # the files stay: a rerun writes beside them and starts a new record + logger.warning("Ignoring the record of converted files: %s. The files already in %s are kept, " + "and this run writes new ones beside them", reason, record.parent) + return {} + + +def _save_owners(record: Path, owners: dict[str, str]) -> None: + if record.is_dir() and not record.is_symlink(): + raise InputError(f"{record} is a directory, not the record of converted files: remove it or use another --out-dir") + if record.is_symlink(): # save_artifact would write through it, out of out_dir (#545) + record.unlink() + files = [{"name": name, "source": source} for name, source in sorted(owners.items())] + save_artifact("docling_sources", {"files": files}, record) diff --git a/dokimi/dokimi/personas/analyzer.py b/dokimi/dokimi/personas/analyzer.py index 0aa0cf318..7ee1aa847 100644 --- a/dokimi/dokimi/personas/analyzer.py +++ b/dokimi/dokimi/personas/analyzer.py @@ -4,6 +4,7 @@ """Analyze project to gather signals for persona generation.""" import logging +import re from typing import Dict, List, Any, Optional from pathlib import Path @@ -18,6 +19,8 @@ logger = logging.getLogger(__name__) +_BULLET = re.compile(r"\s*[-*•]\s") # a list marker and a space: "**Use cases**" is a line, not a bullet + class PersonaAnalyzer: """ @@ -110,13 +113,13 @@ def _analyze_with_pygithub(self, client: "Github", repo_name: str) -> Dict[str, repo = client.get_repo(repo_name) readme_content = self._get_readme(repo) - use_cases = self._extract_use_cases(repo) + use_cases = self._extract_use_cases_from_text(readme_content) issues_analysis = self._analyze_issues(repo) analysis = { "readme_content": readme_content, "use_cases": use_cases, "issues_analysis": issues_analysis, - "api_patterns": self._extract_api_patterns(repo), + "api_patterns": self._extract_api_patterns_from_text(readme_content), "description": repo.description or "", "topics": repo.get_topics() } @@ -172,24 +175,25 @@ def _analyze_with_gh_cli(self, repo_name: str) -> Dict[str, Any]: @staticmethod def _extract_use_cases_from_text(text: str) -> List[str]: - """Extract use cases from text (fallback for gh CLI).""" + """The list items of the use-case sections, lowercased, at most 10. + + A line that is not a bullet and names "use case", "application" or "when to use" opens a + section; a heading without them closes it.""" if not text: return [] use_cases = [] - lines = text.lower().split('\n') in_use_case_section = False - for line in lines: - if any(keyword in line for keyword in ['use case', 'application', 'when to use']): - in_use_case_section = True - continue - - if in_use_case_section: - if line.startswith('#'): - in_use_case_section = False - elif line.strip().startswith(('-', '*', '•')): + for line in text.lower().split('\n'): + # A bullet is an item, never a section header, even when it names the keyword (#687). + if _BULLET.match(line): + if in_use_case_section: use_cases.append(line.strip().lstrip('-*• ')) + elif any(keyword in line for keyword in ['use case', 'application', 'when to use']): + in_use_case_section = True + elif line.startswith('#'): + in_use_case_section = False return use_cases[:10] @@ -220,39 +224,6 @@ def _get_readme(self, repo) -> str: except Exception: # noqa: BLE001 - a repo without a readable README has no README text return "" - def _extract_use_cases(self, repo) -> List[str]: - """ - Extract use cases from README and docs. - - Looks for sections like: - - Use Cases - - When to Use - - Applications - - Examples - """ - use_cases: List[str] = [] - readme = self._get_readme(repo) - - if not readme: - return use_cases - - # Simple heuristic: look for list items after "use case" headers - lines = readme.lower().split('\n') - in_use_case_section = False - - for line in lines: - if any(keyword in line for keyword in ['use case', 'application', 'when to use']): - in_use_case_section = True - continue - - if in_use_case_section: - if line.startswith('#'): # New section - in_use_case_section = False - elif line.strip().startswith(('-', '*', '•')): - use_cases.append(line.strip().lstrip('-*• ')) - - return use_cases[:10] # Limit to top 10 - def _analyze_issues(self, repo, limit: int = 50) -> Dict[str, Any]: """ Analyze recent issues for common question patterns. @@ -307,35 +278,6 @@ def _analyze_issues(self, repo, limit: int = 50) -> Dict[str, Any]: logger.error(f"Issue analysis error: {e}") return {"common_questions": [], "common_labels": [], "sample_issues": []} - def _extract_api_patterns(self, repo) -> List[str]: - """ - Extract common API patterns from code examples. - - Looks for: - - Frequently used class/function names - - Import patterns - - Code snippets in docs - """ - # Simplified version: extract from README code blocks - readme = self._get_readme(repo) - if not readme: - return [] - - patterns = [] - in_code_block = False - - for line in readme.split('\n'): - if line.strip().startswith('```'): - in_code_block = not in_code_block - continue - - if in_code_block: - # Extract likely API calls (simplified heuristic) - if '::' in line or '.' in line: - patterns.append(line.strip()) - - return patterns[:20] - def _empty_analysis(self) -> Dict[str, Any]: """Return empty analysis structure.""" return { diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index 08334b0d8..b2989f8c9 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -101,6 +101,7 @@ class KindFields: "question_quality": KindFields(subject=("library_name",), counts=("questions", "trivial_questions", "recommendations")), "report_qa": KindFields(time=("generated_at",), counts=("records",)), + "docling_sources": KindFields(counts=("files",)), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/docling_sources.v1.json b/dokimi/dokimi/schemas/docling_sources.v1.json new file mode 100644 index 000000000..56fed9dee --- /dev/null +++ b/dokimi/dokimi/schemas/docling_sources.v1.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/docling_sources.v1.json", + "title": "Docling import sources v1", + "description": "Which source document each Markdown file in a `data import docling` output directory was converted from (docs/ingestion-docling.md), kept there as `.dokimi-docling.json`. A rerun replaces the file the record gives a source; another source with the same name gets the next numbered file.", + "type": "object", + "required": ["schema_version", "files"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "docling_sources.v1"}, + "run_id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}(?![\\s\\S])", "description": "The envelope: the id of the run that saved this file (dokimi.artifacts.run_id). Absent from files saved before the envelope."}, + "dokimi_version": {"type": "string", "minLength": 1, "description": "The envelope: the dokimi version that saved this file. Absent from files saved before the envelope."}, + "files": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "source"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "pattern": "^[^/]+\\.md(?![\\s\\S])", "description": "The Markdown file's name in the output directory: a name, not a path (a `\\` is a name character on POSIX; on Windows a name holding one is not used)."}, + "source": {"type": "string", "minLength": 1, "description": "The source document it was converted from, `/`-separated: relative to the output directory, or absolute when there is none (another Windows drive)."} + } + }, + "description": "One entry per file, sorted by name." + } + } +} diff --git a/dokimi/scripts/duplicates_baseline.json b/dokimi/scripts/duplicates_baseline.json index 6d3207ef7..dab4dde94 100644 --- a/dokimi/scripts/duplicates_baseline.json +++ b/dokimi/scripts/duplicates_baseline.json @@ -2,7 +2,6 @@ "groups": { "3c397e7b5b50": {"count": 3, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/repos/cli_traces.py", "dokimi/repos/events.py"]}, "4751a8be1f57": {"count": 2, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/eval/agent_runner.py"]}, - "87fd5d156635": {"count": 2, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/personas/analyzer.py"]}, "925b1461ae22": {"count": 2, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/report/eval_report.py"]}, "96a248688260": {"count": 2, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/commands/baseline.py", "dokimi/eval/judge.py"]}, "9912f72bb7c0": {"count": 2, "rationale": "pre-existing when the gate was introduced", "where": ["dokimi/repos/reach.py", "dokimi/repos/skill_reach.py"]}, diff --git a/dokimi/tests/fixtures/artifacts/docling_sources.v1.json b/dokimi/tests/fixtures/artifacts/docling_sources.v1.json new file mode 100644 index 000000000..8839f1f4b --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/docling_sources.v1.json @@ -0,0 +1,7 @@ +{ + "schema_version": "docling_sources.v1", + "files": [ + {"name": "manual-pdf-2.md", "source": "../raw/v2/manual.pdf"}, + {"name": "manual-pdf.md", "source": "../raw/manual.pdf"} + ] +} diff --git a/dokimi/tests/test_687_low.py b/dokimi/tests/test_687_low.py new file mode 100644 index 000000000..e758bd653 --- /dev/null +++ b/dokimi/tests/test_687_low.py @@ -0,0 +1,320 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""#687 Low: state that lies after a rerun or a failed step (https://github.com/intel/dokimi/issues/687).""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from dokimi.artifacts import load_artifact +from dokimi.harnesses import container_agent as container_mod +from dokimi.harnesses import load_task +from dokimi.harnesses.registry import create_harness +from dokimi.ingest import docling_loader +from dokimi.ingest.docling_loader import SOURCES_NAME, materialize_markdown +from dokimi.personas.analyzer import PersonaAnalyzer +from dokimi.process import reply +from dokimi.utils import InputError +# _hermetic is autouse: no operator Claude settings, provider env or host `claude` reach a test. +from tests.test_docker_solver_harness import _fixture_task, _hermetic, _pipeline # noqa: F401 + +# ── ingest/docling_loader.py: a rerun replaces its own output ───────────────── +# The rules are docs/ingestion-docling.md "Materialize a corpus". + + +@pytest.fixture +def converted(monkeypatch): + """Docling replaced by a converter whose output names the call, so a stale file shows.""" + calls: list[Path] = [] + + def convert(path): + calls.append(Path(path)) + return f"# {Path(path).name}, conversion {len(calls)}\n" + monkeypatch.setattr(docling_loader, "convert_to_markdown", convert) + return calls + + +def _docs(tmp_path: Path, *rels: str) -> list[Path]: + paths = [tmp_path / "raw" / rel for rel in rels] + for path in paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"%PDF") + return paths + + +def _md(out: Path) -> dict[str, str]: + return {p.name: p.read_text(encoding="utf-8") for p in sorted(out.glob("*.md"))} + + +def test_a_rerun_replaces_its_own_output_instead_of_writing_a_numbered_copy(tmp_path, converted): + docs, out = _docs(tmp_path, "a.pdf", "sub/a.pdf"), tmp_path / "md" + assert materialize_markdown(docs, out) == [out / "a-pdf.md", out / "a-pdf-2.md"] + + assert materialize_markdown(docs, out) == [out / "a-pdf.md", out / "a-pdf-2.md"] + assert _md(out) == {"a-pdf.md": "# a.pdf, conversion 3\n", "a-pdf-2.md": "# a.pdf, conversion 4\n"} + record = load_artifact("docling_sources", out / SOURCES_NAME) + assert record["files"] == [{"name": "a-pdf-2.md", "source": "../raw/sub/a.pdf"}, + {"name": "a-pdf.md", "source": "../raw/a.pdf"}] + + +def test_another_source_with_the_name_gets_a_counter_across_runs(tmp_path, converted): + (first,), (second,) = _docs(tmp_path, "x/a.pdf"), _docs(tmp_path, "y/a.pdf") + out = tmp_path / "md" + assert materialize_markdown([first], out) == [out / "a-pdf.md"] + assert materialize_markdown([second], out) == [out / "a-pdf-2.md"] + + assert materialize_markdown([first], out) == [out / "a-pdf.md"] + assert materialize_markdown([second], out) == [out / "a-pdf-2.md"] + assert sorted(_md(out)) == ["a-pdf-2.md", "a-pdf.md"] + + +def test_the_same_source_by_another_path_is_the_same_source(tmp_path, converted, monkeypatch): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + monkeypatch.chdir(tmp_path) + + assert materialize_markdown([Path("raw/a.pdf")], Path("md")) == [Path("md/a-pdf.md")] + assert sorted(_md(out)) == ["a-pdf.md"] + + +def test_a_failed_record_save_leaves_no_file_the_rerun_cannot_place(tmp_path, converted, monkeypatch): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + + def full(kind, data, path): + raise OSError(28, "No space left on device") + with monkeypatch.context() as patch: + patch.setattr(docling_loader, "save_artifact", full) + with pytest.raises(OSError, match="No space"): + materialize_markdown([doc], out) + + assert materialize_markdown([doc], out) == [out / "a-pdf.md"] + assert sorted(_md(out)) == ["a-pdf.md"] + + +def test_a_file_no_record_names_is_kept(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + out.mkdir() + (out / "a-pdf.md").write_text("hand-written\n", encoding="utf-8") + + assert materialize_markdown([doc], out) == [out / "a-pdf-2.md"] + assert (out / "a-pdf.md").read_text(encoding="utf-8") == "hand-written\n" + + +def _link(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except OSError: + pytest.skip("symlinks are not allowed here") + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs symlinks") +def test_a_link_at_a_recorded_name_is_not_written_through(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + (out / "a-pdf.md").unlink() + (tmp_path / "elsewhere.md").write_text("outside\n", encoding="utf-8") + _link(out / "a-pdf.md", tmp_path / "elsewhere.md") + + assert materialize_markdown([doc], out) == [out / "a-pdf-2.md"] + assert (tmp_path / "elsewhere.md").read_text(encoding="utf-8") == "outside\n" + assert (out / "a-pdf.md").is_symlink() + assert [f["name"] for f in load_artifact("docling_sources", out / SOURCES_NAME)["files"]] == ["a-pdf-2.md"] + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs symlinks") +def test_a_link_at_the_record_is_neither_read_nor_written_through(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + (out / SOURCES_NAME).rename(tmp_path / "kept.json") + before = (tmp_path / "kept.json").read_text(encoding="utf-8") + _link(out / SOURCES_NAME, tmp_path / "kept.json") + + assert materialize_markdown([doc], out) == [out / "a-pdf-2.md"] + assert (tmp_path / "kept.json").read_text(encoding="utf-8") == before + assert not (out / SOURCES_NAME).is_symlink() + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs symlinks") +def test_a_linked_source_is_a_source_of_its_own(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + _link(tmp_path / "raw" / "cur.pdf", doc) + inputs = docling_loader.discover_documents(tmp_path / "raw") + assert materialize_markdown(inputs, out) == [out / "a-pdf.md", out / "cur-pdf.md"] + + (doc.parent / "v2.pdf").write_bytes(b"%PDF") + (tmp_path / "raw" / "cur.pdf").unlink() + _link(tmp_path / "raw" / "cur.pdf", doc.parent / "v2.pdf") # repointed: still the source cur.pdf + assert materialize_markdown(inputs, out) == [out / "a-pdf.md", out / "cur-pdf.md"] + assert sorted(_md(out)) == ["a-pdf.md", "cur-pdf.md"] + + +def test_a_source_whose_file_was_deleted_writes_it_again_under_its_name(tmp_path, converted): + docs, out = _docs(tmp_path, "a.pdf", "sub/a.pdf"), tmp_path / "md" + materialize_markdown(docs, out) + (out / "a-pdf.md").unlink() + + assert materialize_markdown(docs[1:], out) == [out / "a-pdf-2.md"] + assert materialize_markdown(docs[:1], out) == [out / "a-pdf.md"] + + +def test_a_new_source_does_not_take_the_name_of_a_deleted_file(tmp_path, converted): + v1, v2 = _docs(tmp_path, "v1/m.pdf", "v2/m.pdf") + out = tmp_path / "md" + materialize_markdown([v1, v2], out) + (out / "m-pdf.md").unlink() + (v0,) = _docs(tmp_path, "v0/m.pdf") + + assert materialize_markdown([v0, v1, v2], out) == [out / "m-pdf-3.md", out / "m-pdf.md", out / "m-pdf-2.md"] + assert _md(out)["m-pdf.md"] == "# m.pdf, conversion 4\n" # v1's text, not v0's + + +def test_a_name_whose_file_and_source_are_gone_is_free_again(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + (out / "a-pdf.md").unlink() + (tmp_path / "raw").rename(tmp_path / "corpus") + + assert materialize_markdown([tmp_path / "corpus" / "a.pdf"], out) == [out / "a-pdf.md"] + assert load_artifact("docling_sources", out / SOURCES_NAME)["files"] == [ + {"name": "a-pdf.md", "source": "../corpus/a.pdf"}] + + +@pytest.mark.skipif(not hasattr(os, "symlink"), reason="needs symlinks") +def test_a_recorded_source_is_found_when_the_output_directory_is_a_link(tmp_path, converted): + v1, v0 = _docs(tmp_path / "work", "v1/m.pdf", "v0/m.pdf") + (tmp_path / "data" / "md").mkdir(parents=True) + out = tmp_path / "work" / "md" + _link(out, tmp_path / "data" / "md") + materialize_markdown([v1], out) + (out / "m-pdf.md").unlink() + + assert materialize_markdown([v0, v1], out) == [out / "m-pdf-2.md", out / "m-pdf.md"] + + +@pytest.mark.skipif(os.name == "nt", reason="a backslash is a path separator on Windows") +def test_a_source_name_with_a_backslash_is_converted_and_rerun(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a\\b.pdf"), tmp_path / "md" + assert materialize_markdown([doc], out) == [out / "a\\b-pdf.md"] + assert materialize_markdown([doc], out) == [out / "a\\b-pdf.md"] + assert sorted(_md(out)) == ["a\\b-pdf.md"] + + +def test_a_directory_at_a_recorded_name_is_taken(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + (out / "a-pdf.md").unlink() + (out / "a-pdf.md").mkdir() + + assert materialize_markdown([doc], out) == [out / "a-pdf-2.md"] + + +def test_a_name_differing_only_in_letter_case_stays_with_the_deleted_files_source(tmp_path, converted): + upper, lower = _docs(tmp_path, "x/A.pdf", "y/a.pdf") + out = tmp_path / "md" + materialize_markdown([upper], out) + (out / "A-pdf.md").unlink() + + assert materialize_markdown([lower], out) == [out / "a-pdf-2.md"] + + +def test_a_directory_at_the_record_name_is_refused_naming_the_record(tmp_path, converted): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + (out / SOURCES_NAME).mkdir(parents=True) + + with pytest.raises(InputError, match=SOURCES_NAME): + materialize_markdown([doc], out) + assert not list(out.glob("*.md")) + + +def test_an_unreadable_record_is_named_and_its_files_are_kept(tmp_path, converted, caplog): + (doc,), out = _docs(tmp_path, "a.pdf"), tmp_path / "md" + materialize_markdown([doc], out) + (out / SOURCES_NAME).write_text("{not json", encoding="utf-8") + + with caplog.at_level(logging.WARNING, logger="dokimi.ingest.docling_loader"): + assert materialize_markdown([doc], out) == [out / "a-pdf-2.md"] + assert SOURCES_NAME in caplog.text and "are kept" in caplog.text + assert (out / "a-pdf.md").read_text(encoding="utf-8") == "# a.pdf, conversion 1\n" + assert load_artifact("docling_sources", out / SOURCES_NAME)["files"] == [ + {"name": "a-pdf-2.md", "source": "../raw/a.pdf"}] + + +# ── personas/analyzer.py: the use-case section parser ───────────────────────── + + +def test_a_use_case_bullet_that_names_the_keyword_is_kept(): + readme = "## Use cases\n- Web applications\n- Batch jobs\n\n## Install\n- pip install x\n" + assert PersonaAnalyzer._extract_use_cases_from_text(readme) == ["web applications", "batch jobs"] + + +def test_a_bullet_that_names_the_keyword_opens_no_section(): + readme = "## Features\n- Runs any application\n- Fast\n" + assert PersonaAnalyzer._extract_use_cases_from_text(readme) == [] + + +def test_a_bold_use_case_line_still_opens_a_section(): + readme = "**Use cases**\n* Robotics\n\n## Install\n- pip install x\n" + assert PersonaAnalyzer._extract_use_cases_from_text(readme) == ["robotics"] + + +def test_the_pygithub_path_keeps_a_keyword_bullet_and_reads_the_readme_once(): + client, repo = Mock(), Mock(description="", get_issues=Mock(return_value=[])) + client.get_repo.return_value = repo + repo.get_readme.return_value.decoded_content = b"## When to use\n- Desktop applications\n- CLIs\n" + repo.get_topics.return_value = [] + + analysis = PersonaAnalyzer()._analyze_with_pygithub(client, "o/r") + assert analysis["use_cases"] == ["desktop applications", "clis"] + assert repo.get_readme.call_count == 1 + + +# ── harnesses/container_agent.py: a sidecar whose removal failed is not stopped ─ + + +def _sidecar_rm(call) -> bool: + """A removal of the solve's sidecar, not of the verifier's that open() also clears.""" + return call.argv[1] == "rm" and "egress" in call.argv[-1] and not call.argv[-1].endswith("-verify") + + +def test_a_failed_sidecar_removal_errors_the_cell_and_close_retries_it(tmp_path): + task = load_task(_fixture_task(tmp_path)) + cli = _pipeline(reward=1.0, answer=lambda call: reply(stderr="Error response from daemon: busy", returncode=1) + if _sidecar_rm(call) else None) + + result = create_harness("docker-claude", runner=cli).run(task, output_dir=tmp_path / "out") + + assert (result.returncode, result.reward, result.passed) == (1, None, False) + assert "docker rm dokimi-egress-" in result.metrics["error"] + run_log = (tmp_path / "out" / "run.log").read_text(encoding="utf-8") + assert "[sidecar removal] docker rm dokimi-egress-" in run_log + assert not [c for c in cli.calls if c.argv[1] == "exec" and "test.sh" in " ".join(c.argv)] + # open() clears a leftover, stop_proxy() fails, close() tries again and still drops the network + removals = [i for i, c in enumerate(cli.calls) if _sidecar_rm(c)] + assert len(removals) == 3 + assert [i for i, c in enumerate(cli.calls) if list(c.argv[1:3]) == ["network", "rm"]][-1] > removals[-1] + + +def test_a_verifier_sidecar_whose_removal_failed_is_removed_again_on_close(): + removals: list[str] = [] + + def docker(args, log=None, *, check=True, timeout=None): + if args[0] == "create": + raise RuntimeError("docker create failed (exit 1); see run.log") + if args[0] == "rm": + removals.append(args[-1]) + if args[-1] == "p-verify" and removals.count("p-verify") == 2: # the removal after the failed start + return reply(stderr="Error response from daemon: busy", returncode=1) + return reply() + + sandbox = container_mod.ContainerSandbox(network="n", proxy_name="p", allow=[], binary=None) + with pytest.raises(RuntimeError, match="docker create failed"): + sandbox.verifier_sidecar(docker, None, ["pypi.org"]) + sandbox.close(docker, None, None) + + assert removals.count("p-verify") == 3 diff --git a/dokimi/tests/test_personas_analyzer.py b/dokimi/tests/test_personas_analyzer.py index 23fb3e9a5..50da89826 100644 --- a/dokimi/tests/test_personas_analyzer.py +++ b/dokimi/tests/test_personas_analyzer.py @@ -153,21 +153,16 @@ def test_get_readme_lets_keyboard_interrupt_through(self): def test_extract_use_cases(self, mock_github_repo): """Test use case extraction from README.""" - analyzer = PersonaAnalyzer() - analyzer._get_readme = Mock(return_value=mock_github_repo.get_readme().decoded_content.decode()) + readme = mock_github_repo.get_readme().decoded_content.decode() - use_cases = analyzer._extract_use_cases(mock_github_repo) + use_cases = PersonaAnalyzer._extract_use_cases_from_text(readme) assert len(use_cases) > 0 assert any("computing" in uc.lower() for uc in use_cases) def test_extract_use_cases_empty_readme(self): """Test use case extraction with empty README.""" - repo = Mock() - analyzer = PersonaAnalyzer() - analyzer._get_readme = Mock(return_value="") - - use_cases = analyzer._extract_use_cases(repo) + use_cases = PersonaAnalyzer._extract_use_cases_from_text("") assert use_cases == [] def test_analyze_issues(self, mock_github_repo, mock_github_issues): @@ -218,8 +213,7 @@ def test_analyze_issues_lets_keyboard_interrupt_through(self, mock_github_repo): def test_extract_api_patterns(self, mock_github_repo): """Test API pattern extraction.""" - analyzer = PersonaAnalyzer() - analyzer._get_readme = Mock(return_value=""" + readme = """ ```cpp tbb::parallel_for(range, [](int i) { // do work @@ -228,9 +222,9 @@ def test_extract_api_patterns(self, mock_github_repo): tbb::task_arena arena; arena.initialize(); ``` -""") +""" - patterns = analyzer._extract_api_patterns(mock_github_repo) + patterns = PersonaAnalyzer._extract_api_patterns_from_text(readme) # Code-block lines with :: or . are kept; the comment and the closing brace are not. assert patterns == [ diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 83d3b387e..9dcbf001d 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -77,6 +77,7 @@ "question_panel.v1": ({}, "demo", None, "2026-11-01T00:00:00+00:00"), "question_quality.v1": ({}, "demo", None, None), "report_qa.v1": ({}, None, None, "2026-11-01T00:00:00+00:00"), + "docling_sources.v1": ({}, None, None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 4cf85a722..d60c9051a 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -69,6 +69,7 @@ "question_panel.v1": {"questions": 2}, "question_quality.v1": {"questions": 2, "trivial_questions": 1, "recommendations": 1}, "report_qa.v1": {"records": 1}, + "docling_sources.v1": {"files": 2}, } FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts"