Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **`data import docling` reruns replace their own files** — a rerun into the same `--out-dir`
wrote `<stem>-<ext>-2.md` beside the file the first run wrote, so the docs dir filled with
copies. The output directory now keeps `.dokimi-docling.json` (`docling_sources.v1`), which
names each file's source; a rerun replaces that file atomically, and another source with the
same name still gets the next number, also across runs. A file the record does not name and a
link are still never written over. A directory converted before this change gets one more
numbered copy on its first rerun, then stays put (#687).
- **`eval personas discover` keeps a use-case bullet that names its section** — a README bullet
such as `- Web applications` under `## Use cases` was read as a section header and dropped,
and the same bullet outside a use-case section opened one (#687).
- **Docker harness cells: an egress sidecar whose removal failed is no longer taken as gone** —
a `docker rm` of the sidecar that exited non-zero still marked it stopped, so the verifier ran
beside it and teardown did not try the removal again. The cell now errors before the
verifier runs, as for any failed docker step, and teardown retries the removal before it drops
the network (#687).
12 changes: 12 additions & 0 deletions dokimi/docs/ingestion-docling.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,18 @@ This converts every supported document under `--input` to a `.md` file in
`.pptx`, `.xlsx`, `.html`/`.htm`, and common image formats (`.png`, `.jpg`,
`.tiff`, …).

Each source becomes `<stem>-<ext>.md` (`manual.pdf` → `manual-pdf.md`). The
output directory keeps `.dokimi-docling.json`, a `docling_sources.v1` record of
which source each file came from, so a rerun replaces each source's own file.
A second source with the same name, such as `v2/manual.pdf`, gets the next
numbered name, `manual-pdf-2.md`, in the same run or a later one. A file the
record does not name, a directory and a symbolic link are never written over:
the source gets the next free name instead. A deleted file's name stays with its
source while that source exists, and names that differ only in letter case count as one. A directory at
the record's own name is refused with a message naming it. If the record cannot be read, a
warning names it, the files stay, and the run writes new numbered ones beside
them.

If Docling is not installed, the command exits with an install hint and the rest
of the pipeline is unaffected — the dependency is fully optional.

Expand Down
2 changes: 1 addition & 1 deletion dokimi/docs/reference/cli-interface.json
Original file line number Diff line number Diff line change
Expand Up @@ -1560,7 +1560,7 @@
"params": [
{"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null},
{"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null},
{"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'report_summary'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'docling_sources'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'report_summary'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--subject", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}
],
"path": "dokimi runs list",
Expand Down
6 changes: 4 additions & 2 deletions dokimi/dokimi/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@
path under a tree is), ``repo_hits_judged`` (a judge's labels of one replay cell's credited pairs),
``dokimi_config`` (a repository's subjects and where their evals are), ``reach_matrix`` (the reach
conformance matrix), ``security_report`` (unsafe agent behaviour per arm of a run batch), ``personas`` and ``persona_analysis`` (proposed user personas and the GitHub
signals behind them), ``question_panel`` and ``question_quality`` (reviews of a question set) and
``report_qa`` (the Q&A sidecar of ``report eval``). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in
signals behind them), ``question_panel`` and ``question_quality`` (reviews of a question set),
``report_qa`` (the Q&A sidecar of ``report eval``) and ``docling_sources`` (which source each file of a
``data import docling`` output directory came from). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in
``dokimi/schemas/`` on save and load, so fixtures and baselines stay
forward-compatible and self-describing.

Expand Down Expand Up @@ -103,6 +104,7 @@
"question_panel": "question_panel.v1",
"question_quality": "question_quality.v1",
"report_qa": "report_qa.v1",
"docling_sources": "docling_sources.v1",
"report_summary": "report_summary.v1",
}

Expand Down
12 changes: 9 additions & 3 deletions dokimi/dokimi/harnesses/container_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,8 @@ def verifier_sidecar(self, docker: Docker, log, allow: list[str]) -> "ContainerS
try:
side._start_proxy(docker, log)
except BaseException: # started, it holds the upstream proxy env and the network
side._stopped = not docker(["rm", "-f", side.proxy_name], log, check=False).timed_out
removed = docker(["rm", "-f", side.proxy_name], log, check=False)
side._stopped = not removed.timed_out and removed.returncode == 0
raise
return side

Expand Down Expand Up @@ -358,16 +359,21 @@ def stop_proxy(self, docker: Docker, log) -> None:
removed = docker(["rm", "-f", self.proxy_name], log, check=False)
if removed.timed_out: # still running, with the credentials: not stopped, close() retries
raise subprocess.TimeoutExpired(removed.argv, DOCKER_STEP_TIMEOUT)
if removed.returncode != 0: # `rm -f` of a gone container exits 0: this one may still run (#687)
raise RuntimeError(f"docker rm {self.proxy_name} failed (exit {removed.returncode}); see run.log")
self._stopped = True

def stop_proxy_quietly(self, docker: Docker, log) -> None:
""":meth:`stop_proxy` once nothing runs behind the sidecar: a removal that times out is
logged, not raised, and :meth:`close` retries it."""
""":meth:`stop_proxy` once nothing runs behind the sidecar: a removal that times out or
fails is logged, not raised, and :meth:`close` retries it."""
try:
self.stop_proxy(docker, log)
except subprocess.TimeoutExpired as exc: # teardown is best-effort: go on to the network
if log is not None:
log.write(f"\n[sidecar removal] {self.proxy_name} timed out after {exc.timeout:g}s\n")
except RuntimeError as exc:
if log is not None:
log.write(f"\n[sidecar removal] {exc}\n")

def close(self, docker: Docker, log, output_dir: Optional[Path]) -> None:
"""Stop the sidecars, save the solve's decisions as ``egress.jsonl``, drop the network.
Expand Down
91 changes: 80 additions & 11 deletions dokimi/dokimi/ingest/docling_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,17 @@
from __future__ import annotations

import importlib.util
import itertools
import logging
import os
from pathlib import Path

from dokimi.artifacts import load_artifact, save_artifact
from dokimi.atomic_write import write_text_atomic
from dokimi.utils import InputError

logger = logging.getLogger(__name__)

# Document types Docling can convert to Markdown.
SUPPORTED_SUFFIXES = frozenset({
".pdf", ".docx", ".pptx", ".xlsx", ".html", ".htm",
Expand All @@ -26,6 +34,9 @@

_INSTALL_HINT = "Docling is not installed. Install it with: pip install 'dokimi[ocr]'"

# In the output directory: which source each .md was converted from (schemas/docling_sources.v1.json).
SOURCES_NAME = ".dokimi-docling.json"


def docling_available() -> bool:
"""True if the optional ``docling`` package is importable."""
Expand Down Expand Up @@ -64,23 +75,81 @@ def convert_to_markdown(path) -> str:
def materialize_markdown(inputs, out_dir) -> list[Path]:
"""Convert ``inputs`` to Markdown files under ``out_dir``; return written paths.

Makes a corpus of PDFs/Office/scans benchmark-ready by emitting one ``.md``
per source document.
Makes a corpus of PDFs/Office/scans benchmark-ready by emitting one ``<stem>-<ext>.md`` per
source document. A source converted into ``out_dir`` before replaces its own file (#687): the
``.dokimi-docling.json`` record there names each file's source. Another source with the same
name, a file no record names and a link of any kind are left alone, and the next free
``<stem>-<ext>-N.md`` is written instead.
"""
out_dir = Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
record = out_dir / SOURCES_NAME
# an entry whose file and source are both gone holds its name no longer; the source is
# found as _source_id built it, lexically, even when out_dir is a link
base = os.path.abspath(out_dir)
owners = {name: source for name, source in _read_owners(record).items()
if os.path.lexists(out_dir / name) or os.path.lexists(os.path.normpath(os.path.join(base, source)))}
written: list[Path] = []
for src in inputs:
src = Path(src)
markdown = convert_to_markdown(src)
# Disambiguate by suffix and a counter so same-named inputs (a.pdf, a.docx,
# or two a.pdf in different dirs) don't overwrite each other.
base = f"{src.stem}-{src.suffix.lstrip('.').lower()}"
dest = out_dir / f"{base}.md"
i = 2
while os.path.lexists(dest): # a link of any kind is taken: a write would follow it (#545)
dest = out_dir / f"{base}-{i}.md"
i += 1
dest.write_text(markdown, encoding="utf-8")
source = _source_id(src, out_dir)
dest = _dest(src, out_dir, owners, source)
# Recorded before the write: a file is never left that no record names (a rerun would write a -2).
owners = {name: s for name, s in owners.items() if s != source} | {dest.name: source}
_save_owners(record, owners)
write_text_atomic(dest, markdown)
written.append(dest)
return written


def _source_id(src: Path, out_dir: Path) -> str:
"""The source as the record names it: relative to ``out_dir``, so the two can move together.
Links are not followed: a link is a source of its own, whatever it points at."""
absolute = os.path.abspath(src)
try:
return Path(os.path.relpath(absolute, os.path.abspath(out_dir))).as_posix()
except ValueError: # on another Windows drive there is no relative path
return Path(absolute).as_posix()


def _dest(src: Path, out_dir: Path, owners: dict[str, str], source: str) -> Path:
"""The file the record gives ``source``, else the first free name: one no other source holds in
the record (letter case aside), even when its file was deleted, and no file or link of any kind
is at (#545)."""
for name, owner in owners.items():
path = out_dir / name
if owner != source or path.name != name: # a `\` is a separator on Windows: not a name there
continue
Comment on lines +122 to +123

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bug · medium
Must-fix — case-sensitive source identity breaks Windows reruns. Trigger: on a case-insensitive Windows filesystem, import raw/manual.pdf, change its contents, then import the same file as RAW/MANUAL.PDF into the same output directory. Evidence: _source_id() preserves those spellings, this comparison rejects the existing owner, and the casefolded filename reservation allocates MANUAL-pdf-2.md; the caller does not normalize casing. Impact: the old Markdown remains alongside the updated conversion, so downstream corpus ingestion includes stale, duplicate content instead of replacing the source's file. Normalize source identities/comparisons using Windows path semantics (including the ownership cleanup at line 99), without resolving symlinks, and add a regression for case-varied source paths. Workaround: use identical source-path casing on every import. Severity: medium, a bounded rerun-contract failure involving regenerable Markdown, not loss of original documents.

if path.is_file() and not path.is_symlink() or not os.path.lexists(path):
return path
base = f"{src.stem}-{src.suffix.lstrip('.').lower()}"
names = itertools.chain([f"{base}.md"], (f"{base}-{i}.md" for i in itertools.count(2)))
held = {name.casefold() for name in owners} # A-pdf.md and a-pdf.md are one file on a case-insensitive disk
return next(out_dir / n for n in names if n.casefold() not in held and not os.path.lexists(out_dir / n))


def _read_owners(record: Path) -> dict[str, str]:
"""File name -> source, from the record an earlier run left; empty when there is none to use."""
if not os.path.lexists(record):
return {}
if record.is_symlink():
reason = f"{record} is a link"
else:
try:
return {f["name"]: f["source"] for f in load_artifact("docling_sources", record)["files"]}
except InputError as exc:
reason = str(exc)
# the files stay: a rerun writes beside them and starts a new record
logger.warning("Ignoring the record of converted files: %s. The files already in %s are kept, "
"and this run writes new ones beside them", reason, record.parent)
return {}


def _save_owners(record: Path, owners: dict[str, str]) -> None:
if record.is_dir() and not record.is_symlink():
raise InputError(f"{record} is a directory, not the record of converted files: remove it or use another --out-dir")
if record.is_symlink(): # save_artifact would write through it, out of out_dir (#545)
record.unlink()
files = [{"name": name, "source": source} for name, source in sorted(owners.items())]
save_artifact("docling_sources", {"files": files}, record)
Loading
Loading