From 5eac7af7972cc883f04bd31c6b85ffc509a6ea3a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 21:35:22 -0800 Subject: [PATCH 01/20] feat(repo): events.jsonl lines carry a format version, documented with a schema Each line gets v (1); a line without it reads as version 1, so older traces keep working. The version is per line, not a header line: an empty file means the agent never started, and a header would make it non-empty. docs/events-format.md states the promise: within a major version kinds and fields are only added, and readers skip a kind they do not know. The schema repo_event.v1.json pins the lines the converters write. Signed-off-by: Nikolay Petrov --- ...4_120000_napetrov_events-schema-version.md | 12 ++ dokimi/docs/README.md | 2 + dokimi/docs/events-format.md | 67 ++++++++++ dokimi/docs/repo-evaluation-contract.md | 2 + dokimi/dokimi/repos/event_format.py | 25 ++++ dokimi/dokimi/repos/process_metrics.py | 7 +- dokimi/dokimi/repos/transcript.py | 4 +- dokimi/dokimi/schemas/repo_event.v1.json | 46 +++++++ dokimi/mkdocs.yml | 1 + .../fixtures/artifacts/repo_event.v1.json | 9 ++ dokimi/tests/test_repo_event_format.py | 124 ++++++++++++++++++ dokimi/tests/test_repo_reach_bash.py | 2 +- dokimi/tests/test_repo_run.py | 2 +- 13 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120000_napetrov_events-schema-version.md create mode 100644 dokimi/docs/events-format.md create mode 100644 dokimi/dokimi/repos/event_format.py create mode 100644 dokimi/dokimi/schemas/repo_event.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/repo_event.v1.json create mode 100644 dokimi/tests/test_repo_event_format.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_events-schema-version.md b/dokimi/changelog.d/20261114_120000_napetrov_events-schema-version.md new file mode 100644 index 000000000..9a4926880 --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_events-schema-version.md @@ -0,0 +1,12 @@ + + +### Added + +- **`events.jsonl` carries a format version, and the format is documented.** Every line of a + repository run's `events.jsonl` now has `v` (1); a line without it is read as version 1, so + older traces keep working. `docs/events-format.md` describes the lines and the promise that + within a major version kinds and fields are only added, and readers skip a kind they do not + know. The schema is `schemas/repo_event.v1.json`. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index f078ad73d..50bd102dc 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -363,6 +363,8 @@ code. ## Repository evaluation +- [events-format.md](events-format.md) — the normalized `events.jsonl` trace of a + repository run: its lines, the version field and the promise that a version's fields only grow. - [repo-evaluation-contract.md](repo-evaluation-contract.md) — *(proposed, partly implemented)* what a repository provides to be evaluated as a whole: `repo-eval.yaml`, `task.json`, the `grade` entry point and `grade.json`, diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md new file mode 100644 index 000000000..58b89b22a --- /dev/null +++ b/dokimi/docs/events-format.md @@ -0,0 +1,67 @@ +# `events.jsonl` format + +`events.jsonl` is the normalized trace of one agent run in a repository batch +(`repo run`, contract [§4.1](repo-evaluation-contract.md#41-eventsjsonl)). Each +harness's stdout is converted to it, so graders, probes and the security scan +read one format whatever harness ran. A cell's trace lives under its directory +and in `traces/` for `reach` runs. + +## Lines + +One JSON object per line. Every line has a `kind` and a `t`; the other fields +depend on the kind: + +```json +{"kind": "command", "rc": 0, "t": 12.4, "text": "bazel test //x", "v": 1} +{"kind": "read", "t": 3.1, "path": "AGENTS.md", "v": 1} +{"kind": "edit", "t": 5.0, "path": "src/foo.cpp", "v": 1} +{"kind": "tool", "denied": true, "name": "WebFetch", "t": 20.2, "v": 1} +{"kind": "tool", "name": "Skill", "skill": "lint-fix", "t": 2.0, "v": 1} +``` + +| Kind | Fields | +|---|---| +| `command` | `text`; `rc` (integer, or `null` when no result came back); `output` and `output_truncated` when the harness returned output | +| `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | +| `edit` | `path`; `rc`; `deleted` when the file was deleted | +| `tool` | `name`; `skill` on a `Skill` call | + +Any line may also carry `agent` (the id of the parent's `Agent`/`Task` call that +started it), and `denied: true` when the call was refused and never ran. The +exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. + +## Version + +Every line carries `v`: the major version of the format it was written under. +Version 1 is the format this page describes. + +- A line with no `v` was written before the field existed. It has the version 1 + shape and reads as version 1. +- The version is per line, not in a header line. An empty file is what the runner + writes for an agent that never started; a header would make it non-empty. A + line also stays self-describing when a file is sliced or appended to. + +## Promise + +Within a major version, kinds and fields are only added. A reader written for +version 1 keeps working on every later version 1 file, and a field that existed +keeps its name, type and meaning. A change that would break a version 1 reader +(a renamed or removed field, a changed meaning, a changed `null`) is version 2: +a new schema `repo_event.v2.json`, with `v` set to 2 on the lines it writes. A +file may mix versions, since traces are re-derived and appended to. + +## Reading + +- Ignore a kind you do not know. Version 1 readers do so already: a new kind + (say a `turn` event) must not fail an old reader. +- Ignore a field you do not know. +- Accept a line with no `v`. `dokimi.repos.event_format.event_version` returns 1 + for it. +- A line that is not a JSON object is skipped, as the readers in + `dokimi/repos/` do. + +## Null values + +`t` is `null` when the harness writes no timestamps (Codex, Copilot). `rc` is +`null` when no result came back, for example at the turn or budget limit: a +grader must not count `null` as a failure. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c1c0de52..0127b585e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -504,6 +504,8 @@ reads the upstream base (`base_sha`) and the workspace start commit ### 4.1 `events.jsonl` +The line format, its version and the promise that comes with it: [events-format.md](events-format.md). + ```json {"t": 12.4, "kind": "command", "text": "bazel test //cpp/...:x_host", "rc": 0} {"t": 3.1, "kind": "read", "path": "AGENTS.md"} diff --git a/dokimi/dokimi/repos/event_format.py b/dokimi/dokimi/repos/event_format.py new file mode 100644 index 000000000..670aff022 --- /dev/null +++ b/dokimi/dokimi/repos/event_format.py @@ -0,0 +1,25 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The format version of ``events.jsonl`` (docs/events-format.md). + +Each line carries its own ``v``: the major version of the format it was written under. A line with no +``v`` was written before the field existed and reads as version 1. There is no header line: an empty +``events.jsonl`` is what the runner writes for an agent that never started, and a header would make it +non-empty. Per-line versions also keep a line self-describing when a file is sliced or appended to. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +EVENT_VERSION = 1 # the major version this code writes; a version change adds repo_event.vN.json + +#: The kinds the converters write in version 1. Readers must accept any other kind (a later minor addition). +KNOWN_KINDS = frozenset({"command", "read", "edit", "tool"}) + + +def event_version(event: Mapping[str, Any]) -> int: + """The format version of one event; an event with no ``v`` is the unversioned format, read as 1.""" + v = event.get("v") + return v if isinstance(v, int) and not isinstance(v, bool) else 1 diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index ba09f24d6..db4eb7b08 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -5,7 +5,8 @@ The framework counts what every repository asks of a trace: -- ``tool_calls``: every event, so every tool call of the parent and its subagents, and each ``read`` event kept +- ``tool_calls``: every event of a known kind (:data:`~dokimi.repos.event_format.KNOWN_KINDS`), so every tool call + of the parent and its subagents, and each ``read`` event kept from a shell command, a denied one or one of a failed multi-file read (``cat a missing``) too; - ``commands``: the ``command`` events; - ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means @@ -34,6 +35,8 @@ from pathlib import PurePosixPath from typing import Any +from dokimi.repos.event_format import KNOWN_KINDS + COUNTER_FIELDS = ("text", "output", "any") @@ -88,7 +91,7 @@ def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[ reads = {w for e in events if e.get("kind") == "read" and e.get("path") and not e.get("denied") and e.get("rc") in (0, None) and (w := _workspace_path(str(e["path"]))) is not None} return { - "tool_calls": len(events), + "tool_calls": sum(e.get("kind") in KNOWN_KINDS for e in events), "commands": len(commands), "failed_commands": sum(e.get("rc") not in (0, None) and not e.get("denied") for e in commands), "denied_commands": sum(bool(e.get("denied")) for e in commands), diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 7d9408625..73f77faab 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -16,6 +16,8 @@ from pathlib import PurePosixPath, PureWindowsPath from typing import Any +from dokimi.repos.event_format import EVENT_VERSION + OUTPUT_LIMIT = 8192 OUTPUT_KEEP = 4096 @@ -60,7 +62,7 @@ def reads(self) -> set[str]: return {e["path"] for e in self.events if shown_read(e) and e.get("path")} def events_jsonl(self) -> str: - return "".join(json.dumps(e, sort_keys=True) + "\n" for e in self.events) + return "".join(json.dumps({**e, "v": EVENT_VERSION}, sort_keys=True) + "\n" for e in self.events) def shown_read(event: dict[str, Any]) -> bool: diff --git a/dokimi/dokimi/schemas/repo_event.v1.json b/dokimi/dokimi/schemas/repo_event.v1.json new file mode 100644 index 000000000..fd6d569e7 --- /dev/null +++ b/dokimi/dokimi/schemas/repo_event.v1.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/repo_event.v1.json", + "title": "One line of events.jsonl, format version 1", + "description": "A line of a repository run's normalized trace (docs/events-format.md). One JSON object per line; this schema validates one line. Within version 1 a kind and a field are only ever added, so a reader must ignore a kind it does not know (this schema allows any kind) and a field it does not know. A line with no `v` is an unversioned line written before the version field, and has the same shape as a version 1 line.", + "type": "object", + "properties": { + "v": {"const": 1, "description": "The format version of this line. Absent on a line written before the field existed."}, + "kind": {"type": "string", "description": "What the event is. Readers ignore a kind they do not know."}, + "t": {"type": ["number", "null"], "description": "Seconds since the first timestamped trace line; null when the harness has no timestamps."}, + "agent": {"type": "string", "description": "The id of the parent's Agent/Task call that started this event; absent on the parent's events."}, + "denied": {"type": "boolean", "description": "The call was refused and never ran."}, + "output": {"type": "string", "description": "A command's output as the harness returned it, clipped to 8 KB."}, + "output_truncated": {"type": "boolean"} + }, + "required": ["kind", "t"], + "allOf": [ + { + "if": {"properties": {"kind": {"const": "command"}}, "required": ["kind"]}, + "then": { + "required": ["text", "rc"], + "properties": {"text": {"type": "string"}, "rc": {"type": ["integer", "null"]}} + } + }, + { + "if": {"properties": {"kind": {"const": "read"}}, "required": ["kind"]}, + "then": { + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}} + } + }, + { + "if": {"properties": {"kind": {"const": "edit"}}, "required": ["kind"]}, + "then": { + "required": ["path"], + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}, "deleted": {"type": "boolean"}} + } + }, + { + "if": {"properties": {"kind": {"const": "tool"}}, "required": ["kind"]}, + "then": { + "required": ["name"], + "properties": {"name": {"type": "string"}, "skill": {"type": "string"}} + } + } + ] +} diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index e6b963f81..95af4a915 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -129,6 +129,7 @@ nav: - grading-tiers.md - Repository evaluation: - repo-evaluation-contract.md + - events-format.md - behavioural-security.md - repo-evaluation-mergeability.md - repo-evaluation-agent-modes.md diff --git a/dokimi/tests/fixtures/artifacts/repo_event.v1.json b/dokimi/tests/fixtures/artifacts/repo_event.v1.json new file mode 100644 index 000000000..5ca40ad89 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/repo_event.v1.json @@ -0,0 +1,9 @@ +{ + "kind": "command", + "output": "ERROR: icpx is not found", + "output_truncated": false, + "rc": 1, + "t": 31.0, + "text": "bazel test :x", + "v": 1 +} diff --git a/dokimi/tests/test_repo_event_format.py b/dokimi/tests/test_repo_event_format.py new file mode 100644 index 000000000..2f325195f --- /dev/null +++ b/dokimi/tests/test_repo_event_format.py @@ -0,0 +1,124 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The version field and the schema of ``events.jsonl`` (docs/events-format.md). + +The schema (``repo_event.v1.json``) pins what version 1 lines look like; the converters' output +must validate against it, an unversioned line must still read, and a reader must skip a kind it +does not know. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi.artifacts import SCHEMA_DIR, schema_errors +from dokimi.repos.event_format import EVENT_VERSION, KNOWN_KINDS, event_version +from dokimi.repos.events import convert +from dokimi.repos.process_metrics import process_metrics + +DOCS = Path(__file__).resolve().parents[1] / "docs" / "events-format.md" + + +def _lines(*records: dict) -> str: + return "\n".join(json.dumps(r) for r in records) + "\n" + + +def _claude_trace() -> str: + usage = {"input_tokens": 3, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 100, "output_tokens": 5} + + def call(msg_id: str, *blocks: dict, stamp: str) -> dict: + return {"type": "assistant", "timestamp": stamp, + "message": {"id": msg_id, "model": "m", "content": list(blocks), "usage": usage}} + + def result(tool_id: str, text: str, stamp: str) -> dict: + return {"type": "user", "timestamp": stamp, + "message": {"content": [{"type": "tool_result", "tool_use_id": tool_id, "content": text}]}} + + return _lines( + {"type": "system", "subtype": "init", "model": "m", "claude_code_version": "2.0.0", "tools": []}, + call("a", {"type": "tool_use", "id": "b1", "name": "Bash", "input": {"command": "cat AGENTS.md"}}, + stamp="2026-11-14T10:00:00Z"), + result("b1", "rules", "2026-11-14T10:00:01Z"), + call("b", {"type": "tool_use", "id": "e1", "name": "Edit", "input": {"file_path": "src/a.py"}}, + {"type": "tool_use", "id": "w1", "name": "WebFetch", "input": {"url": "x"}}, + stamp="2026-11-14T10:00:02Z"), + result("e1", "ok", "2026-11-14T10:00:03Z"), + result("w1", "denied", "2026-11-14T10:00:03Z"), + {"type": "result", "result": "done", "num_turns": 2, "is_error": False, "subtype": "success", + "usage": {"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100}}, + ) + + +def _codex_trace() -> str: + return _lines( + {"type": "item.started", "item": {"id": "i1", "type": "command_execution", "command": "cat src/a.py"}}, + {"type": "item.completed", "item": {"id": "i1", "type": "command_execution", + "command": "cat src/a.py", "aggregated_output": "x", "exit_code": 0}}, + {"type": "item.completed", "item": {"id": "m1", "type": "agent_message", "text": "done"}}, + {"type": "turn.completed", "usage": {"input_tokens": 10, "cached_input_tokens": 2, "output_tokens": 3}}, + ) + + +def _converted(raw_format: str, raw: str) -> list[dict]: + """The events of a converted trace as the writer puts them on the lines of ``events.jsonl``.""" + return [json.loads(line) for line in convert(raw_format, raw).events_jsonl().splitlines()] + + +def test_the_schema_accepts_the_committed_fixture(): + fixture = json.loads((Path(__file__).parent / "fixtures" / "artifacts" / "repo_event.v1.json").read_text()) + assert schema_errors("repo_event.v1", fixture) == [] + + +def test_every_converted_line_is_versioned_and_validates(): + lines = [*_converted("claude-stream-json", _claude_trace()), *_converted("codex-json", _codex_trace())] + assert {e["kind"] for e in lines} == KNOWN_KINDS # the fixtures cover every kind the schema names + for event in lines: + assert event["v"] == EVENT_VERSION + assert schema_errors("repo_event.v1", event) == [], event + + +def test_the_writer_only_adds_the_version_to_each_event(): + transcript = convert("claude-stream-json", _claude_trace()) + written = [json.loads(line) for line in transcript.events_jsonl().splitlines()] + assert [{k: v for k, v in e.items() if k != "v"} for e in written] == transcript.events + + +def test_an_unversioned_line_is_accepted_and_reads_as_version_1(): + legacy = {"t": 1.0, "kind": "command", "text": "ls", "rc": 0} + assert schema_errors("repo_event.v1", legacy) == [] + assert event_version(legacy) == 1 + + +def test_a_line_of_a_later_major_version_is_not_a_version_1_line(): + later = {"t": 1.0, "kind": "command", "text": "ls", "rc": 0, "v": 2} + assert schema_errors("repo_event.v1", later) != [] + assert event_version(later) == 2 + + +def test_a_reader_skips_a_kind_it_does_not_know(): + future = {"t": 1.0, "kind": "probe", "name": "p", "v": 1} + assert schema_errors("repo_event.v1", future) == [] # the schema admits any kind + base = [{"t": 1.0, "kind": "command", "text": "ls", "rc": 0}] + assert process_metrics([*base, future], lambda _: False)["tool_calls"] == process_metrics(base, lambda _: False)["tool_calls"] + assert process_metrics([*base, future], lambda _: False)["commands"] == 1 + + +def test_unversioned_and_converted_give_the_same_process_metrics(): + versioned = _converted("claude-stream-json", _claude_trace()) + unversioned = [{k: v for k, v in e.items() if k != "v"} for e in versioned] + assert process_metrics(versioned, lambda p: p == "AGENTS.md") == process_metrics(unversioned, lambda p: p == "AGENTS.md") + + +def test_the_schema_names_exactly_the_known_kinds(): + schema = json.loads((SCHEMA_DIR / "repo_event.v1.json").read_text(encoding="utf-8")) + named = {clause["if"]["properties"]["kind"]["const"] for clause in schema["allOf"]} + assert named == KNOWN_KINDS + + +def test_the_doc_names_every_known_kind_and_the_version_field(): + text = DOCS.read_text(encoding="utf-8") + for kind in sorted(KNOWN_KINDS): + assert f"| `{kind}` |" in text, kind + assert "`v`" in text and "repo_event.v2.json" in text diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 798c8b584..9a95700eb 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -216,7 +216,7 @@ def test_a_bash_read_of_the_probe_shows_whether_the_directory_guidance_loads(tmp assert [s["id"] for s in payload["scenarios"]] == ["bash=sub"] and not payload["scenarios"][0]["error"] assert payload["summary"]["matrix"] == {"AGENTS.md": {"bash=sub": "loaded"}, "sub/AGENTS.md": {"bash=sub": cls}} events = [json.loads(x) for x in (tmp_path / "out/traces/00-bash_sub.events.jsonl").read_text().splitlines()] - assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0} + assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0, "v": 1} @pytest.mark.parametrize("mode", ["hook", "permissions"]) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 78c0f3f58..f816d6b01 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -57,7 +57,7 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert meta["agent_network"] == "host" assert meta["egress_proxy"] is False and len(meta["start_sha"]) == 40 events = [json.loads(line) for line in (rd / "events.jsonl").read_text().splitlines()] - assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None}] # relative to /eval/run/workspace + assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None, "v": 1}] # relative to /eval/run/workspace assert (rd / "answer.txt").read_text() == "done: src/calc.py" assert (rd / "config/settings.json").read_text() == '{"hook": "python3 /eval/support/no_background.py"}' result = json.loads((rd / "result.json").read_text()) From 482a728b715e82b1656030c88c4a616cc410e272 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 21:46:07 -0800 Subject: [PATCH 02/20] feat(repo): turn and compaction events per model call in events.jsonl A Claude run's turns come from the parse its token totals use (context/trace.py, exposed per call with the message timestamp), so the turns of a complete trace sum to the run's input, output and cache totals. Codex's come from its turn.completed usage, with a count it did not report as null; Copilot's trace has no usage and writes no turn events. A compaction names the turn it came after. process.tool_calls counts the tool kinds only, so the new kinds do not change it. Existing kinds and their fields are unchanged. Signed-off-by: Nikolay Petrov --- ...61114_120500_napetrov_trace-turn-events.md | 13 ++ dokimi/docs/events-format.md | 21 ++- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/context/trace.py | 15 +- dokimi/dokimi/repos/cli_traces.py | 18 +-- dokimi/dokimi/repos/event_format.py | 6 +- dokimi/dokimi/repos/events.py | 5 +- dokimi/dokimi/repos/process_metrics.py | 10 +- dokimi/dokimi/repos/turn_events.py | 71 +++++++++ dokimi/dokimi/schemas/repo_event.v1.json | 28 +++- dokimi/tests/test_context_trace.py | 6 +- dokimi/tests/test_repo_cli_pins.py | 2 +- dokimi/tests/test_repo_cli_traces.py | 5 +- dokimi/tests/test_repo_event_format.py | 1 + dokimi/tests/test_repo_turn_events.py | 148 ++++++++++++++++++ 15 files changed, 320 insertions(+), 31 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md create mode 100644 dokimi/dokimi/repos/turn_events.py create mode 100644 dokimi/tests/test_repo_turn_events.py diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md new file mode 100644 index 000000000..419c78586 --- /dev/null +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -0,0 +1,13 @@ + + +### Added + +- **`events.jsonl` has a `turn` event per model call and a `compaction` event per compaction.** + A Claude run's turns come from the parse its token totals use, so their sums are the run's + `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens`; a subagent's + turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report + as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts + the tool events only. The format is documented in `docs/events-format.md`. diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 58b89b22a..007d7bd85 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -26,6 +26,24 @@ depend on the kind: | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | +| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count | +| `compaction` | `turn`: the index of the turn it came after, `null` before the first call | + +A `turn` is one model call, so a Claude run's trace has one per call. The sums +of a complete Claude trace's `turn` events are the run's `result.json` token +totals (`input_tokens`, `output_tokens`, `cache_read_tokens`, +`cache_write_tokens`), over the same scope: with `usage_scope` `tree` the events +of subagents are in the sum (they carry `agent`), with `main` they are not. Codex +reports usage on each `turn.completed`, one `turn` event per turn, with `t` +`null`; a count Codex did not report is `null`. Copilot's trace carries no +usage, so it writes no `turn` events and its token counts stay `null`. +`compaction` is the point where Claude Code summarised the history; it has no +tokens of its own. + +Turn and compaction events are not tool calls: `result.json` `process.tool_calls` +counts the tool kinds only (`command`, `read`, `edit`, `tool`). A reader that +reads the tool path of a run keeps to those kinds too. + Any line may also carry `agent` (the id of the parent's `Agent`/`Task` call that started it), and `denied: true` when the call was refused and never ran. The exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. @@ -33,7 +51,8 @@ exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. ## Version Every line carries `v`: the major version of the format it was written under. -Version 1 is the format this page describes. +Version 1 is the format this page describes; it has the `turn` and `compaction` +kinds, added to it within version 1. - A line with no `v` was written before the field existed. It has the version 1 shape and reads as version 1. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 0127b585e..e8c1c6421 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -551,7 +551,7 @@ parent's and subagents' together, so a grader need not: | Key | Meaning | |---|---| -| `tool_calls` | every event: each tool call the agent made, and each shell-derived `read` event kept, a denied one or one of a failed multi-file read too (see `guidance_read` and §7) | +| `tool_calls` | every tool event (`command`, `read`, `edit`, `tool`, not the `turn` and `compaction` events of [events-format.md](events-format.md)): each tool call the agent made, and each shell-derived `read` event kept, a denied one or one of a failed multi-file read too (see `guidance_read` and §7) | | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | diff --git a/dokimi/dokimi/context/trace.py b/dokimi/dokimi/context/trace.py index a5c1f81d6..2eee109c7 100644 --- a/dokimi/dokimi/context/trace.py +++ b/dokimi/dokimi/context/trace.py @@ -13,7 +13,8 @@ so the first call's prompt is everything loaded before the agent did anything (system prompt, tool schemas, memory files, skill listing, the task), and the series after it is how the history grew. ``new_tokens`` (input + cache creation) is what the call added beyond the cached -prefix. +prefix. Each call also keeps its parts (``input_tokens``, ``cache_read_tokens``, ``cache_write_tokens``) and +its message timestamp (``at``): the per-turn events of ``events.jsonl`` read them (:mod:`dokimi.repos.turn_events`). Calls a subagent made carry ``parent_tool_use_id``; they run in their own window, so the summary reads the main thread and counts subagent calls apart. A ``compact_boundary`` system @@ -38,7 +39,8 @@ # Tools whose call can change what a later identical call returns; one ends every earlier repeat. MUTATING = frozenset({"Bash", "Edit", "MultiEdit", "NotebookEdit", "Write"}) -TOKEN_FIELDS = ("prompt_tokens", "new_tokens", "output_tokens") # the counts of one call +TOKEN_FIELDS = ("prompt_tokens", "new_tokens", "output_tokens", "input_tokens", "cache_read_tokens", + "cache_write_tokens") # the counts of one call class TraceError(ValueError): @@ -165,12 +167,13 @@ def _apply(out: ContextTrace, rec: dict[str, Any], by_message: dict[str, dict[st "pre_tokens": meta.get("pre_tokens", meta.get("preTokens")), "post_tokens": meta.get("post_tokens"), # newer CLIs only "after_call": len(out.calls) - 1, + "at": rec.get("timestamp"), }) elif kind == "assistant": message = rec.get("message") or {} usage = message.get("usage") if isinstance(usage, dict): - call = _call(usage, agent) + call = _call(usage, agent, rec.get("timestamp")) key = message.get("id") if key and key in by_message: _merge(by_message[key], call) # a later block of the same call: same prompt, fuller output @@ -216,15 +219,19 @@ def _merge(call: dict[str, Any], later: dict[str, Any]) -> None: call[k] = max(call[k], later[k]) -def _call(usage: dict[str, Any], agent: Any) -> dict[str, Any]: +def _call(usage: dict[str, Any], agent: Any, at: Any) -> dict[str, Any]: fresh = int(usage.get("input_tokens") or 0) created = int(usage.get("cache_creation_input_tokens") or 0) read = int(usage.get("cache_read_input_tokens") or 0) return { "agent": agent, + "at": at, # the message's timestamp, as the trace has it; the first block's "prompt_tokens": fresh + created + read, "new_tokens": fresh + created, "output_tokens": int(usage.get("output_tokens") or 0), + "input_tokens": fresh, + "cache_read_tokens": read, + "cache_write_tokens": created, } diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index e59477d4f..74811cc02 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -31,6 +31,7 @@ from .shell_words import (_DIRS, _KEYWORDS, _case_closes, _case_opens, _cd, _command_words, _is_activate, _is_cd, _loses, _run_name, _runs_dir, _time_keyword, _unquoted) from .transcript import Transcript, _from_cwd, _parse_time, clip_output, relative_path +from .turn_events import codex_turn _BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) _PRINTERS = {"cat", "head", "tail", "nl", "sed"} @@ -530,17 +531,16 @@ def _codex_apply(out: Transcript, rec: dict[str, Any], messages: list[str], work def _codex_usage(out: Transcript, usage: dict[str, Any]) -> None: + turn = codex_turn(out.num_turns or 0, usage) # its turn event, and the totals: both from one reading out.num_turns = (out.num_turns or 0) + 1 out.result_seen = True # a turn ended: the agent started, whatever else it printed - fresh, produced = int(usage.get("input_tokens") or 0), int(usage.get("output_tokens") or 0) - cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit - if fresh or produced: # a missing or all-zero usage is not a measured zero: the counts stay None - # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens. - # A malformed record with more cached than input never goes negative - out.input_tokens = (out.input_tokens or 0) + max(fresh - cached, 0) - out.output_tokens = (out.output_tokens or 0) + produced - if usage.get("cached_input_tokens") is not None: # reported, so a zero is measured; absent, it stays None - out.cache_read_tokens = (out.cache_read_tokens or 0) + cached + out.events.append(turn) + if turn["input"] is not None: # a missing or all-zero usage is not a measured zero: the counts stay None + # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens + out.input_tokens = (out.input_tokens or 0) + turn["input"] + out.output_tokens = (out.output_tokens or 0) + turn["output"] + if turn["cache_read"] is not None: # reported, so a zero is measured; absent, it stays None + out.cache_read_tokens = (out.cache_read_tokens or 0) + turn["cache_read"] out.usage_scope = "main" # the turns are the parent's; Codex reports no subagent tokens here diff --git a/dokimi/dokimi/repos/event_format.py b/dokimi/dokimi/repos/event_format.py index 670aff022..80c76a92d 100644 --- a/dokimi/dokimi/repos/event_format.py +++ b/dokimi/dokimi/repos/event_format.py @@ -15,8 +15,12 @@ EVENT_VERSION = 1 # the major version this code writes; a version change adds repo_event.vN.json +#: The kinds that are tool activity: what ``tool_calls`` counts. +TOOL_KINDS = frozenset({"command", "read", "edit", "tool"}) +#: The per-model-call kinds (:mod:`dokimi.repos.turn_events`): not tool calls. +TURN_KINDS = frozenset({"turn", "compaction"}) #: The kinds the converters write in version 1. Readers must accept any other kind (a later minor addition). -KNOWN_KINDS = frozenset({"command", "read", "edit", "tool"}) +KNOWN_KINDS = TOOL_KINDS | TURN_KINDS def event_version(event: Mapping[str, Any]) -> int: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 956171d08..151081b95 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -47,6 +47,7 @@ from dokimi.context.trace import parse_trace from .cli_traces import _failed_read, codex_json, copilot_json, shell_walk +from .turn_events import claude_events from .transcript import Transcript, _parse_time, clip_output, relative_path TRACE_FORMATS = ("claude-stream-json", "codex-json", "copilot-json", "text") @@ -125,8 +126,10 @@ def _claude_stream_json(raw: str, workdir: str | None, cwd: str = "") -> Transcr out.errors.append(f"line {n}: unexpected shape ({exc})") _walk_bash(out, stream) _settle_bash(out, stream.bash.values()) - main = parse_trace(raw).main_calls() + trace = parse_trace(raw) + main = trace.main_calls() out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None + out.events.extend(claude_events(trace, start)) return out diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index db4eb7b08..5f01ee31b 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -5,9 +5,9 @@ The framework counts what every repository asks of a trace: -- ``tool_calls``: every event of a known kind (:data:`~dokimi.repos.event_format.KNOWN_KINDS`), so every tool call - of the parent and its subagents, and each ``read`` event kept - from a shell command, a denied one or one of a failed multi-file read (``cat a missing``) too; +- ``tool_calls``: every tool event (:data:`~dokimi.repos.event_format.TOOL_KINDS`), so every tool call of the + parent and its subagents, and each ``read`` event kept from a shell command, a denied one or one of a + failed multi-file read (``cat a missing``) too; - ``commands``: the ``command`` events; - ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means no result came back, which is not a failure; a permission denial carries ``rc`` 1 but never ran); @@ -35,7 +35,7 @@ from pathlib import PurePosixPath from typing import Any -from dokimi.repos.event_format import KNOWN_KINDS +from dokimi.repos.event_format import TOOL_KINDS COUNTER_FIELDS = ("text", "output", "any") @@ -91,7 +91,7 @@ def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[ reads = {w for e in events if e.get("kind") == "read" and e.get("path") and not e.get("denied") and e.get("rc") in (0, None) and (w := _workspace_path(str(e["path"]))) is not None} return { - "tool_calls": sum(e.get("kind") in KNOWN_KINDS for e in events), + "tool_calls": sum(e.get("kind") in TOOL_KINDS for e in events), "commands": len(commands), "failed_commands": sum(e.get("rc") not in (0, None) and not e.get("denied") for e in commands), "denied_commands": sum(bool(e.get("denied")) for e in commands), diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py new file mode 100644 index 000000000..31f4fa60c --- /dev/null +++ b/dokimi/dokimi/repos/turn_events.py @@ -0,0 +1,71 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The per-model-call events of ``events.jsonl``: ``turn`` and ``compaction`` (docs/events-format.md). + +A ``turn`` is one model call and the tokens it was billed: ``input`` (uncached), ``output``, +``cache_read``, ``cache_write`` and ``prompt_tokens`` (the whole prompt, ``input + cache_read + +cache_write``). Claude's come from :mod:`dokimi.context.trace`, the parse the run's own token totals +use, so the turns of a complete trace sum to those totals. Codex's come from its ``turn.completed`` +usage. Copilot's trace carries no usage, so it has no turn events. A count the harness did not report +is ``null``, never ``0``. + +A ``compaction`` follows the turn it came after; its ``turn`` is that turn's index, or ``null`` before +the first call. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from dokimi.context.trace import ContextTrace + +from .transcript import _parse_time + + +def claude_events(trace: ContextTrace, start: datetime | None) -> list[dict[str, Any]]: + """The turn events of every model call (a subagent's carry ``agent``), then the compactions.""" + events: list[dict[str, Any]] = [] + for call in trace.calls: + event: dict[str, Any] = { + "kind": "turn", "t": _seconds(call["at"], start), "turn": call["i"], + "input": call["input_tokens"], "output": call["output_tokens"], + "cache_read": call["cache_read_tokens"], "cache_write": call["cache_write_tokens"], + "prompt_tokens": call["prompt_tokens"], + } + if call["agent"]: + event["agent"] = call["agent"] + events.append(event) + for compaction in trace.compactions: + after = compaction["after_call"] + events.append({"kind": "compaction", "t": _seconds(compaction["at"], start), + "turn": after if after >= 0 else None}) + return events + + +def codex_turn(turn: int, usage: Mapping[str, Any]) -> dict[str, Any]: + """The turn event of one Codex ``turn.completed`` usage. Codex counts the cached part inside + ``input_tokens``, so ``input`` is the uncached rest and ``prompt_tokens`` the reported total. + A usage with no tokens is no measured turn: every count is ``null``.""" + fresh = int(usage.get("input_tokens") or 0) + produced = int(usage.get("output_tokens") or 0) + cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit + measured = bool(fresh or produced) # a missing or all-zero usage is not a measured zero + reported = usage.get("cached_input_tokens") is not None # reported, so a zero is measured + return { + "kind": "turn", "t": None, "turn": turn, + "input": max(fresh - cached, 0) if measured else None, + "output": produced if measured else None, + "cache_read": cached if measured and reported else None, + "cache_write": None, # Codex reports no cache writes + "prompt_tokens": fresh if measured else None, + } + + +def _seconds(stamp: Any, start: datetime | None) -> float | None: + when = _parse_time(stamp) + if when is None or start is None: + return None + return round((when - start).total_seconds(), 3) diff --git a/dokimi/dokimi/schemas/repo_event.v1.json b/dokimi/dokimi/schemas/repo_event.v1.json index fd6d569e7..9df4c925b 100644 --- a/dokimi/dokimi/schemas/repo_event.v1.json +++ b/dokimi/dokimi/schemas/repo_event.v1.json @@ -10,7 +10,6 @@ "t": {"type": ["number", "null"], "description": "Seconds since the first timestamped trace line; null when the harness has no timestamps."}, "agent": {"type": "string", "description": "The id of the parent's Agent/Task call that started this event; absent on the parent's events."}, "denied": {"type": "boolean", "description": "The call was refused and never ran."}, - "output": {"type": "string", "description": "A command's output as the harness returned it, clipped to 8 KB."}, "output_truncated": {"type": "boolean"} }, "required": ["kind", "t"], @@ -19,7 +18,11 @@ "if": {"properties": {"kind": {"const": "command"}}, "required": ["kind"]}, "then": { "required": ["text", "rc"], - "properties": {"text": {"type": "string"}, "rc": {"type": ["integer", "null"]}} + "properties": { + "text": {"type": "string"}, + "rc": {"type": ["integer", "null"]}, + "output": {"type": "string", "description": "The command's output as the harness returned it, clipped to 8 KB."} + } } }, { @@ -41,6 +44,27 @@ "required": ["name"], "properties": {"name": {"type": "string"}, "skill": {"type": "string"}} } + }, + { + "if": {"properties": {"kind": {"const": "turn"}}, "required": ["kind"]}, + "then": { + "required": ["turn", "input", "output", "cache_read", "cache_write", "prompt_tokens"], + "properties": { + "turn": {"type": "integer", "minimum": 0}, + "input": {"type": ["integer", "null"]}, + "output": {"type": ["integer", "null"]}, + "cache_read": {"type": ["integer", "null"]}, + "cache_write": {"type": ["integer", "null"]}, + "prompt_tokens": {"type": ["integer", "null"]} + } + } + }, + { + "if": {"properties": {"kind": {"const": "compaction"}}, "required": ["kind"]}, + "then": { + "required": ["turn"], + "properties": {"turn": {"type": ["integer", "null"], "minimum": 0}} + } } ] } diff --git a/dokimi/tests/test_context_trace.py b/dokimi/tests/test_context_trace.py index b989c0044..8da20430a 100644 --- a/dokimi/tests/test_context_trace.py +++ b/dokimi/tests/test_context_trace.py @@ -94,7 +94,7 @@ def test_tool_results_failures_and_repeats(): assert s["repeated_tool_calls"] == 1 assert s["tool_result_chars"] == 300 + 300 + len("File does not exist.") + 50 + 4 assert s["largest_tool_results"][0] == {"tool": "Bash", "calls": 3, "chars": 650, "max_chars": 300} - assert trace.compactions == [{"trigger": "auto", "pre_tokens": 20815, "post_tokens": 9000, "after_call": 2}] + assert trace.compactions == [{"trigger": "auto", "pre_tokens": 20815, "post_tokens": 9000, "after_call": 2, "at": None}] def test_without_a_window_there_is_no_fill(): @@ -342,8 +342,8 @@ def test_camel_case_and_bare_compactions(): {"type": "system", "subtype": "compact_boundary"}, )) assert got.compactions == [ - {"trigger": "manual", "pre_tokens": 7, "post_tokens": None, "after_call": 0}, - {"trigger": None, "pre_tokens": None, "post_tokens": None, "after_call": 0}, + {"trigger": "manual", "pre_tokens": 7, "post_tokens": None, "after_call": 0, "at": None}, + {"trigger": None, "pre_tokens": None, "post_tokens": None, "after_call": 0, "at": None}, ] diff --git a/dokimi/tests/test_repo_cli_pins.py b/dokimi/tests/test_repo_cli_pins.py index 58406abf0..dae5b9b49 100644 --- a/dokimi/tests/test_repo_cli_pins.py +++ b/dokimi/tests/test_repo_cli_pins.py @@ -81,7 +81,7 @@ def test_a_turn_marks_the_run_as_started_even_with_no_event(): codex = convert("codex-json", _jsonl({"type": "turn.completed", "usage": {"input_tokens": 1, "output_tokens": 1}}), "/ws") copilot = convert("copilot-json", _jsonl({"type": "assistant.turn_start", "data": {}}), "/ws") for trace in (codex, copilot): - assert trace.events == [] and trace.result_seen is True + assert [e for e in trace.events if e["kind"] != "turn"] == [] and trace.result_seen is True assert convert("codex-json", "", "/ws").result_seen is False assert convert("copilot-json", "", "/ws").result_seen is False for raw in ('{"type":"error","message":"x"}\n', "not json at all\n", '{"type":"other"}\n'): diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fe9d5c381..911ff624a 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -419,7 +419,7 @@ def test_codex_json_to_events(): t = convert("codex-json", raw, "/ws") assert t.answer == "DONE" and t.num_turns == 1 and (t.input_tokens, t.output_tokens) == (100, 7) assert t.cost_usd is None # Codex reports no cost: the batch budget counts the cap - assert t.events == [ + assert [e for e in t.events if e["kind"] != "turn"] == [ {"t": None, "kind": "read", "path": "AGENTS.md"}, {"t": None, "kind": "command", "text": "cat /ws/AGENTS.md", "rc": 0, "output": "# guide\n", "output_truncated": False}, @@ -554,8 +554,7 @@ def test_codex_incomplete_command_is_recorded_with_rc_null(): {"type": "turn.completed", "usage": {"input_tokens": 10, "output_tokens": 1}}, ) t = convert("codex-json", raw, None) - assert len(t.events) == 1 - assert t.events[0] == {"t": None, "kind": "command", "text": "make build", "rc": None} + assert [e for e in t.events if e["kind"] != "turn"] == [{"t": None, "kind": "command", "text": "make build", "rc": None}] def test_codex_failed_command_does_not_count_as_a_read(): diff --git a/dokimi/tests/test_repo_event_format.py b/dokimi/tests/test_repo_event_format.py index 2f325195f..61df84d91 100644 --- a/dokimi/tests/test_repo_event_format.py +++ b/dokimi/tests/test_repo_event_format.py @@ -46,6 +46,7 @@ def result(tool_id: str, text: str, stamp: str) -> dict: stamp="2026-11-14T10:00:02Z"), result("e1", "ok", "2026-11-14T10:00:03Z"), result("w1", "denied", "2026-11-14T10:00:03Z"), + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:04Z"}, {"type": "result", "result": "done", "num_turns": 2, "is_error": False, "subtype": "success", "usage": {"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100}}, ) diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py new file mode 100644 index 000000000..2550c7f11 --- /dev/null +++ b/dokimi/tests/test_repo_turn_events.py @@ -0,0 +1,148 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``turn`` and ``compaction`` events of ``events.jsonl`` (docs/events-format.md). + +The acceptance rule: the turns of a complete Claude trace sum to the run's own token totals, in the +scope the totals were taken in (tree with subagents, or the parent's alone). +""" + +from __future__ import annotations + +import json + +from dokimi.artifacts import schema_errors +from dokimi.context.consistency import run_from_events +from dokimi.repos.event_format import TOOL_KINDS +from dokimi.repos.events import convert +from dokimi.repos.process_metrics import process_metrics + +TURN_FIELDS = ("input", "output", "cache_read", "cache_write") + + +def _lines(*records: dict) -> str: + return "\n".join(json.dumps(r) for r in records) + "\n" + + +def _usage(inp: int, out: int, read: int, create: int) -> dict: + return {"input_tokens": inp, "output_tokens": out, "cache_read_input_tokens": read, + "cache_creation_input_tokens": create} + + +def _call(msg_id: str, usage: dict, stamp: str, *, blocks: list | None = None, parent: str | None = None) -> dict: + record = {"type": "assistant", "timestamp": stamp, + "message": {"id": msg_id, "model": "m", "content": blocks or [], "usage": usage}} + if parent is not None: + record["parent_tool_use_id"] = parent + return record + + +def _claude_run(*, tree: bool) -> str: + """Three model calls: the parent's first call split over two records (one message id, a fuller output on the + second), a subagent's call, then the parent's second call; a compaction after the first call.""" + main_a = _usage(3, 5, 100, 0) + sub = _usage(2, 4, 50, 10) + main_b = _usage(1, 9, 120, 20) + if tree: # the result's modelUsage: the scope the totals sum over subagents + totals = {"inputTokens": 6, "outputTokens": 20, "cacheReadInputTokens": 270, "cacheCreationInputTokens": 30} + result_usage: dict = {"modelUsage": {"m": totals}} + else: # the parent's usage alone + result_usage = {"usage": {"input_tokens": 4, "output_tokens": 16, "cache_read_input_tokens": 220, + "cache_creation_input_tokens": 20}} + return _lines( + {"type": "system", "subtype": "init", "model": "m", "claude_code_version": "2.0.0", "tools": []}, + _call("a", main_a, "2026-11-14T10:00:00Z", blocks=[{"type": "text", "text": "thinking"}]), + _call("a", {**main_a, "output_tokens": 7}, "2026-11-14T10:00:00.500Z", + blocks=[{"type": "tool_use", "id": "agent-1", "name": "Agent", "input": {"prompt": "x"}}]), + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:03Z", + "compact_metadata": {"trigger": "auto", "pre_tokens": 150000}}, + _call("s", sub, "2026-11-14T10:00:02Z", parent="agent-1"), + _call("b", main_b, "2026-11-14T10:00:05Z"), + {"type": "result", "result": "done", "num_turns": 2, "is_error": False, "subtype": "success", + **result_usage}, + ) + + +def _turns(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] == "turn"] + + +def test_the_turns_of_a_tree_scope_run_sum_to_its_totals(): + transcript = convert("claude-stream-json", _claude_run(tree=True)) + turns = _turns(transcript.events) + assert transcript.usage_scope == "tree" + assert [sum(t[f] for t in turns) for f in TURN_FIELDS] == [ + transcript.input_tokens, transcript.output_tokens, transcript.cache_read_tokens, transcript.cache_write_tokens] + + +def test_the_turns_of_a_main_scope_run_sum_to_the_parents_totals(): + transcript = convert("claude-stream-json", _claude_run(tree=False)) + parent = [t for t in _turns(transcript.events) if "agent" not in t] + assert transcript.usage_scope == "main" + assert [sum(t[f] for t in parent) for f in TURN_FIELDS] == [ + transcript.input_tokens, transcript.output_tokens, transcript.cache_read_tokens, transcript.cache_write_tokens] + assert [t["agent"] for t in _turns(transcript.events) if "agent" in t] == ["agent-1"] + + +def test_one_turn_per_model_call_with_its_prompt_and_its_time(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + turns = _turns(events) + assert [t["turn"] for t in turns] == [0, 1, 2] # the subagent's call counts; the split call is one call + assert [t["output"] for t in turns] == [7, 4, 9] # the fuller of the two records + assert [t["prompt_tokens"] for t in turns] == [103, 62, 141] # input + cache_read + cache_write + assert [t["t"] for t in turns] == [0.0, 2.0, 5.0] # seconds from the first timestamped line + + +def test_a_compaction_follows_the_turn_it_came_after(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + assert [e for e in events if e["kind"] == "compaction"] == [{"kind": "compaction", "t": 3.0, "turn": 0}] + + +def test_a_compaction_before_any_call_has_no_turn(): + raw = _lines({"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:00Z"}) + assert [e for e in convert("claude-stream-json", raw).events if e["kind"] == "compaction"] == [ + {"kind": "compaction", "t": 0.0, "turn": None}] + + +def test_codex_turns_come_from_turn_completed_and_a_missing_count_is_null(): + raw = _lines( + {"type": "turn.completed", "usage": {"input_tokens": 10, "cached_input_tokens": 4, "output_tokens": 3}}, + {"type": "turn.completed", "usage": {"input_tokens": 6, "output_tokens": 2}}, + {"type": "turn.completed"}, + ) + transcript = convert("codex-json", raw) + turns = _turns(transcript.events) + assert turns == [ + {"kind": "turn", "t": None, "turn": 0, "input": 6, "output": 3, "cache_read": 4, "cache_write": None, + "prompt_tokens": 10}, + {"kind": "turn", "t": None, "turn": 1, "input": 6, "output": 2, "cache_read": None, "cache_write": None, + "prompt_tokens": 6}, + {"kind": "turn", "t": None, "turn": 2, "input": None, "output": None, "cache_read": None, "cache_write": None, + "prompt_tokens": None}, + ] + assert transcript.input_tokens == 12 and transcript.output_tokens == 5 and transcript.cache_read_tokens == 4 + + +def test_copilot_writes_no_turn_events(): + raw = _lines({"type": "assistant.message", "data": {"content": "done"}}) + assert _turns(convert("copilot-json", raw).events) == [] + + +def test_turn_and_compaction_events_are_not_tool_calls_or_tool_path_steps(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + tool_only = [e for e in events if e["kind"] in TOOL_KINDS] + assert len(tool_only) < len(events) # the run has turn and compaction events + assert process_metrics(events, lambda _: False)["tool_calls"] == len(tool_only) + assert run_from_events(events).kinds() == run_from_events(tool_only).kinds() + + +def test_every_turn_and_compaction_line_validates_against_the_schema(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + for event in events: + if event["kind"] in ("turn", "compaction"): + assert schema_errors("repo_event.v1", {**event, "v": 1}) == [], event + + +def test_a_turn_without_its_counts_does_not_validate(): + turn = {"kind": "turn", "t": None, "turn": 0, "input": 1, "output": 1} # no cache or prompt counts + assert schema_errors("repo_event.v1", {**turn, "v": 1}) != [] From 3b05a4fa22ffeea26ae078b732c477ac31b82a0e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 22:31:53 -0800 Subject: [PATCH 03/20] docs(repo): license header on events-format.md Signed-off-by: Nikolay Petrov (cherry picked from commit f3a7af164a931fc8f8e35945dfc03c600a64ef8d) --- dokimi/docs/events-format.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 007d7bd85..025d934d5 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -1,3 +1,8 @@ + + # `events.jsonl` format `events.jsonl` is the normalized trace of one agent run in a repository batch From 4a53ce9bc04408077e1a405db540ec7e0bc40964 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 11:44:56 +0000 Subject: [PATCH 04/20] Put the turn and compaction rows inside the events table Signed-off-by: Anton Volkov --- dokimi/docs/events-format.md | 1 - 1 file changed, 1 deletion(-) diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 025d934d5..51d9b7828 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -30,7 +30,6 @@ depend on the kind: | `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | - | `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count | | `compaction` | `turn`: the index of the turn it came after, `null` before the first call | From 29725a7577a205e36b570fb5c969ec857009f7ef Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:58:08 -0600 Subject: [PATCH 05/20] fix(repo): null for unreported turn counts, tool-only first_useful steps - Claude turns: keep which usage fields a record carried, so an absent count is null in the turn event, not 0 (context/trace.py, turn_events.py). - Codex turns: decide presence per count; `{"input_tokens": 6}` no longer gives output 0, and the totals add each reported count on its own. - first_useful counts tool events only, so a turn before the first useful command no longer adds a step. - A non-string parent_tool_use_id is written as a string `agent`. - events-format.md: a model listed only in the result has no turn. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- ...61114_120500_napetrov_trace-turn-events.md | 2 +- dokimi/docs/events-format.md | 4 +- dokimi/dokimi/context/trace.py | 4 ++ dokimi/dokimi/repos/cli_traces.py | 11 ++++-- dokimi/dokimi/repos/turn_events.py | 24 +++++++----- dokimi/dokimi/repos/useful.py | 7 +++- dokimi/tests/test_repo_turn_events.py | 39 +++++++++++++++++++ 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md index 419c78586..b48b183ae 100644 --- a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 ### Added - **`events.jsonl` has a `turn` event per model call and a `compaction` event per compaction.** - A Claude run's turns come from the parse its token totals use, so their sums are the run's + A Claude run's turns come from the parse its token totals use, so their sums are, for models with assistant records, the run's `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens`; a subagent's turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 51d9b7828..ede56cf54 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -37,7 +37,9 @@ A `turn` is one model call, so a Claude run's trace has one per call. The sums of a complete Claude trace's `turn` events are the run's `result.json` token totals (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`), over the same scope: with `usage_scope` `tree` the events -of subagents are in the sum (they carry `agent`), with `main` they are not. Codex +of subagents are in the sum (they carry `agent`), with `main` they are not. A +model that the result's `modelUsage` lists but that made no assistant record has +no `turn`; its tokens are in the totals only. Codex reports usage on each `turn.completed`, one `turn` event per turn, with `t` `null`; a count Codex did not report is `null`. Copilot's trace carries no usage, so it writes no `turn` events and its token counts stay `null`. diff --git a/dokimi/dokimi/context/trace.py b/dokimi/dokimi/context/trace.py index 2eee109c7..10b747309 100644 --- a/dokimi/dokimi/context/trace.py +++ b/dokimi/dokimi/context/trace.py @@ -41,6 +41,7 @@ MUTATING = frozenset({"Bash", "Edit", "MultiEdit", "NotebookEdit", "Write"}) TOKEN_FIELDS = ("prompt_tokens", "new_tokens", "output_tokens", "input_tokens", "cache_read_tokens", "cache_write_tokens") # the counts of one call +_USAGE_FIELDS = ("input_tokens", "output_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") class TraceError(ValueError): @@ -217,12 +218,14 @@ def _merge(call: dict[str, Any], later: dict[str, Any]) -> None: """Fold a later block into its call; the call keeps its thread and its largest counts.""" for k in TOKEN_FIELDS: call[k] = max(call[k], later[k]) + call["reported"] = sorted({*call["reported"], *later["reported"]}) def _call(usage: dict[str, Any], agent: Any, at: Any) -> dict[str, Any]: fresh = int(usage.get("input_tokens") or 0) created = int(usage.get("cache_creation_input_tokens") or 0) read = int(usage.get("cache_read_input_tokens") or 0) + reported = [k for k in _USAGE_FIELDS if usage.get(k) is not None] # a zero above may be an absent count return { "agent": agent, "at": at, # the message's timestamp, as the trace has it; the first block's @@ -232,6 +235,7 @@ def _call(usage: dict[str, Any], agent: Any, at: Any) -> dict[str, Any]: "input_tokens": fresh, "cache_read_tokens": read, "cache_write_tokens": created, + "reported": reported, # which usage fields the record carried; the counts above fill absent ones with 0 } diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 74811cc02..ee045e466 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -535,12 +535,15 @@ def _codex_usage(out: Transcript, usage: dict[str, Any]) -> None: out.num_turns = (out.num_turns or 0) + 1 out.result_seen = True # a turn ended: the agent started, whatever else it printed out.events.append(turn) - if turn["input"] is not None: # a missing or all-zero usage is not a measured zero: the counts stay None - # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens + # A missing or all-zero usage is not a measured zero, and a count it leaves out stays None on its own. + # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens + if turn["input"] is not None: out.input_tokens = (out.input_tokens or 0) + turn["input"] + if turn["output"] is not None: out.output_tokens = (out.output_tokens or 0) + turn["output"] - if turn["cache_read"] is not None: # reported, so a zero is measured; absent, it stays None - out.cache_read_tokens = (out.cache_read_tokens or 0) + turn["cache_read"] + if turn["cache_read"] is not None: # reported, so a zero is measured; absent, it stays None + out.cache_read_tokens = (out.cache_read_tokens or 0) + turn["cache_read"] + if turn["input"] is not None or turn["output"] is not None: out.usage_scope = "main" # the turns are the parent's; Codex reports no subagent tokens here diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py index 31f4fa60c..f09326e0a 100644 --- a/dokimi/dokimi/repos/turn_events.py +++ b/dokimi/dokimi/repos/turn_events.py @@ -29,14 +29,18 @@ def claude_events(trace: ContextTrace, start: datetime | None) -> list[dict[str, """The turn events of every model call (a subagent's carry ``agent``), then the compactions.""" events: list[dict[str, Any]] = [] for call in trace.calls: + seen = set(call["reported"]) + prompt = seen & {"input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"} event: dict[str, Any] = { "kind": "turn", "t": _seconds(call["at"], start), "turn": call["i"], - "input": call["input_tokens"], "output": call["output_tokens"], - "cache_read": call["cache_read_tokens"], "cache_write": call["cache_write_tokens"], - "prompt_tokens": call["prompt_tokens"], + "input": call["input_tokens"] if "input_tokens" in seen else None, + "output": call["output_tokens"] if "output_tokens" in seen else None, + "cache_read": call["cache_read_tokens"] if "cache_read_input_tokens" in seen else None, + "cache_write": call["cache_write_tokens"] if "cache_creation_input_tokens" in seen else None, + "prompt_tokens": call["prompt_tokens"] if prompt else None, } if call["agent"]: - event["agent"] = call["agent"] + event["agent"] = str(call["agent"]) events.append(event) for compaction in trace.compactions: after = compaction["after_call"] @@ -53,14 +57,16 @@ def codex_turn(turn: int, usage: Mapping[str, Any]) -> dict[str, Any]: produced = int(usage.get("output_tokens") or 0) cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit measured = bool(fresh or produced) # a missing or all-zero usage is not a measured zero - reported = usage.get("cached_input_tokens") is not None # reported, so a zero is measured + has_input = measured and usage.get("input_tokens") is not None # each count is null on its own when absent + has_output = measured and usage.get("output_tokens") is not None + has_cached = measured and usage.get("cached_input_tokens") is not None # reported, so a zero is measured return { "kind": "turn", "t": None, "turn": turn, - "input": max(fresh - cached, 0) if measured else None, - "output": produced if measured else None, - "cache_read": cached if measured and reported else None, + "input": max(fresh - cached, 0) if has_input else None, + "output": produced if has_output else None, + "cache_read": cached if has_cached else None, "cache_write": None, # Codex reports no cache writes - "prompt_tokens": fresh if measured else None, + "prompt_tokens": fresh if has_input else None, } diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index df390e4f4..42523ac56 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -18,6 +18,7 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Any +from .event_format import TOOL_KINDS from .manifest import match_glob from .process_counts import as_measure from .process_metrics import _workspace_path @@ -67,7 +68,11 @@ def first_useful( return _record(None, "not_declared") if events is None: return _record(useful.kind, "unobserved") - for steps, event in enumerate(events, 1): + steps = 0 + for event in events: + if event.get("kind") not in TOOL_KINDS: # a turn or compaction is accounting, not a step + continue + steps += 1 try: found = hits(useful, event) except TimeoutError: # the pattern ran past its budget on this event: no verdict diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 2550c7f11..2e50ae369 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -146,3 +146,42 @@ def test_every_turn_and_compaction_line_validates_against_the_schema(): def test_a_turn_without_its_counts_does_not_validate(): turn = {"kind": "turn", "t": None, "turn": 0, "input": 1, "output": 1} # no cache or prompt counts assert schema_errors("repo_event.v1", {**turn, "v": 1}) != [] + + +def test_a_claude_usage_missing_counts_are_null_not_zero(): + raw = _lines(_call("a", {"input_tokens": 10, "output_tokens": 2}, "2026-11-14T10:00:00Z"), + _call("b", {"input_tokens": 5}, "2026-11-14T10:00:01Z"), + _call("b", {"output_tokens": 4}, "2026-11-14T10:00:02Z")) + assert _turns(convert("claude-stream-json", raw).events) == [ + {"kind": "turn", "t": 0.0, "turn": 0, "input": 10, "output": 2, "cache_read": None, "cache_write": None, + "prompt_tokens": 10}, + {"kind": "turn", "t": 1.0, "turn": 1, "input": 5, "output": 4, "cache_read": None, "cache_write": None, + "prompt_tokens": 5}, + ] + + +def test_a_codex_usage_with_one_side_reports_the_other_as_null(): + raw = _lines({"type": "turn.completed", "usage": {"input_tokens": 6}}, + {"type": "turn.completed", "usage": {"output_tokens": 2}}) + transcript = convert("codex-json", raw) + assert [[t[k] for k in ("input", "output", "prompt_tokens")] for t in _turns(transcript.events)] == [ + [6, None, 6], [None, 2, None]] + assert transcript.input_tokens == 6 and transcript.output_tokens == 2 + + +def test_a_turn_of_a_non_string_parent_tool_use_id_validates_against_the_schema(): + raw = _lines(_call("a", _usage(3, 1, 0, 0), "2026-11-14T10:00:00Z", parent=7)) + turns = _turns(convert("claude-stream-json", raw).events) + assert len(turns) == 1 and turns[0]["agent"] == "7" + assert schema_errors("repo_event.v1", {**turns[0], "v": 1}) == [] + + +def test_a_turn_before_the_first_useful_command_is_not_a_step(): + from dokimi.repos.useful import first_useful + from dokimi.repos.useful_spec import parse_useful + raw = _lines({"type": "turn.completed", "usage": {"input_tokens": 3, "output_tokens": 1}}, + {"type": "item.completed", "item": {"id": "c", "type": "command_execution", "command": "pytest -q", + "aggregated_output": "", "exit_code": 0, "status": "completed"}}) + events = convert("codex-json", raw).events + assert [e["kind"] for e in events][:2] == ["turn", "command"] + assert first_useful(parse_useful({"command": "pytest"}), events)["steps"] == 1 From f3edd97cd9e864009504f13f1754d9a1c80d43e3 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:39:43 -0600 Subject: [PATCH 06/20] test(repo): pin prompt_tokens of a split Claude message to its parts A message id split over two records, one reporting input_tokens and the other cache_read_input_tokens, must give prompt_tokens equal to input + cache_read + cache_write. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/repos/turn_events.py | 3 ++- dokimi/tests/test_repo_turn_events.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py index f09326e0a..ce668a6bf 100644 --- a/dokimi/dokimi/repos/turn_events.py +++ b/dokimi/dokimi/repos/turn_events.py @@ -37,7 +37,8 @@ def claude_events(trace: ContextTrace, start: datetime | None) -> list[dict[str, "output": call["output_tokens"] if "output_tokens" in seen else None, "cache_read": call["cache_read_tokens"] if "cache_read_input_tokens" in seen else None, "cache_write": call["cache_write_tokens"] if "cache_creation_input_tokens" in seen else None, - "prompt_tokens": call["prompt_tokens"] if prompt else None, + "prompt_tokens": (call["input_tokens"] + call["cache_read_tokens"] + call["cache_write_tokens"] + if prompt else None), } if call["agent"]: event["agent"] = str(call["agent"]) diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 2e50ae369..6abf6c804 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -151,15 +151,27 @@ def test_a_turn_without_its_counts_does_not_validate(): def test_a_claude_usage_missing_counts_are_null_not_zero(): raw = _lines(_call("a", {"input_tokens": 10, "output_tokens": 2}, "2026-11-14T10:00:00Z"), _call("b", {"input_tokens": 5}, "2026-11-14T10:00:01Z"), - _call("b", {"output_tokens": 4}, "2026-11-14T10:00:02Z")) + _call("b", {"output_tokens": 4}, "2026-11-14T10:00:02Z"), + _call("c", {"input_tokens": 3}, "2026-11-14T10:00:03Z")) assert _turns(convert("claude-stream-json", raw).events) == [ {"kind": "turn", "t": 0.0, "turn": 0, "input": 10, "output": 2, "cache_read": None, "cache_write": None, "prompt_tokens": 10}, {"kind": "turn", "t": 1.0, "turn": 1, "input": 5, "output": 4, "cache_read": None, "cache_write": None, "prompt_tokens": 5}, + {"kind": "turn", "t": 3.0, "turn": 2, "input": 3, "output": None, "cache_read": None, "cache_write": None, + "prompt_tokens": 3}, ] +def test_a_split_message_prompt_tokens_is_the_sum_of_its_parts(): + raw = _lines(_call("a", {"input_tokens": 5, "output_tokens": 1, "cache_creation_input_tokens": 0}, + "2026-11-14T10:00:00Z"), + _call("a", {"cache_read_input_tokens": 10}, "2026-11-14T10:00:01Z")) + turn = _turns(convert("claude-stream-json", raw).events)[0] + assert turn["input"] == 5 and turn["cache_read"] == 10 and turn["cache_write"] == 0 + assert turn["prompt_tokens"] == 15 + + def test_a_codex_usage_with_one_side_reports_the_other_as_null(): raw = _lines({"type": "turn.completed", "usage": {"input_tokens": 6}}, {"type": "turn.completed", "usage": {"output_tokens": 2}}) From 3efa955056afa424b1b71d013e28902eef2549d8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:08:33 -0800 Subject: [PATCH 07/20] Address review: record the threads already fixed on this branch Each id below was fixed by an earlier commit of this PR (29725a7, 3b05a4f or 4a53ce9); its pinning test in dokimi/tests/test_repo_turn_events.py passes. Thread: PRRT_kwDOVBg7g863mzX4: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863mzYH: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863mzYR: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863mzkM: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863mzkP: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863mzkU: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863m8B2: already fixed in 3b05a4f Thread: PRRT_kwDOVBg7g863pF-s: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863pF-u: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863pF-y: already fixed in 29725a7 Thread: PRRT_kwDOVBg7g863pF-1: already fixed in 29725a7 Thread: PRR_kwDOVBg7g88AAAABYDoDZg: already fixed in 29725a7 Thread: PRR_kwDOVBg7g88AAAABYEru8Q: already fixed in 29725a7 and 3b05a4f Thread: PRR_kwDOVBg7g88AAAABYE2nRA: already fixed in 4a53ce9 Thread: IC_kwDOVBg7g88AAAABmy8o5A: already fixed in 29725a7 Thread: IC_kwDOVBg7g88AAAABm3tzsQ: already fixed in 29725a7 Signed-off-by: Nikolay Petrov From b6c7505b0dcfe4ee0c6b6fbf311714e3080c94a6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:19:48 -0800 Subject: [PATCH 08/20] docs(events): a call without usage writes no turn event A Claude assistant record with usage null is not a turn: the context budget counts only calls with a counter, so the event stream does the same. A compaction after it has turn null. Pinned by a test. Thread: PRRT_kwDOVBg7g863xYVA Thread: PRR_kwDOVBg7g88AAAABYItxig Signed-off-by: Nikolay Petrov --- dokimi/docs/events-format.md | 4 +++- dokimi/tests/test_repo_turn_events.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index ede56cf54..bf47c8ee9 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -33,7 +33,9 @@ depend on the kind: | `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count | | `compaction` | `turn`: the index of the turn it came after, `null` before the first call | -A `turn` is one model call, so a Claude run's trace has one per call. The sums +A `turn` is one model call, so a Claude run's trace has one per call. A call +whose assistant record has no `usage` object writes no `turn`; a `compaction` +after it has `turn: null` unless an earlier call wrote one. The sums of a complete Claude trace's `turn` events are the run's `result.json` token totals (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`), over the same scope: with `usage_scope` `tree` the events diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 6abf6c804..946868db5 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -98,6 +98,17 @@ def test_a_compaction_follows_the_turn_it_came_after(): assert [e for e in events if e["kind"] == "compaction"] == [{"kind": "compaction", "t": 3.0, "turn": 0}] +def test_a_call_without_usage_writes_no_turn_and_a_compaction_after_it_has_none(): + raw = _lines( + {"type": "assistant", "timestamp": "2026-11-14T10:00:00Z", + "message": {"id": "a1", "model": "m", "content": [], "usage": None}}, + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:05Z"}, + ) + events = convert("claude-stream-json", raw).events + assert _turns(events) == [] + assert [e for e in events if e["kind"] == "compaction"] == [{"kind": "compaction", "t": 5.0, "turn": None}] + + def test_a_compaction_before_any_call_has_no_turn(): raw = _lines({"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:00Z"}) assert [e for e in convert("claude-stream-json", raw).events if e["kind"] == "compaction"] == [ From a652c4020482c2a745a2476af8b8d42011637391 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:20:47 -0800 Subject: [PATCH 09/20] fix(events): a Codex turn with only cached input reports the cached count A usage with cached_input_tokens alone was taken as unmeasured, so the reported cache_read was written as null. The count is measured when any Codex count is reported. Thread: PRRT_kwDOVBg7g8630Bn2 Thread: PRR_kwDOVBg7g88AAAABYKC9zA Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/turn_events.py | 2 +- dokimi/tests/test_repo_turn_events.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py index ce668a6bf..7c2aa0e58 100644 --- a/dokimi/dokimi/repos/turn_events.py +++ b/dokimi/dokimi/repos/turn_events.py @@ -57,7 +57,7 @@ def codex_turn(turn: int, usage: Mapping[str, Any]) -> dict[str, Any]: fresh = int(usage.get("input_tokens") or 0) produced = int(usage.get("output_tokens") or 0) cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit - measured = bool(fresh or produced) # a missing or all-zero usage is not a measured zero + measured = bool(fresh or produced or cached) # a missing or all-zero usage is not a measured zero has_input = measured and usage.get("input_tokens") is not None # each count is null on its own when absent has_output = measured and usage.get("output_tokens") is not None has_cached = measured and usage.get("cached_input_tokens") is not None # reported, so a zero is measured diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 946868db5..8879b1c0a 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -115,6 +115,13 @@ def test_a_compaction_before_any_call_has_no_turn(): {"kind": "compaction", "t": 0.0, "turn": None}] +def test_a_codex_turn_with_only_cached_input_reports_that_count(): + raw = _lines({"type": "turn.completed", "usage": {"cached_input_tokens": 5}}) + assert _turns(convert("codex-json", raw).events) == [ + {"kind": "turn", "t": None, "turn": 0, "input": None, "output": None, "cache_read": 5, "cache_write": None, + "prompt_tokens": None}] + + def test_codex_turns_come_from_turn_completed_and_a_missing_count_is_null(): raw = _lines( {"type": "turn.completed", "usage": {"input_tokens": 10, "cached_input_tokens": 4, "output_tokens": 3}}, From 58331dbe0c284d77492f353d393777fccd519acc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:20:47 -0800 Subject: [PATCH 10/20] Thread: PRRT_kwDOVBg7g863pF-p: already fixed in 3efa955056 Signed-off-by: Nikolay Petrov From 6dc4baba58687d8ba9c4f6c6f5a3566739da5734 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:31:34 -0800 Subject: [PATCH 11/20] test(repo): keep both checks of the incomplete-command test when turn events are filtered out Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_cli_traces.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 911ff624a..03cedf605 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -554,7 +554,9 @@ def test_codex_incomplete_command_is_recorded_with_rc_null(): {"type": "turn.completed", "usage": {"input_tokens": 10, "output_tokens": 1}}, ) t = convert("codex-json", raw, None) - assert [e for e in t.events if e["kind"] != "turn"] == [{"t": None, "kind": "command", "text": "make build", "rc": None}] + steps = [e for e in t.events if e["kind"] != "turn"] # the turn event is accounting, not the command + assert len(steps) == 1 + assert steps[0] == {"t": None, "kind": "command", "text": "make build", "rc": None} def test_codex_failed_command_does_not_count_as_a_read(): From c602d3bb07b940acab95d80ea6dcc5749519d7f8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 16:53:36 +0000 Subject: [PATCH 12/20] Restore the line break before issued_s in the useful.py docstring Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/useful.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index cfb2f75e4..57e086fe3 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -6,8 +6,9 @@ The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never hits, nor does an edit whose tool result is not a success (``rc`` 0 is the only success). ``steps`` counts the tool -events up to and including the hit, as ``process.tool_calls`` does; turn and compaction events are not steps.``issued_s`` is the hit's ``t``: the time the call was -issued, since its result is not timed, so a test run's pass is not timed. +events up to and including the hit, as ``process.tool_calls`` does; turn and compaction events are not steps. +``issued_s`` is the hit's ``t``: the time the call was issued, since its result +is not timed, so a test run's pass is not timed. """ from __future__ import annotations From a37f0f58c1477e7779d56619d80098b8a264ef03 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:47:15 -0800 Subject: [PATCH 13/20] test(repo): pin the null counts of an output-only Claude usage An assistant record with only output_tokens must leave input and prompt_tokens null, not zero. The test is test_an_output_only_claude_usage_leaves_input_and_prompt_tokens_null (added in the merge above, red on the mutant that drops the presence check). Thread: PRRT_kwDOVBg7g864jInm Review: PRR_kwDOVBg7g88AAAABYjf_0Q Review: PRR_kwDOVBg7g88AAAABYjo25A: already fixed in c602d3bb0 (the useful.py docstring line) Signed-off-by: Nikolay Petrov From 4ebb365c1692d9c4e28ac9cc2e4a38cb4923e4f5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:51:03 -0800 Subject: [PATCH 14/20] docs(repo): resolve the events-format.md conflict markers left by the merge Signed-off-by: Nikolay Petrov --- dokimi/docs/events-format.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index d0c086e2c..dde55a35e 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -30,7 +30,6 @@ depend on the kind: | `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | -<<<<<<< HEAD | `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count | | `compaction` | `turn`: the index of the turn it came after, `null` before the first call | @@ -52,8 +51,6 @@ tokens of its own. Turn and compaction events are not tool calls: `result.json` `process.tool_calls` counts the tool kinds only (`command`, `read`, `edit`, `tool`). A reader that reads the tool path of a run keeps to those kinds too. -======= ->>>>>>> b62613a11aaaa310ff39bf7eed57f89d96ccde1f Any line may also carry `agent` (the id of the parent's `Agent`/`Task` call that started it), and `denied: true` when the call was refused and never ran. The @@ -62,12 +59,8 @@ exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. ## Version Every line carries `v`: the major version of the format it was written under. -<<<<<<< HEAD Version 1 is the format this page describes; it has the `turn` and `compaction` kinds, added to it within version 1. -======= -Version 1 is the format this page describes. ->>>>>>> b62613a11aaaa310ff39bf7eed57f89d96ccde1f - A line with no `v` was written before the field existed. It has the version 1 shape and reads as version 1. @@ -96,10 +89,6 @@ file may mix versions, since traces are re-derived and appended to. ## Null values -<<<<<<< HEAD -`t` is `null` when the harness writes no timestamps (Codex, Copilot). `rc` is -======= `t` is `null` when a record has no parseable timestamp (always for Codex). `rc` is ->>>>>>> b62613a11aaaa310ff39bf7eed57f89d96ccde1f `null` when no result came back, for example at the turn or budget limit: a grader must not count `null` as a failure. From cd23951d8e868326f4e53415b26249079c30e1b5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:15:17 -0800 Subject: [PATCH 15/20] fix(repo): a Codex turn with only cached input sets the usage scope Its cache_read total was set while usage_scope stayed None, so the totals had no scope. Found by the self-review of this round. Thread: PRRT_kwDOVBg7g864jInm Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 2 +- dokimi/tests/test_repo_turn_events.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index fab54e8d6..b820561b5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -552,7 +552,7 @@ def _codex_usage(out: Transcript, usage: dict[str, Any]) -> None: out.output_tokens = (out.output_tokens or 0) + turn["output"] if turn["cache_read"] is not None: # reported, so a zero is measured; absent, it stays None out.cache_read_tokens = (out.cache_read_tokens or 0) + turn["cache_read"] - if turn["input"] is not None or turn["output"] is not None: + if any(turn[k] is not None for k in ("input", "output", "cache_read")): out.usage_scope = "main" # the turns are the parent's; Codex reports no subagent tokens here diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 77b5665c1..3fc6dec8e 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -221,3 +221,8 @@ def test_a_turn_before_the_first_useful_command_is_not_a_step(): events = convert("codex-json", raw).events assert [e["kind"] for e in events][:2] == ["turn", "command"] assert first_useful(parse_useful({"command": "pytest"}), events)["steps"] == 1 + + +def test_a_codex_turn_with_only_cached_input_sets_the_scope_of_its_totals(): + transcript = convert("codex-json", _lines({"type": "turn.completed", "usage": {"cached_input_tokens": 5}})) + assert (transcript.cache_read_tokens, transcript.usage_scope) == (5, "main") From 9da247c6abcd86663b55c474e0165a5c8a193632 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:15:17 -0800 Subject: [PATCH 16/20] docs(repo): say the turn sums match the totals only when the usage does Claude's turns come from its assistant records, the totals from the result record; they agree when the result's usage equals the per-call usage, which no real run has checked yet. Also say event lines are not in time order across kinds, and that a Claude prompt_tokens can be partial. Review: PRR_kwDOVBg7g88AAAABYjf_0Q Signed-off-by: Nikolay Petrov --- .../20261114_120500_napetrov_trace-turn-events.md | 4 ++-- dokimi/docs/events-format.md | 8 ++++---- dokimi/dokimi/repos/turn_events.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md index b48b183ae..31dd20101 100644 --- a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -6,8 +6,8 @@ SPDX-License-Identifier: Apache-2.0 ### Added - **`events.jsonl` has a `turn` event per model call and a `compaction` event per compaction.** - A Claude run's turns come from the parse its token totals use, so their sums are, for models with assistant records, the run's - `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens`; a subagent's + A Claude run's turns come from its assistant records; their sums should match the run's + `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens` when the result's usage equals its per-call usage; a subagent's turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts the tool events only. The format is documented in `docs/events-format.md`. diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index dde55a35e..dc3226a1f 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -30,20 +30,20 @@ depend on the kind: | `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | -| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count | +| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count (a Claude `prompt_tokens` sums the parts it reported, so it can be partial) | | `compaction` | `turn`: the index of the turn it came after, `null` before the first call | A `turn` is one model call, so a Claude run's trace has one per call. A call whose assistant record has no `usage` object writes no `turn`; a `compaction` after it has `turn: null` unless an earlier call wrote one. The sums -of a complete Claude trace's `turn` events are the run's `result.json` token +of a complete Claude trace's `turn` events should equal the run's `result.json` token totals (`input_tokens`, `output_tokens`, `cache_read_tokens`, -`cache_write_tokens`), over the same scope: with `usage_scope` `tree` the events +`cache_write_tokens`) when the result's usage equals its per-call usage (not yet checked on a real run), over the same scope: with `usage_scope` `tree` the events of subagents are in the sum (they carry `agent`), with `main` they are not. A model that the result's `modelUsage` lists but that made no assistant record has no `turn`; its tokens are in the totals only. Codex reports usage on each `turn.completed`, one `turn` event per turn, with `t` -`null`; a count Codex did not report is `null`. Copilot's trace carries no +`null`; a count Codex did not report is `null`. Lines are not in time order across kinds (Claude's `turn` and `compaction` lines follow its tool lines), so sort by `t`. Copilot's trace carries no usage, so it writes no `turn` events and its token counts stay `null`. `compaction` is the point where Claude Code summarised the history; it has no tokens of its own. diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py index 7c2aa0e58..bc4a8b4de 100644 --- a/dokimi/dokimi/repos/turn_events.py +++ b/dokimi/dokimi/repos/turn_events.py @@ -5,8 +5,8 @@ A ``turn`` is one model call and the tokens it was billed: ``input`` (uncached), ``output``, ``cache_read``, ``cache_write`` and ``prompt_tokens`` (the whole prompt, ``input + cache_read + -cache_write``). Claude's come from :mod:`dokimi.context.trace`, the parse the run's own token totals -use, so the turns of a complete trace sum to those totals. Codex's come from its ``turn.completed`` +cache_write``). Claude's come from :mod:`dokimi.context.trace`, the parse the tool events use; their sums +should match the run's totals when the result's usage equals its per-call usage. Codex's come from its ``turn.completed`` usage. Copilot's trace carries no usage, so it has no turn events. A count the harness did not report is ``null``, never ``0``. From 14cf91550fa7ee80b4b4a2849d1841ae5d7bba68 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:17:03 -0800 Subject: [PATCH 17/20] docs(repo): sort a Claude trace's turn lines by t; say when prompt_tokens is low Codex's turn lines have t null and are in trace order, so the sort advice is for Claude only. A Claude prompt_tokens is low when one of its parts is null. The turn_events docstring names the same parse as the PR body. Signed-off-by: Nikolay Petrov --- .../20261114_120500_napetrov_trace-turn-events.md | 4 ++-- dokimi/docs/events-format.md | 4 ++-- dokimi/dokimi/repos/turn_events.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md index 31dd20101..db03a2d0e 100644 --- a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: Apache-2.0 - **`events.jsonl` has a `turn` event per model call and a `compaction` event per compaction.** A Claude run's turns come from its assistant records; their sums should match the run's - `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens` when the result's usage equals its per-call usage; a subagent's - turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report + `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens` when the result's + usage equals its per-call usage; a subagent's turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts the tool events only. The format is documented in `docs/events-format.md`. diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index dc3226a1f..5536222aa 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -30,7 +30,7 @@ depend on the kind: | `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | -| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count (a Claude `prompt_tokens` sums the parts it reported, so it can be partial) | +| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count (a Claude `prompt_tokens` sums the parts it reported, so it is low when a part is `null`) | | `compaction` | `turn`: the index of the turn it came after, `null` before the first call | A `turn` is one model call, so a Claude run's trace has one per call. A call @@ -43,7 +43,7 @@ of subagents are in the sum (they carry `agent`), with `main` they are not. A model that the result's `modelUsage` lists but that made no assistant record has no `turn`; its tokens are in the totals only. Codex reports usage on each `turn.completed`, one `turn` event per turn, with `t` -`null`; a count Codex did not report is `null`. Lines are not in time order across kinds (Claude's `turn` and `compaction` lines follow its tool lines), so sort by `t`. Copilot's trace carries no +`null`; a count Codex did not report is `null`. In a Claude trace the `turn` and `compaction` lines follow its tool lines, so sort those by `t`; Codex's lines are in trace order. Copilot's trace carries no usage, so it writes no `turn` events and its token counts stay `null`. `compaction` is the point where Claude Code summarised the history; it has no tokens of its own. diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py index bc4a8b4de..376bc1588 100644 --- a/dokimi/dokimi/repos/turn_events.py +++ b/dokimi/dokimi/repos/turn_events.py @@ -5,10 +5,10 @@ A ``turn`` is one model call and the tokens it was billed: ``input`` (uncached), ``output``, ``cache_read``, ``cache_write`` and ``prompt_tokens`` (the whole prompt, ``input + cache_read + -cache_write``). Claude's come from :mod:`dokimi.context.trace`, the parse the tool events use; their sums -should match the run's totals when the result's usage equals its per-call usage. Codex's come from its ``turn.completed`` -usage. Copilot's trace carries no usage, so it has no turn events. A count the harness did not report -is ``null``, never ``0``. +cache_write``). Claude's come from :mod:`dokimi.context.trace` (the parse the context budget uses); +their sums should match the run's totals when the result's usage equals its per-call usage. +Codex's come from its ``turn.completed`` usage. Copilot's trace carries no usage, so it has no +turn events. A count the harness did not report is ``null``, never ``0``. A ``compaction`` follows the turn it came after; its ``turn`` is that turn's index, or ``null`` before the first call. From de77efbabc81fbc53edba2fc2310780b2cbe85df Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:08:43 -0800 Subject: [PATCH 18/20] feat(repo): events.jsonl lines are format version 2; repo_event.v1.json is unchanged The turn lines put an integer output on a line whose v1 schema has a string output at the top level, so v1 was edited in place, which schema_compat refuses. v1 is restored to main; version 2 (repo_event.v2.json) carries output on command lines only and the turn and compaction kinds, and the writer stamps v 2. Signed-off-by: Nikolay Petrov --- ...61114_120500_napetrov_trace-turn-events.md | 2 +- dokimi/docs/events-format.md | 18 +++-- dokimi/dokimi/repos/event_format.py | 2 +- dokimi/dokimi/schemas/repo_event.v1.json | 32 ++------- dokimi/dokimi/schemas/repo_event.v2.json | 70 +++++++++++++++++++ .../fixtures/artifacts/repo_event.v2.json | 9 +++ dokimi/tests/test_repo_event_format.py | 22 +++--- dokimi/tests/test_repo_turn_events.py | 4 +- dokimi/tests/test_skill_reach.py | 4 +- 9 files changed, 111 insertions(+), 52 deletions(-) create mode 100644 dokimi/dokimi/schemas/repo_event.v2.json create mode 100644 dokimi/tests/fixtures/artifacts/repo_event.v2.json diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md index db03a2d0e..4654a92d8 100644 --- a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -10,4 +10,4 @@ SPDX-License-Identifier: Apache-2.0 `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens` when the result's usage equals its per-call usage; a subagent's turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts - the tool events only. The format is documented in `docs/events-format.md`. + the tool events only. Lines are format version 2 (`repo_event.v2.json`): `output` and `output_truncated` sit on `command` lines only; `repo_event.v1.json` is unchanged. The format is documented in `docs/events-format.md`. diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 5536222aa..1f61671ff 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -54,13 +54,17 @@ reads the tool path of a run keeps to those kinds too. Any line may also carry `agent` (the id of the parent's `Agent`/`Task` call that started it), and `denied: true` when the call was refused and never ran. The -exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. +exact field list is the JSON Schema `dokimi/schemas/repo_event.v2.json` (lines this code writes); +lines written before version 2 are described by `repo_event.v1.json`, which is unchanged. ## Version Every line carries `v`: the major version of the format it was written under. -Version 1 is the format this page describes; it has the `turn` and `compaction` -kinds, added to it within version 1. +This page describes version 2, which lines written by this code carry as `v: 2`. +Version 2 differs from version 1 in two ways: `output` and `output_truncated` are +on `command` lines only (version 1 had them on every line, which an integer +`output` on a `turn` line contradicts), and the `turn` and `compaction` kinds +are in it. Version 1 lines stay valid under `repo_event.v1.json`. - A line with no `v` was written before the field existed. It has the version 1 shape and reads as version 1. @@ -72,10 +76,10 @@ kinds, added to it within version 1. Within a major version, kinds and fields are only added. A reader written for version 1 keeps working on every later version 1 file, and a field that existed -keeps its name, type and meaning. A change that would break a version 1 reader -(a renamed or removed field, a changed meaning, a changed `null`) is version 2: -a new schema `repo_event.v2.json`, with `v` set to 2 on the lines it writes. A -file may mix versions, since traces are re-derived and appended to. +keeps its name, type and meaning. A change that would break a reader of the +current version is a new version: a new schema file `repo_event.vN.json`, with +`v` set to N on the lines it writes. Released schema files are not edited in +place. A file may mix versions, since traces are re-derived and appended to. ## Reading diff --git a/dokimi/dokimi/repos/event_format.py b/dokimi/dokimi/repos/event_format.py index 80c76a92d..b7703f07a 100644 --- a/dokimi/dokimi/repos/event_format.py +++ b/dokimi/dokimi/repos/event_format.py @@ -13,7 +13,7 @@ from typing import Any, Mapping -EVENT_VERSION = 1 # the major version this code writes; a version change adds repo_event.vN.json +EVENT_VERSION = 2 # the major version this code writes; a version change adds repo_event.vN.json #: The kinds that are tool activity: what ``tool_calls`` counts. TOOL_KINDS = frozenset({"command", "read", "edit", "tool"}) diff --git a/dokimi/dokimi/schemas/repo_event.v1.json b/dokimi/dokimi/schemas/repo_event.v1.json index 4c715c8f2..fd6d569e7 100644 --- a/dokimi/dokimi/schemas/repo_event.v1.json +++ b/dokimi/dokimi/schemas/repo_event.v1.json @@ -9,7 +9,9 @@ "kind": {"type": "string", "description": "What the event is. Readers ignore a kind they do not know."}, "t": {"type": ["number", "null"], "description": "Seconds since the first timestamped trace line; null when the harness has no timestamps."}, "agent": {"type": "string", "description": "The id of the parent's Agent/Task call that started this event; absent on the parent's events."}, - "denied": {"type": "boolean", "description": "The call was refused and never ran."} + "denied": {"type": "boolean", "description": "The call was refused and never ran."}, + "output": {"type": "string", "description": "A command's output as the harness returned it, clipped to 8 KB."}, + "output_truncated": {"type": "boolean"} }, "required": ["kind", "t"], "allOf": [ @@ -17,12 +19,7 @@ "if": {"properties": {"kind": {"const": "command"}}, "required": ["kind"]}, "then": { "required": ["text", "rc"], - "properties": { - "text": {"type": "string"}, - "rc": {"type": ["integer", "null"]}, - "output": {"type": "string", "description": "The command's output as the harness returned it, clipped to 8 KB."}, - "output_truncated": {"type": "boolean"} - } + "properties": {"text": {"type": "string"}, "rc": {"type": ["integer", "null"]}} } }, { @@ -44,27 +41,6 @@ "required": ["name"], "properties": {"name": {"type": "string"}, "skill": {"type": "string"}} } - }, - { - "if": {"properties": {"kind": {"const": "turn"}}, "required": ["kind"]}, - "then": { - "required": ["turn", "input", "output", "cache_read", "cache_write", "prompt_tokens"], - "properties": { - "turn": {"type": "integer", "minimum": 0}, - "input": {"type": ["integer", "null"]}, - "output": {"type": ["integer", "null"]}, - "cache_read": {"type": ["integer", "null"]}, - "cache_write": {"type": ["integer", "null"]}, - "prompt_tokens": {"type": ["integer", "null"]} - } - } - }, - { - "if": {"properties": {"kind": {"const": "compaction"}}, "required": ["kind"]}, - "then": { - "required": ["turn"], - "properties": {"turn": {"type": ["integer", "null"], "minimum": 0}} - } } ] } diff --git a/dokimi/dokimi/schemas/repo_event.v2.json b/dokimi/dokimi/schemas/repo_event.v2.json new file mode 100644 index 000000000..ee39adcac --- /dev/null +++ b/dokimi/dokimi/schemas/repo_event.v2.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/repo_event.v2.json", + "title": "One line of events.jsonl, format version 2", + "description": "A line of a repository run's normalized trace (docs/events-format.md). One JSON object per line; this schema validates one line. Within version 2 a kind and a field are only ever added, so a reader must ignore a kind it does not know (this schema allows any kind) and a field it does not know. A line with no `v` is a version 1 line, written before the version field existed; version 1 lines validate against repo_event.v1.json.", + "type": "object", + "properties": { + "v": {"const": 2, "description": "The format version of this line. Absent on a line written before the field existed."}, + "kind": {"type": "string", "description": "What the event is. Readers ignore a kind they do not know."}, + "t": {"type": ["number", "null"], "description": "Seconds since the first timestamped trace line; null when the harness has no timestamps."}, + "agent": {"type": "string", "description": "The id of the parent's Agent/Task call that started this event; absent on the parent's events."}, + "denied": {"type": "boolean", "description": "The call was refused and never ran."} + }, + "required": ["kind", "t"], + "allOf": [ + { + "if": {"properties": {"kind": {"const": "command"}}, "required": ["kind"]}, + "then": { + "required": ["text", "rc"], + "properties": { + "text": {"type": "string"}, + "rc": {"type": ["integer", "null"]}, + "output": {"type": "string", "description": "The command's output as the harness returned it, clipped to 8 KB."}, + "output_truncated": {"type": "boolean"} + } + } + }, + { + "if": {"properties": {"kind": {"const": "read"}}, "required": ["kind"]}, + "then": { + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}} + } + }, + { + "if": {"properties": {"kind": {"const": "edit"}}, "required": ["kind"]}, + "then": { + "required": ["path"], + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}, "deleted": {"type": "boolean"}} + } + }, + { + "if": {"properties": {"kind": {"const": "tool"}}, "required": ["kind"]}, + "then": { + "required": ["name"], + "properties": {"name": {"type": "string"}, "skill": {"type": "string"}} + } + }, + { + "if": {"properties": {"kind": {"const": "turn"}}, "required": ["kind"]}, + "then": { + "required": ["turn", "input", "output", "cache_read", "cache_write", "prompt_tokens"], + "properties": { + "turn": {"type": "integer", "minimum": 0}, + "input": {"type": ["integer", "null"]}, + "output": {"type": ["integer", "null"]}, + "cache_read": {"type": ["integer", "null"]}, + "cache_write": {"type": ["integer", "null"]}, + "prompt_tokens": {"type": ["integer", "null"]} + } + } + }, + { + "if": {"properties": {"kind": {"const": "compaction"}}, "required": ["kind"]}, + "then": { + "required": ["turn"], + "properties": {"turn": {"type": ["integer", "null"], "minimum": 0}} + } + } + ] +} diff --git a/dokimi/tests/fixtures/artifacts/repo_event.v2.json b/dokimi/tests/fixtures/artifacts/repo_event.v2.json new file mode 100644 index 000000000..0b1bad3a3 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/repo_event.v2.json @@ -0,0 +1,9 @@ +{ + "kind": "command", + "output": "ERROR: icpx is not found", + "output_truncated": false, + "rc": 1, + "t": 31.0, + "text": "bazel test :x", + "v": 2 +} diff --git a/dokimi/tests/test_repo_event_format.py b/dokimi/tests/test_repo_event_format.py index fd7713536..864de2294 100644 --- a/dokimi/tests/test_repo_event_format.py +++ b/dokimi/tests/test_repo_event_format.py @@ -79,22 +79,22 @@ def test_every_converted_line_is_versioned_and_validates(): assert {e["kind"] for e in lines} == KNOWN_KINDS # the fixtures cover every kind the schema names for event in lines: assert event["v"] == EVENT_VERSION - assert schema_errors("repo_event.v1", event) == [], event + assert schema_errors("repo_event.v2", event) == [], event @pytest.mark.parametrize("line", [ - {"kind": "command", "text": "ls", "t": 1.0, "v": 1}, - {"kind": "command", "rc": 0, "t": 1.0, "v": 1}, - {"kind": "edit", "t": 1.0, "v": 1}, - {"kind": "tool", "t": 1.0, "v": 1}, - {"kind": "read", "path": 3, "t": 1.0, "v": 1}, - {"kind": "command", "text": "ls", "rc": 0, "t": "1.0", "v": 1}, - {"kind": "read", "path": "a.md", "v": 1}, - {"t": 1.0, "v": 1}, + {"kind": "command", "text": "ls", "t": 1.0, "v": 2}, + {"kind": "command", "rc": 0, "t": 1.0, "v": 2}, + {"kind": "edit", "t": 1.0, "v": 2}, + {"kind": "tool", "t": 1.0, "v": 2}, + {"kind": "read", "path": 3, "t": 1.0, "v": 2}, + {"kind": "command", "text": "ls", "rc": 0, "t": "1.0", "v": 2}, + {"kind": "read", "path": "a.md", "v": 2}, + {"t": 1.0, "v": 2}, ], ids=["command-no-rc", "command-no-text", "edit-no-path", "tool-no-name", "read-path-int", "string-t", "no-t", "no-kind"]) def test_a_line_that_breaks_the_version_1_shape_is_refused(line): - assert schema_errors("repo_event.v1", line) != [] + assert schema_errors("repo_event.v2", line) != [] def test_the_writer_only_adds_the_version_to_each_event(): @@ -132,7 +132,7 @@ def test_unversioned_and_converted_give_the_same_process_metrics(): def test_the_schema_names_exactly_the_known_kinds(): - schema = json.loads((SCHEMA_DIR / "repo_event.v1.json").read_text(encoding="utf-8")) + schema = json.loads((SCHEMA_DIR / "repo_event.v2.json").read_text(encoding="utf-8")) named = {clause["if"]["properties"]["kind"]["const"] for clause in schema["allOf"]} assert named == KNOWN_KINDS diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py index 3fc6dec8e..ad56b04b7 100644 --- a/dokimi/tests/test_repo_turn_events.py +++ b/dokimi/tests/test_repo_turn_events.py @@ -158,7 +158,7 @@ def test_every_turn_and_compaction_line_validates_against_the_schema(): events = convert("claude-stream-json", _claude_run(tree=True)).events for event in events: if event["kind"] in ("turn", "compaction"): - assert schema_errors("repo_event.v1", {**event, "v": 1}) == [], event + assert schema_errors("repo_event.v2", {**event, "v": 2}) == [], event def test_a_turn_without_its_counts_does_not_validate(): @@ -209,7 +209,7 @@ def test_a_turn_of_a_non_string_parent_tool_use_id_validates_against_the_schema( raw = _lines(_call("a", _usage(3, 1, 0, 0), "2026-11-14T10:00:00Z", parent=7)) turns = _turns(convert("claude-stream-json", raw).events) assert len(turns) == 1 and turns[0]["agent"] == "7" - assert schema_errors("repo_event.v1", {**turns[0], "v": 1}) == [] + assert schema_errors("repo_event.v2", {**turns[0], "v": 2}) == [] def test_a_turn_before_the_first_useful_command_is_not_a_step(): diff --git a/dokimi/tests/test_skill_reach.py b/dokimi/tests/test_skill_reach.py index dee2c1df8..c5f355a49 100644 --- a/dokimi/tests/test_skill_reach.py +++ b/dokimi/tests/test_skill_reach.py @@ -871,8 +871,8 @@ def test_a_skill_reach_trace_is_versioned_and_validates(tmp_path, monkeypatch): for line in p.read_text(encoding="utf-8").splitlines()] assert lines for event in lines: - assert event["v"] == 1 - assert schema_errors("repo_event.v1", event) == [], event + assert event["v"] == 2 + assert schema_errors("repo_event.v2", event) == [], event def test_a_new_run_clears_the_traces_of_an_earlier_one(tmp_path): From fa5c31acf4e43157d6a41112a754d8ceea667d88 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:10:52 -0800 Subject: [PATCH 19/20] docs(repo): the events format examples are version 2 and validate against its schema Test-narrowed: test_a_line_that_breaks_the_version_1_shape_is_refused is renamed to the version 2 shape test; version 1 refusal of turn lines is checked in test_repo_turn_events.py Signed-off-by: Nikolay Petrov --- dokimi/docs/events-format.md | 14 ++++++++------ dokimi/dokimi/repos/event_format.py | 2 +- dokimi/tests/test_repo_event_format.py | 12 ++++++++++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index 1f61671ff..f4efa4245 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -17,11 +17,13 @@ One JSON object per line. Every line has a `kind` and a `t`; the other fields depend on the kind: ```json -{"kind": "command", "rc": 0, "t": 12.4, "text": "bazel test //x", "v": 1} -{"kind": "read", "t": 3.1, "path": "AGENTS.md", "v": 1} -{"kind": "edit", "t": 5.0, "path": "src/foo.cpp", "v": 1} -{"kind": "tool", "denied": true, "name": "WebFetch", "t": 20.2, "v": 1} -{"kind": "tool", "name": "Skill", "skill": "lint-fix", "t": 2.0, "v": 1} +{"kind": "command", "rc": 0, "t": 12.4, "text": "bazel test //x", "v": 2} +{"kind": "read", "t": 3.1, "path": "AGENTS.md", "v": 2} +{"kind": "edit", "t": 5.0, "path": "src/foo.cpp", "v": 2} +{"kind": "tool", "denied": true, "name": "WebFetch", "t": 20.2, "v": 2} +{"kind": "tool", "name": "Skill", "skill": "lint-fix", "t": 2.0, "v": 2} +{"kind": "turn", "turn": 3, "input": 1200, "output": 85, "cache_read": 9000, "cache_write": 0, "prompt_tokens": 10200, "t": 14.0, "v": 2} +{"kind": "compaction", "turn": 7, "t": 40.5, "v": 2} ``` | Kind | Fields | @@ -84,7 +86,7 @@ place. A file may mix versions, since traces are re-derived and appended to. ## Reading - Ignore a kind you do not know. Version 1 readers do so already: a new kind - (say a `turn` event) must not fail an old reader. + (say a `probe` event) must not fail an old reader. - Ignore a field you do not know. - Accept a line with no `v`. `dokimi.repos.event_format.event_version` returns 1 for it. diff --git a/dokimi/dokimi/repos/event_format.py b/dokimi/dokimi/repos/event_format.py index b7703f07a..59142bf96 100644 --- a/dokimi/dokimi/repos/event_format.py +++ b/dokimi/dokimi/repos/event_format.py @@ -19,7 +19,7 @@ TOOL_KINDS = frozenset({"command", "read", "edit", "tool"}) #: The per-model-call kinds (:mod:`dokimi.repos.turn_events`): not tool calls. TURN_KINDS = frozenset({"turn", "compaction"}) -#: The kinds the converters write in version 1. Readers must accept any other kind (a later minor addition). +#: The kinds the converters write in version 2 (version 1 wrote the tool kinds only). Readers accept any other kind. KNOWN_KINDS = TOOL_KINDS | TURN_KINDS diff --git a/dokimi/tests/test_repo_event_format.py b/dokimi/tests/test_repo_event_format.py index 864de2294..150bc44b0 100644 --- a/dokimi/tests/test_repo_event_format.py +++ b/dokimi/tests/test_repo_event_format.py @@ -3,7 +3,7 @@ """The version field and the schema of ``events.jsonl`` (docs/events-format.md). -The schema (``repo_event.v1.json``) pins what version 1 lines look like; the converters' output +The schema (``repo_event.v2.json``) pins what version 2 lines look like; the converters' output must validate against it, an unversioned line must still read, and a reader must skip a kind it does not know. """ @@ -93,7 +93,7 @@ def test_every_converted_line_is_versioned_and_validates(): {"t": 1.0, "v": 2}, ], ids=["command-no-rc", "command-no-text", "edit-no-path", "tool-no-name", "read-path-int", "string-t", "no-t", "no-kind"]) -def test_a_line_that_breaks_the_version_1_shape_is_refused(line): +def test_a_line_that_breaks_the_version_2_shape_is_refused(line): assert schema_errors("repo_event.v2", line) != [] @@ -142,3 +142,11 @@ def test_the_doc_names_every_known_kind_and_the_version_field(): for kind in sorted(KNOWN_KINDS): assert f"| `{kind}` |" in text, kind assert "`v`" in text and "repo_event.v2.json" in text + + +def test_the_doc_examples_validate_against_the_version_2_schema(): + block = DOCS.read_text(encoding="utf-8").split("```json\n", 1)[1].split("```", 1)[0] + lines = [json.loads(line) for line in block.splitlines() if line.strip()] + assert len(lines) >= 7 + for line in lines: + assert schema_errors("repo_event.v2", line) == [], line From 0d4a1876453fa16ae1b26fa9a99130c6980cea5e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 16:18:38 -0800 Subject: [PATCH 20/20] test(repo): the two event expectations are version 2 lines The Tests job failed on test_repo_run and test_repo_reach_bash, which compare an events.jsonl line with v 1; the lines are version 2 now. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_reach_bash.py | 2 +- dokimi/tests/test_repo_run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 9a95700eb..3320fb940 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -216,7 +216,7 @@ def test_a_bash_read_of_the_probe_shows_whether_the_directory_guidance_loads(tmp assert [s["id"] for s in payload["scenarios"]] == ["bash=sub"] and not payload["scenarios"][0]["error"] assert payload["summary"]["matrix"] == {"AGENTS.md": {"bash=sub": "loaded"}, "sub/AGENTS.md": {"bash=sub": cls}} events = [json.loads(x) for x in (tmp_path / "out/traces/00-bash_sub.events.jsonl").read_text().splitlines()] - assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0, "v": 1} + assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0, "v": 2} @pytest.mark.parametrize("mode", ["hook", "permissions"]) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index f816d6b01..20c42c05b 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -57,7 +57,7 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert meta["agent_network"] == "host" assert meta["egress_proxy"] is False and len(meta["start_sha"]) == 40 events = [json.loads(line) for line in (rd / "events.jsonl").read_text().splitlines()] - assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None, "v": 1}] # relative to /eval/run/workspace + assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None, "v": 2}] # relative to /eval/run/workspace assert (rd / "answer.txt").read_text() == "done: src/calc.py" assert (rd / "config/settings.json").read_text() == '{"hook": "python3 /eval/support/no_background.py"}' result = json.loads((rd / "result.json").read_text())