diff --git a/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md new file mode 100644 index 000000000..4654a92d8 --- /dev/null +++ b/dokimi/changelog.d/20261114_120500_napetrov_trace-turn-events.md @@ -0,0 +1,13 @@ + + +### Added + +- **`events.jsonl` has a `turn` event per model call and a `compaction` event per compaction.** + A Claude run's turns come from its assistant records; their sums should match the run's + `input_tokens`, `output_tokens`, `cache_read_tokens` and `cache_write_tokens` when the result's + usage equals its per-call usage; a subagent's turns carry `agent`. Codex reports a turn per `turn.completed`, with a count it did not report + as `null`; Copilot's trace has no usage, so it writes no turn events. `process.tool_calls` counts + the tool events only. Lines are format version 2 (`repo_event.v2.json`): `output` and `output_truncated` sit on `command` lines only; `repo_event.v1.json` is unchanged. The format is documented in `docs/events-format.md`. diff --git a/dokimi/docs/events-format.md b/dokimi/docs/events-format.md index adf60899b..f4efa4245 100644 --- a/dokimi/docs/events-format.md +++ b/dokimi/docs/events-format.md @@ -17,11 +17,13 @@ One JSON object per line. Every line has a `kind` and a `t`; the other fields depend on the kind: ```json -{"kind": "command", "rc": 0, "t": 12.4, "text": "bazel test //x", "v": 1} -{"kind": "read", "t": 3.1, "path": "AGENTS.md", "v": 1} -{"kind": "edit", "t": 5.0, "path": "src/foo.cpp", "v": 1} -{"kind": "tool", "denied": true, "name": "WebFetch", "t": 20.2, "v": 1} -{"kind": "tool", "name": "Skill", "skill": "lint-fix", "t": 2.0, "v": 1} +{"kind": "command", "rc": 0, "t": 12.4, "text": "bazel test //x", "v": 2} +{"kind": "read", "t": 3.1, "path": "AGENTS.md", "v": 2} +{"kind": "edit", "t": 5.0, "path": "src/foo.cpp", "v": 2} +{"kind": "tool", "denied": true, "name": "WebFetch", "t": 20.2, "v": 2} +{"kind": "tool", "name": "Skill", "skill": "lint-fix", "t": 2.0, "v": 2} +{"kind": "turn", "turn": 3, "input": 1200, "output": 85, "cache_read": 9000, "cache_write": 0, "prompt_tokens": 10200, "t": 14.0, "v": 2} +{"kind": "compaction", "turn": 7, "t": 40.5, "v": 2} ``` | Kind | Fields | @@ -30,15 +32,41 @@ depend on the kind: | `read` | `path` (absent on a refused relative read whose file is not known); `rc` 0 or 1 when the harness reported one | | `edit` | `path`; `rc`; `deleted` when the file was deleted | | `tool` | `name`; `skill` on a `Skill` call | +| `turn` | `turn` (the model call's index in the trace, from 0; a subagent's calls count too); `input` (uncached input), `output`, `cache_read`, `cache_write`, `prompt_tokens` (the whole prompt: `input + cache_read + cache_write`); each `null` when the harness reported no such count (a Claude `prompt_tokens` sums the parts it reported, so it is low when a part is `null`) | +| `compaction` | `turn`: the index of the turn it came after, `null` before the first call | + +A `turn` is one model call, so a Claude run's trace has one per call. A call +whose assistant record has no `usage` object writes no `turn`; a `compaction` +after it has `turn: null` unless an earlier call wrote one. The sums +of a complete Claude trace's `turn` events should equal the run's `result.json` token +totals (`input_tokens`, `output_tokens`, `cache_read_tokens`, +`cache_write_tokens`) when the result's usage equals its per-call usage (not yet checked on a real run), over the same scope: with `usage_scope` `tree` the events +of subagents are in the sum (they carry `agent`), with `main` they are not. A +model that the result's `modelUsage` lists but that made no assistant record has +no `turn`; its tokens are in the totals only. Codex +reports usage on each `turn.completed`, one `turn` event per turn, with `t` +`null`; a count Codex did not report is `null`. In a Claude trace the `turn` and `compaction` lines follow its tool lines, so sort those by `t`; Codex's lines are in trace order. Copilot's trace carries no +usage, so it writes no `turn` events and its token counts stay `null`. +`compaction` is the point where Claude Code summarised the history; it has no +tokens of its own. + +Turn and compaction events are not tool calls: `result.json` `process.tool_calls` +counts the tool kinds only (`command`, `read`, `edit`, `tool`). A reader that +reads the tool path of a run keeps to those kinds too. Any line may also carry `agent` (the id of the parent's `Agent`/`Task` call that started it), and `denied: true` when the call was refused and never ran. The -exact field list is the JSON Schema `dokimi/schemas/repo_event.v1.json`. +exact field list is the JSON Schema `dokimi/schemas/repo_event.v2.json` (lines this code writes); +lines written before version 2 are described by `repo_event.v1.json`, which is unchanged. ## Version Every line carries `v`: the major version of the format it was written under. -Version 1 is the format this page describes. +This page describes version 2, which lines written by this code carry as `v: 2`. +Version 2 differs from version 1 in two ways: `output` and `output_truncated` are +on `command` lines only (version 1 had them on every line, which an integer +`output` on a `turn` line contradicts), and the `turn` and `compaction` kinds +are in it. Version 1 lines stay valid under `repo_event.v1.json`. - A line with no `v` was written before the field existed. It has the version 1 shape and reads as version 1. @@ -50,15 +78,15 @@ Version 1 is the format this page describes. Within a major version, kinds and fields are only added. A reader written for version 1 keeps working on every later version 1 file, and a field that existed -keeps its name, type and meaning. A change that would break a version 1 reader -(a renamed or removed field, a changed meaning, a changed `null`) is version 2: -a new schema `repo_event.v2.json`, with `v` set to 2 on the lines it writes. A -file may mix versions, since traces are re-derived and appended to. +keeps its name, type and meaning. A change that would break a reader of the +current version is a new version: a new schema file `repo_event.vN.json`, with +`v` set to N on the lines it writes. Released schema files are not edited in +place. A file may mix versions, since traces are re-derived and appended to. ## Reading - Ignore a kind you do not know. Version 1 readers do so already: a new kind - (say a `turn` event) must not fail an old reader. + (say a `probe` event) must not fail an old reader. - Ignore a field you do not know. - Accept a line with no `v`. `dokimi.repos.event_format.event_version` returns 1 for it. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e0b1be55b..d76704efe 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -552,7 +552,7 @@ parent's and subagents' together, so a grader need not: | Key | Meaning | |---|---| -| `tool_calls` | every event of a known kind: each tool call the agent made, and each shell-derived `read` event kept, a denied one or one of a failed multi-file read too (see `guidance_read` and §7) | +| `tool_calls` | every tool event (`command`, `read`, `edit`, `tool`, not the `turn` and `compaction` events of [events-format.md](events-format.md)): each tool call the agent made, and each shell-derived `read` event kept, a denied one or one of a failed multi-file read too (see `guidance_read` and §7) | | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | @@ -572,8 +572,8 @@ hits the task's `useful` key (§3). The definition depends on the grader: | review (`dokimi:review_*`) | none | none, so `not_declared`: the answer is the product, so no event is the useful result. A declared key is measured all the same | | any other repository grader | either form, or none | none declared, `not_declared` | -A refused event (`denied`) never hits, nor does an `edit` without a success result (`rc` 0): an error result (`rc` 1) means the file was not changed, and no result at all (a killed run) proves nothing. An edit that deletes its file (a Codex `file_change` of kind `delete`, a Copilot `*** Delete File:` patch) is no hit either: the target is gone, not the useful change. `steps` counts the events of a known kind up to and -including the hit, the parent's and subagents' (as `process.tool_calls` does). +A refused event (`denied`) never hits, nor does an `edit` without a success result (`rc` 0): an error result (`rc` 1) means the file was not changed, and no result at all (a killed run) proves nothing. An edit that deletes its file (a Codex `file_change` of kind `delete`, a Copilot `*** Delete File:` patch) is no hit either: the target is gone, not the useful change. `steps` counts the tool events up to and +including the hit, the parent's and subagents' (as `process.tool_calls` does); turn and compaction events are not steps. `issued_s` is the hit's `t`: the time the call was issued, since a call's result is not timed. `first_useful` is `{"kind": "command"|"edit"|null, "reason": ..., "issued_s": number|null, "steps": int|null}`. `reason` is `null` when the hit was diff --git a/dokimi/dokimi/context/trace.py b/dokimi/dokimi/context/trace.py index a5c1f81d6..10b747309 100644 --- a/dokimi/dokimi/context/trace.py +++ b/dokimi/dokimi/context/trace.py @@ -13,7 +13,8 @@ so the first call's prompt is everything loaded before the agent did anything (system prompt, tool schemas, memory files, skill listing, the task), and the series after it is how the history grew. ``new_tokens`` (input + cache creation) is what the call added beyond the cached -prefix. +prefix. Each call also keeps its parts (``input_tokens``, ``cache_read_tokens``, ``cache_write_tokens``) and +its message timestamp (``at``): the per-turn events of ``events.jsonl`` read them (:mod:`dokimi.repos.turn_events`). Calls a subagent made carry ``parent_tool_use_id``; they run in their own window, so the summary reads the main thread and counts subagent calls apart. A ``compact_boundary`` system @@ -38,7 +39,9 @@ # Tools whose call can change what a later identical call returns; one ends every earlier repeat. MUTATING = frozenset({"Bash", "Edit", "MultiEdit", "NotebookEdit", "Write"}) -TOKEN_FIELDS = ("prompt_tokens", "new_tokens", "output_tokens") # the counts of one call +TOKEN_FIELDS = ("prompt_tokens", "new_tokens", "output_tokens", "input_tokens", "cache_read_tokens", + "cache_write_tokens") # the counts of one call +_USAGE_FIELDS = ("input_tokens", "output_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") class TraceError(ValueError): @@ -165,12 +168,13 @@ def _apply(out: ContextTrace, rec: dict[str, Any], by_message: dict[str, dict[st "pre_tokens": meta.get("pre_tokens", meta.get("preTokens")), "post_tokens": meta.get("post_tokens"), # newer CLIs only "after_call": len(out.calls) - 1, + "at": rec.get("timestamp"), }) elif kind == "assistant": message = rec.get("message") or {} usage = message.get("usage") if isinstance(usage, dict): - call = _call(usage, agent) + call = _call(usage, agent, rec.get("timestamp")) key = message.get("id") if key and key in by_message: _merge(by_message[key], call) # a later block of the same call: same prompt, fuller output @@ -214,17 +218,24 @@ def _merge(call: dict[str, Any], later: dict[str, Any]) -> None: """Fold a later block into its call; the call keeps its thread and its largest counts.""" for k in TOKEN_FIELDS: call[k] = max(call[k], later[k]) + call["reported"] = sorted({*call["reported"], *later["reported"]}) -def _call(usage: dict[str, Any], agent: Any) -> dict[str, Any]: +def _call(usage: dict[str, Any], agent: Any, at: Any) -> dict[str, Any]: fresh = int(usage.get("input_tokens") or 0) created = int(usage.get("cache_creation_input_tokens") or 0) read = int(usage.get("cache_read_input_tokens") or 0) + reported = [k for k in _USAGE_FIELDS if usage.get(k) is not None] # a zero above may be an absent count return { "agent": agent, + "at": at, # the message's timestamp, as the trace has it; the first block's "prompt_tokens": fresh + created + read, "new_tokens": fresh + created, "output_tokens": int(usage.get("output_tokens") or 0), + "input_tokens": fresh, + "cache_read_tokens": read, + "cache_write_tokens": created, + "reported": reported, # which usage fields the record carried; the counts above fill absent ones with 0 } diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 7e58f701c..b820561b5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -31,6 +31,7 @@ from .shell_words import (_DIRS, _KEYWORDS, _case_closes, _case_opens, _cd, _command_words, _is_activate, _is_cd, _loses, _run_name, _runs_dir, _time_keyword, _unquoted) from .transcript import Transcript, _from_cwd, _parse_time, clip_output, relative_path +from .turn_events import codex_turn _BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) _PRINTERS = {"cat", "head", "tail", "nl", "sed"} @@ -539,17 +540,19 @@ def _codex_keep_record(out: Transcript, kind: Any, rec: dict[str, Any]) -> None: def _codex_usage(out: Transcript, usage: dict[str, Any]) -> None: + turn = codex_turn(out.num_turns or 0, usage) # its turn event, and the totals: both from one reading out.num_turns = (out.num_turns or 0) + 1 out.result_seen = True # a turn ended: the agent started, whatever else it printed - fresh, produced = int(usage.get("input_tokens") or 0), int(usage.get("output_tokens") or 0) - cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit - if fresh or produced: # a missing or all-zero usage is not a measured zero: the counts stay None - # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens. - # A malformed record with more cached than input never goes negative - out.input_tokens = (out.input_tokens or 0) + max(fresh - cached, 0) - out.output_tokens = (out.output_tokens or 0) + produced - if usage.get("cached_input_tokens") is not None: # reported, so a zero is measured; absent, it stays None - out.cache_read_tokens = (out.cache_read_tokens or 0) + cached + out.events.append(turn) + # A missing or all-zero usage is not a measured zero, and a count it leaves out stays None on its own. + # OpenAI counts the cached part inside input_tokens; Claude cells keep cache reads out of input_tokens + if turn["input"] is not None: + out.input_tokens = (out.input_tokens or 0) + turn["input"] + if turn["output"] is not None: + out.output_tokens = (out.output_tokens or 0) + turn["output"] + if turn["cache_read"] is not None: # reported, so a zero is measured; absent, it stays None + out.cache_read_tokens = (out.cache_read_tokens or 0) + turn["cache_read"] + if any(turn[k] is not None for k in ("input", "output", "cache_read")): out.usage_scope = "main" # the turns are the parent's; Codex reports no subagent tokens here diff --git a/dokimi/dokimi/repos/event_format.py b/dokimi/dokimi/repos/event_format.py index 670aff022..59142bf96 100644 --- a/dokimi/dokimi/repos/event_format.py +++ b/dokimi/dokimi/repos/event_format.py @@ -13,10 +13,14 @@ from typing import Any, Mapping -EVENT_VERSION = 1 # the major version this code writes; a version change adds repo_event.vN.json - -#: The kinds the converters write in version 1. Readers must accept any other kind (a later minor addition). -KNOWN_KINDS = frozenset({"command", "read", "edit", "tool"}) +EVENT_VERSION = 2 # the major version this code writes; a version change adds repo_event.vN.json + +#: The kinds that are tool activity: what ``tool_calls`` counts. +TOOL_KINDS = frozenset({"command", "read", "edit", "tool"}) +#: The per-model-call kinds (:mod:`dokimi.repos.turn_events`): not tool calls. +TURN_KINDS = frozenset({"turn", "compaction"}) +#: The kinds the converters write in version 2 (version 1 wrote the tool kinds only). Readers accept any other kind. +KNOWN_KINDS = TOOL_KINDS | TURN_KINDS def event_version(event: Mapping[str, Any]) -> int: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 59b840c64..a16333199 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -47,6 +47,7 @@ from dokimi.context.trace import parse_trace from .cli_traces import _failed_read, codex_json, copilot_json, shell_walk +from .turn_events import claude_events from .transcript import Transcript, _parse_time, clip_output, relative_path TRACE_FORMATS = ("claude-stream-json", "codex-json", "copilot-json", "text") @@ -126,8 +127,10 @@ def _claude_stream_json(raw: str, workdir: str | None, cwd: str = "") -> Transcr out.errors.append(f"line {n}: unexpected shape ({exc})") _walk_bash(out, stream) _settle_bash(out, stream.bash.values()) - main = parse_trace(raw).main_calls() + trace = parse_trace(raw) + main = trace.main_calls() out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None + out.events.extend(claude_events(trace, start)) return out diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index db4eb7b08..5f01ee31b 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -5,9 +5,9 @@ The framework counts what every repository asks of a trace: -- ``tool_calls``: every event of a known kind (:data:`~dokimi.repos.event_format.KNOWN_KINDS`), so every tool call - of the parent and its subagents, and each ``read`` event kept - from a shell command, a denied one or one of a failed multi-file read (``cat a missing``) too; +- ``tool_calls``: every tool event (:data:`~dokimi.repos.event_format.TOOL_KINDS`), so every tool call of the + parent and its subagents, and each ``read`` event kept from a shell command, a denied one or one of a + failed multi-file read (``cat a missing``) too; - ``commands``: the ``command`` events; - ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means no result came back, which is not a failure; a permission denial carries ``rc`` 1 but never ran); @@ -35,7 +35,7 @@ from pathlib import PurePosixPath from typing import Any -from dokimi.repos.event_format import KNOWN_KINDS +from dokimi.repos.event_format import TOOL_KINDS COUNTER_FIELDS = ("text", "output", "any") @@ -91,7 +91,7 @@ def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[ reads = {w for e in events if e.get("kind") == "read" and e.get("path") and not e.get("denied") and e.get("rc") in (0, None) and (w := _workspace_path(str(e["path"]))) is not None} return { - "tool_calls": sum(e.get("kind") in KNOWN_KINDS for e in events), + "tool_calls": sum(e.get("kind") in TOOL_KINDS for e in events), "commands": len(commands), "failed_commands": sum(e.get("rc") not in (0, None) and not e.get("denied") for e in commands), "denied_commands": sum(bool(e.get("denied")) for e in commands), diff --git a/dokimi/dokimi/repos/turn_events.py b/dokimi/dokimi/repos/turn_events.py new file mode 100644 index 000000000..376bc1588 --- /dev/null +++ b/dokimi/dokimi/repos/turn_events.py @@ -0,0 +1,78 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The per-model-call events of ``events.jsonl``: ``turn`` and ``compaction`` (docs/events-format.md). + +A ``turn`` is one model call and the tokens it was billed: ``input`` (uncached), ``output``, +``cache_read``, ``cache_write`` and ``prompt_tokens`` (the whole prompt, ``input + cache_read + +cache_write``). Claude's come from :mod:`dokimi.context.trace` (the parse the context budget uses); +their sums should match the run's totals when the result's usage equals its per-call usage. +Codex's come from its ``turn.completed`` usage. Copilot's trace carries no usage, so it has no +turn events. A count the harness did not report is ``null``, never ``0``. + +A ``compaction`` follows the turn it came after; its ``turn`` is that turn's index, or ``null`` before +the first call. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from dokimi.context.trace import ContextTrace + +from .transcript import _parse_time + + +def claude_events(trace: ContextTrace, start: datetime | None) -> list[dict[str, Any]]: + """The turn events of every model call (a subagent's carry ``agent``), then the compactions.""" + events: list[dict[str, Any]] = [] + for call in trace.calls: + seen = set(call["reported"]) + prompt = seen & {"input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"} + event: dict[str, Any] = { + "kind": "turn", "t": _seconds(call["at"], start), "turn": call["i"], + "input": call["input_tokens"] if "input_tokens" in seen else None, + "output": call["output_tokens"] if "output_tokens" in seen else None, + "cache_read": call["cache_read_tokens"] if "cache_read_input_tokens" in seen else None, + "cache_write": call["cache_write_tokens"] if "cache_creation_input_tokens" in seen else None, + "prompt_tokens": (call["input_tokens"] + call["cache_read_tokens"] + call["cache_write_tokens"] + if prompt else None), + } + if call["agent"]: + event["agent"] = str(call["agent"]) + events.append(event) + for compaction in trace.compactions: + after = compaction["after_call"] + events.append({"kind": "compaction", "t": _seconds(compaction["at"], start), + "turn": after if after >= 0 else None}) + return events + + +def codex_turn(turn: int, usage: Mapping[str, Any]) -> dict[str, Any]: + """The turn event of one Codex ``turn.completed`` usage. Codex counts the cached part inside + ``input_tokens``, so ``input`` is the uncached rest and ``prompt_tokens`` the reported total. + A usage with no tokens is no measured turn: every count is ``null``.""" + fresh = int(usage.get("input_tokens") or 0) + produced = int(usage.get("output_tokens") or 0) + cached = max(int(usage.get("cached_input_tokens") or 0), 0) # a negative count is malformed, not a credit + measured = bool(fresh or produced or cached) # a missing or all-zero usage is not a measured zero + has_input = measured and usage.get("input_tokens") is not None # each count is null on its own when absent + has_output = measured and usage.get("output_tokens") is not None + has_cached = measured and usage.get("cached_input_tokens") is not None # reported, so a zero is measured + return { + "kind": "turn", "t": None, "turn": turn, + "input": max(fresh - cached, 0) if has_input else None, + "output": produced if has_output else None, + "cache_read": cached if has_cached else None, + "cache_write": None, # Codex reports no cache writes + "prompt_tokens": fresh if has_input else None, + } + + +def _seconds(stamp: Any, start: datetime | None) -> float | None: + when = _parse_time(stamp) + if when is None or start is None: + return None + return round((when - start).total_seconds(), 3) diff --git a/dokimi/dokimi/repos/useful.py b/dokimi/dokimi/repos/useful.py index 3a878c430..57e086fe3 100644 --- a/dokimi/dokimi/repos/useful.py +++ b/dokimi/dokimi/repos/useful.py @@ -5,9 +5,10 @@ The first event of the run that hits the task's definition (:mod:`.useful_spec`): a ``command`` event whose text matches and whose ``rc`` is 0, or an ``edit`` event whose path matches a glob. A refused event (``denied``) never -hits, nor does an edit whose tool result is not a success (``rc`` 0 is the only success). ``steps`` counts the events -of a known kind up to and including the hit, as ``process.tool_calls`` does. ``issued_s`` is the hit's ``t``: the time -the call was issued, since its result is not timed, so a test run's pass is not timed. +hits, nor does an edit whose tool result is not a success (``rc`` 0 is the only success). ``steps`` counts the tool +events up to and including the hit, as ``process.tool_calls`` does; turn and compaction events are not steps. +``issued_s`` is the hit's ``t``: the time the call was issued, since its result +is not timed, so a test run's pass is not timed. """ from __future__ import annotations @@ -18,9 +19,9 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Any +from .event_format import TOOL_KINDS from .manifest import match_glob from .process_counts import as_measure -from .event_format import KNOWN_KINDS from .process_metrics import _workspace_path from .useful_spec import Useful @@ -68,8 +69,11 @@ def first_useful( return _record(None, "not_declared") if events is None: return _record(useful.kind, "unobserved") - known = (e for e in events if e.get("kind") in KNOWN_KINDS) # steps count known kinds, as tool_calls does - for steps, event in enumerate(known, 1): + steps = 0 + for event in events: + if event.get("kind") not in TOOL_KINDS: # a turn or compaction is accounting, not a step + continue + steps += 1 try: found = hits(useful, event) except TimeoutError: # the pattern ran past its budget on this event: no verdict diff --git a/dokimi/dokimi/schemas/repo_event.v2.json b/dokimi/dokimi/schemas/repo_event.v2.json new file mode 100644 index 000000000..ee39adcac --- /dev/null +++ b/dokimi/dokimi/schemas/repo_event.v2.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/repo_event.v2.json", + "title": "One line of events.jsonl, format version 2", + "description": "A line of a repository run's normalized trace (docs/events-format.md). One JSON object per line; this schema validates one line. Within version 2 a kind and a field are only ever added, so a reader must ignore a kind it does not know (this schema allows any kind) and a field it does not know. A line with no `v` is a version 1 line, written before the version field existed; version 1 lines validate against repo_event.v1.json.", + "type": "object", + "properties": { + "v": {"const": 2, "description": "The format version of this line. Absent on a line written before the field existed."}, + "kind": {"type": "string", "description": "What the event is. Readers ignore a kind they do not know."}, + "t": {"type": ["number", "null"], "description": "Seconds since the first timestamped trace line; null when the harness has no timestamps."}, + "agent": {"type": "string", "description": "The id of the parent's Agent/Task call that started this event; absent on the parent's events."}, + "denied": {"type": "boolean", "description": "The call was refused and never ran."} + }, + "required": ["kind", "t"], + "allOf": [ + { + "if": {"properties": {"kind": {"const": "command"}}, "required": ["kind"]}, + "then": { + "required": ["text", "rc"], + "properties": { + "text": {"type": "string"}, + "rc": {"type": ["integer", "null"]}, + "output": {"type": "string", "description": "The command's output as the harness returned it, clipped to 8 KB."}, + "output_truncated": {"type": "boolean"} + } + } + }, + { + "if": {"properties": {"kind": {"const": "read"}}, "required": ["kind"]}, + "then": { + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}} + } + }, + { + "if": {"properties": {"kind": {"const": "edit"}}, "required": ["kind"]}, + "then": { + "required": ["path"], + "properties": {"path": {"type": "string"}, "rc": {"type": ["integer", "null"]}, "deleted": {"type": "boolean"}} + } + }, + { + "if": {"properties": {"kind": {"const": "tool"}}, "required": ["kind"]}, + "then": { + "required": ["name"], + "properties": {"name": {"type": "string"}, "skill": {"type": "string"}} + } + }, + { + "if": {"properties": {"kind": {"const": "turn"}}, "required": ["kind"]}, + "then": { + "required": ["turn", "input", "output", "cache_read", "cache_write", "prompt_tokens"], + "properties": { + "turn": {"type": "integer", "minimum": 0}, + "input": {"type": ["integer", "null"]}, + "output": {"type": ["integer", "null"]}, + "cache_read": {"type": ["integer", "null"]}, + "cache_write": {"type": ["integer", "null"]}, + "prompt_tokens": {"type": ["integer", "null"]} + } + } + }, + { + "if": {"properties": {"kind": {"const": "compaction"}}, "required": ["kind"]}, + "then": { + "required": ["turn"], + "properties": {"turn": {"type": ["integer", "null"], "minimum": 0}} + } + } + ] +} diff --git a/dokimi/tests/fixtures/artifacts/repo_event.v2.json b/dokimi/tests/fixtures/artifacts/repo_event.v2.json new file mode 100644 index 000000000..0b1bad3a3 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/repo_event.v2.json @@ -0,0 +1,9 @@ +{ + "kind": "command", + "output": "ERROR: icpx is not found", + "output_truncated": false, + "rc": 1, + "t": 31.0, + "text": "bazel test :x", + "v": 2 +} diff --git a/dokimi/tests/test_context_trace.py b/dokimi/tests/test_context_trace.py index b989c0044..8da20430a 100644 --- a/dokimi/tests/test_context_trace.py +++ b/dokimi/tests/test_context_trace.py @@ -94,7 +94,7 @@ def test_tool_results_failures_and_repeats(): assert s["repeated_tool_calls"] == 1 assert s["tool_result_chars"] == 300 + 300 + len("File does not exist.") + 50 + 4 assert s["largest_tool_results"][0] == {"tool": "Bash", "calls": 3, "chars": 650, "max_chars": 300} - assert trace.compactions == [{"trigger": "auto", "pre_tokens": 20815, "post_tokens": 9000, "after_call": 2}] + assert trace.compactions == [{"trigger": "auto", "pre_tokens": 20815, "post_tokens": 9000, "after_call": 2, "at": None}] def test_without_a_window_there_is_no_fill(): @@ -342,8 +342,8 @@ def test_camel_case_and_bare_compactions(): {"type": "system", "subtype": "compact_boundary"}, )) assert got.compactions == [ - {"trigger": "manual", "pre_tokens": 7, "post_tokens": None, "after_call": 0}, - {"trigger": None, "pre_tokens": None, "post_tokens": None, "after_call": 0}, + {"trigger": "manual", "pre_tokens": 7, "post_tokens": None, "after_call": 0, "at": None}, + {"trigger": None, "pre_tokens": None, "post_tokens": None, "after_call": 0, "at": None}, ] diff --git a/dokimi/tests/test_repo_cli_pins.py b/dokimi/tests/test_repo_cli_pins.py index 58406abf0..dae5b9b49 100644 --- a/dokimi/tests/test_repo_cli_pins.py +++ b/dokimi/tests/test_repo_cli_pins.py @@ -81,7 +81,7 @@ def test_a_turn_marks_the_run_as_started_even_with_no_event(): codex = convert("codex-json", _jsonl({"type": "turn.completed", "usage": {"input_tokens": 1, "output_tokens": 1}}), "/ws") copilot = convert("copilot-json", _jsonl({"type": "assistant.turn_start", "data": {}}), "/ws") for trace in (codex, copilot): - assert trace.events == [] and trace.result_seen is True + assert [e for e in trace.events if e["kind"] != "turn"] == [] and trace.result_seen is True assert convert("codex-json", "", "/ws").result_seen is False assert convert("copilot-json", "", "/ws").result_seen is False for raw in ('{"type":"error","message":"x"}\n', "not json at all\n", '{"type":"other"}\n'): diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fe9d5c381..03cedf605 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -419,7 +419,7 @@ def test_codex_json_to_events(): t = convert("codex-json", raw, "/ws") assert t.answer == "DONE" and t.num_turns == 1 and (t.input_tokens, t.output_tokens) == (100, 7) assert t.cost_usd is None # Codex reports no cost: the batch budget counts the cap - assert t.events == [ + assert [e for e in t.events if e["kind"] != "turn"] == [ {"t": None, "kind": "read", "path": "AGENTS.md"}, {"t": None, "kind": "command", "text": "cat /ws/AGENTS.md", "rc": 0, "output": "# guide\n", "output_truncated": False}, @@ -554,8 +554,9 @@ def test_codex_incomplete_command_is_recorded_with_rc_null(): {"type": "turn.completed", "usage": {"input_tokens": 10, "output_tokens": 1}}, ) t = convert("codex-json", raw, None) - assert len(t.events) == 1 - assert t.events[0] == {"t": None, "kind": "command", "text": "make build", "rc": None} + steps = [e for e in t.events if e["kind"] != "turn"] # the turn event is accounting, not the command + assert len(steps) == 1 + assert steps[0] == {"t": None, "kind": "command", "text": "make build", "rc": None} def test_codex_failed_command_does_not_count_as_a_read(): diff --git a/dokimi/tests/test_repo_event_format.py b/dokimi/tests/test_repo_event_format.py index e20558795..150bc44b0 100644 --- a/dokimi/tests/test_repo_event_format.py +++ b/dokimi/tests/test_repo_event_format.py @@ -3,7 +3,7 @@ """The version field and the schema of ``events.jsonl`` (docs/events-format.md). -The schema (``repo_event.v1.json``) pins what version 1 lines look like; the converters' output +The schema (``repo_event.v2.json``) pins what version 2 lines look like; the converters' output must validate against it, an unversioned line must still read, and a reader must skip a kind it does not know. """ @@ -48,6 +48,7 @@ def result(tool_id: str, text: str, stamp: str) -> dict: stamp="2026-11-14T10:00:02Z"), result("e1", "ok", "2026-11-14T10:00:03Z"), result("w1", "denied", "2026-11-14T10:00:03Z"), + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:04Z"}, {"type": "result", "result": "done", "num_turns": 2, "is_error": False, "subtype": "success", "usage": {"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100}}, ) @@ -78,22 +79,22 @@ def test_every_converted_line_is_versioned_and_validates(): assert {e["kind"] for e in lines} == KNOWN_KINDS # the fixtures cover every kind the schema names for event in lines: assert event["v"] == EVENT_VERSION - assert schema_errors("repo_event.v1", event) == [], event + assert schema_errors("repo_event.v2", event) == [], event @pytest.mark.parametrize("line", [ - {"kind": "command", "text": "ls", "t": 1.0, "v": 1}, - {"kind": "command", "rc": 0, "t": 1.0, "v": 1}, - {"kind": "edit", "t": 1.0, "v": 1}, - {"kind": "tool", "t": 1.0, "v": 1}, - {"kind": "read", "path": 3, "t": 1.0, "v": 1}, - {"kind": "command", "text": "ls", "rc": 0, "t": "1.0", "v": 1}, - {"kind": "read", "path": "a.md", "v": 1}, - {"t": 1.0, "v": 1}, + {"kind": "command", "text": "ls", "t": 1.0, "v": 2}, + {"kind": "command", "rc": 0, "t": 1.0, "v": 2}, + {"kind": "edit", "t": 1.0, "v": 2}, + {"kind": "tool", "t": 1.0, "v": 2}, + {"kind": "read", "path": 3, "t": 1.0, "v": 2}, + {"kind": "command", "text": "ls", "rc": 0, "t": "1.0", "v": 2}, + {"kind": "read", "path": "a.md", "v": 2}, + {"t": 1.0, "v": 2}, ], ids=["command-no-rc", "command-no-text", "edit-no-path", "tool-no-name", "read-path-int", "string-t", "no-t", "no-kind"]) -def test_a_line_that_breaks_the_version_1_shape_is_refused(line): - assert schema_errors("repo_event.v1", line) != [] +def test_a_line_that_breaks_the_version_2_shape_is_refused(line): + assert schema_errors("repo_event.v2", line) != [] def test_the_writer_only_adds_the_version_to_each_event(): @@ -131,7 +132,7 @@ def test_unversioned_and_converted_give_the_same_process_metrics(): def test_the_schema_names_exactly_the_known_kinds(): - schema = json.loads((SCHEMA_DIR / "repo_event.v1.json").read_text(encoding="utf-8")) + schema = json.loads((SCHEMA_DIR / "repo_event.v2.json").read_text(encoding="utf-8")) named = {clause["if"]["properties"]["kind"]["const"] for clause in schema["allOf"]} assert named == KNOWN_KINDS @@ -141,3 +142,11 @@ def test_the_doc_names_every_known_kind_and_the_version_field(): for kind in sorted(KNOWN_KINDS): assert f"| `{kind}` |" in text, kind assert "`v`" in text and "repo_event.v2.json" in text + + +def test_the_doc_examples_validate_against_the_version_2_schema(): + block = DOCS.read_text(encoding="utf-8").split("```json\n", 1)[1].split("```", 1)[0] + lines = [json.loads(line) for line in block.splitlines() if line.strip()] + assert len(lines) >= 7 + for line in lines: + assert schema_errors("repo_event.v2", line) == [], line diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 9a95700eb..3320fb940 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -216,7 +216,7 @@ def test_a_bash_read_of_the_probe_shows_whether_the_directory_guidance_loads(tmp assert [s["id"] for s in payload["scenarios"]] == ["bash=sub"] and not payload["scenarios"][0]["error"] assert payload["summary"]["matrix"] == {"AGENTS.md": {"bash=sub": "loaded"}, "sub/AGENTS.md": {"bash=sub": cls}} events = [json.loads(x) for x in (tmp_path / "out/traces/00-bash_sub.events.jsonl").read_text().splitlines()] - assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0, "v": 1} + assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0, "v": 2} @pytest.mark.parametrize("mode", ["hook", "permissions"]) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index f816d6b01..20c42c05b 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -57,7 +57,7 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert meta["agent_network"] == "host" assert meta["egress_proxy"] is False and len(meta["start_sha"]) == 40 events = [json.loads(line) for line in (rd / "events.jsonl").read_text().splitlines()] - assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None, "v": 1}] # relative to /eval/run/workspace + assert events == [{"kind": "edit", "path": "src/calc.py", "rc": 0, "t": None, "v": 2}] # relative to /eval/run/workspace assert (rd / "answer.txt").read_text() == "done: src/calc.py" assert (rd / "config/settings.json").read_text() == '{"hook": "python3 /eval/support/no_background.py"}' result = json.loads((rd / "result.json").read_text()) diff --git a/dokimi/tests/test_repo_turn_events.py b/dokimi/tests/test_repo_turn_events.py new file mode 100644 index 000000000..ad56b04b7 --- /dev/null +++ b/dokimi/tests/test_repo_turn_events.py @@ -0,0 +1,228 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``turn`` and ``compaction`` events of ``events.jsonl`` (docs/events-format.md). + +The acceptance rule: the turns of a complete Claude trace sum to the run's own token totals, in the +scope the totals were taken in (tree with subagents, or the parent's alone). +""" + +from __future__ import annotations + +import json + +from dokimi.artifacts import schema_errors +from dokimi.context.consistency import run_from_events +from dokimi.repos.event_format import TOOL_KINDS +from dokimi.repos.events import convert +from dokimi.repos.process_metrics import process_metrics + +TURN_FIELDS = ("input", "output", "cache_read", "cache_write") + + +def _lines(*records: dict) -> str: + return "\n".join(json.dumps(r) for r in records) + "\n" + + +def _usage(inp: int, out: int, read: int, create: int) -> dict: + return {"input_tokens": inp, "output_tokens": out, "cache_read_input_tokens": read, + "cache_creation_input_tokens": create} + + +def _call(msg_id: str, usage: dict, stamp: str, *, blocks: list | None = None, parent: str | None = None) -> dict: + record = {"type": "assistant", "timestamp": stamp, + "message": {"id": msg_id, "model": "m", "content": blocks or [], "usage": usage}} + if parent is not None: + record["parent_tool_use_id"] = parent + return record + + +def _claude_run(*, tree: bool) -> str: + """Three model calls: the parent's first call split over two records (one message id, a fuller output on the + second), a subagent's call, then the parent's second call; a compaction after the first call.""" + main_a = _usage(3, 5, 100, 0) + sub = _usage(2, 4, 50, 10) + main_b = _usage(1, 9, 120, 20) + if tree: # the result's modelUsage: the scope the totals sum over subagents + totals = {"inputTokens": 6, "outputTokens": 20, "cacheReadInputTokens": 270, "cacheCreationInputTokens": 30} + result_usage: dict = {"modelUsage": {"m": totals}} + else: # the parent's usage alone + result_usage = {"usage": {"input_tokens": 4, "output_tokens": 16, "cache_read_input_tokens": 220, + "cache_creation_input_tokens": 20}} + return _lines( + {"type": "system", "subtype": "init", "model": "m", "claude_code_version": "2.0.0", "tools": []}, + _call("a", main_a, "2026-11-14T10:00:00Z", blocks=[{"type": "text", "text": "thinking"}]), + _call("a", {**main_a, "output_tokens": 7}, "2026-11-14T10:00:00.500Z", + blocks=[{"type": "tool_use", "id": "agent-1", "name": "Agent", "input": {"prompt": "x"}}]), + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:03Z", + "compact_metadata": {"trigger": "auto", "pre_tokens": 150000}}, + _call("s", sub, "2026-11-14T10:00:02Z", parent="agent-1"), + _call("b", main_b, "2026-11-14T10:00:05Z"), + {"type": "result", "result": "done", "num_turns": 2, "is_error": False, "subtype": "success", + **result_usage}, + ) + + +def _turns(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] == "turn"] + + +def test_the_turns_of_a_tree_scope_run_sum_to_its_totals(): + transcript = convert("claude-stream-json", _claude_run(tree=True)) + turns = _turns(transcript.events) + assert transcript.usage_scope == "tree" + assert [sum(t[f] for t in turns) for f in TURN_FIELDS] == [ + transcript.input_tokens, transcript.output_tokens, transcript.cache_read_tokens, transcript.cache_write_tokens] + + +def test_the_turns_of_a_main_scope_run_sum_to_the_parents_totals(): + transcript = convert("claude-stream-json", _claude_run(tree=False)) + parent = [t for t in _turns(transcript.events) if "agent" not in t] + assert transcript.usage_scope == "main" + assert [sum(t[f] for t in parent) for f in TURN_FIELDS] == [ + transcript.input_tokens, transcript.output_tokens, transcript.cache_read_tokens, transcript.cache_write_tokens] + assert [t["agent"] for t in _turns(transcript.events) if "agent" in t] == ["agent-1"] + + +def test_one_turn_per_model_call_with_its_prompt_and_its_time(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + turns = _turns(events) + assert [t["turn"] for t in turns] == [0, 1, 2] # the subagent's call counts; the split call is one call + assert [t["output"] for t in turns] == [7, 4, 9] # the fuller of the two records + assert [t["prompt_tokens"] for t in turns] == [103, 62, 141] # input + cache_read + cache_write + assert [t["t"] for t in turns] == [0.0, 2.0, 5.0] # seconds from the first timestamped line + + +def test_a_compaction_follows_the_turn_it_came_after(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + assert [e for e in events if e["kind"] == "compaction"] == [{"kind": "compaction", "t": 3.0, "turn": 0}] + + +def test_a_call_without_usage_writes_no_turn_and_a_compaction_after_it_has_none(): + raw = _lines( + {"type": "assistant", "timestamp": "2026-11-14T10:00:00Z", + "message": {"id": "a1", "model": "m", "content": [], "usage": None}}, + {"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:05Z"}, + ) + events = convert("claude-stream-json", raw).events + assert _turns(events) == [] + assert [e for e in events if e["kind"] == "compaction"] == [{"kind": "compaction", "t": 5.0, "turn": None}] + + +def test_a_compaction_before_any_call_has_no_turn(): + raw = _lines({"type": "system", "subtype": "compact_boundary", "timestamp": "2026-11-14T10:00:00Z"}) + assert [e for e in convert("claude-stream-json", raw).events if e["kind"] == "compaction"] == [ + {"kind": "compaction", "t": 0.0, "turn": None}] + + +def test_a_codex_turn_with_only_cached_input_reports_that_count(): + raw = _lines({"type": "turn.completed", "usage": {"cached_input_tokens": 5}}) + assert _turns(convert("codex-json", raw).events) == [ + {"kind": "turn", "t": None, "turn": 0, "input": None, "output": None, "cache_read": 5, "cache_write": None, + "prompt_tokens": None}] + + +def test_codex_turns_come_from_turn_completed_and_a_missing_count_is_null(): + raw = _lines( + {"type": "turn.completed", "usage": {"input_tokens": 10, "cached_input_tokens": 4, "output_tokens": 3}}, + {"type": "turn.completed", "usage": {"input_tokens": 6, "output_tokens": 2}}, + {"type": "turn.completed"}, + ) + transcript = convert("codex-json", raw) + turns = _turns(transcript.events) + assert turns == [ + {"kind": "turn", "t": None, "turn": 0, "input": 6, "output": 3, "cache_read": 4, "cache_write": None, + "prompt_tokens": 10}, + {"kind": "turn", "t": None, "turn": 1, "input": 6, "output": 2, "cache_read": None, "cache_write": None, + "prompt_tokens": 6}, + {"kind": "turn", "t": None, "turn": 2, "input": None, "output": None, "cache_read": None, "cache_write": None, + "prompt_tokens": None}, + ] + assert transcript.input_tokens == 12 and transcript.output_tokens == 5 and transcript.cache_read_tokens == 4 + + +def test_copilot_writes_no_turn_events(): + raw = _lines({"type": "assistant.message", "data": {"content": "done"}}) + assert _turns(convert("copilot-json", raw).events) == [] + + +def test_turn_and_compaction_events_are_not_tool_calls_or_tool_path_steps(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + tool_only = [e for e in events if e["kind"] in TOOL_KINDS] + assert len(tool_only) < len(events) # the run has turn and compaction events + assert process_metrics(events, lambda _: False)["tool_calls"] == len(tool_only) + assert run_from_events(events).kinds() == run_from_events(tool_only).kinds() + + +def test_every_turn_and_compaction_line_validates_against_the_schema(): + events = convert("claude-stream-json", _claude_run(tree=True)).events + for event in events: + if event["kind"] in ("turn", "compaction"): + assert schema_errors("repo_event.v2", {**event, "v": 2}) == [], event + + +def test_a_turn_without_its_counts_does_not_validate(): + turn = {"kind": "turn", "t": None, "turn": 0, "input": 1, "output": 1} # no cache or prompt counts + assert schema_errors("repo_event.v1", {**turn, "v": 1}) != [] + + +def test_a_claude_usage_missing_counts_are_null_not_zero(): + raw = _lines(_call("a", {"input_tokens": 10, "output_tokens": 2}, "2026-11-14T10:00:00Z"), + _call("b", {"input_tokens": 5}, "2026-11-14T10:00:01Z"), + _call("b", {"output_tokens": 4}, "2026-11-14T10:00:02Z"), + _call("c", {"input_tokens": 3}, "2026-11-14T10:00:03Z")) + assert _turns(convert("claude-stream-json", raw).events) == [ + {"kind": "turn", "t": 0.0, "turn": 0, "input": 10, "output": 2, "cache_read": None, "cache_write": None, + "prompt_tokens": 10}, + {"kind": "turn", "t": 1.0, "turn": 1, "input": 5, "output": 4, "cache_read": None, "cache_write": None, + "prompt_tokens": 5}, + {"kind": "turn", "t": 3.0, "turn": 2, "input": 3, "output": None, "cache_read": None, "cache_write": None, + "prompt_tokens": 3}, + ] + + +def test_an_output_only_claude_usage_leaves_input_and_prompt_tokens_null(): + turn = _turns(convert("claude-stream-json", _lines(_call("a", {"output_tokens": 3}, "2026-11-14T10:00:00Z"))) + .events)[0] + assert turn["input"] is None and turn["prompt_tokens"] is None and turn["output"] == 3 + + +def test_a_split_message_prompt_tokens_is_the_sum_of_its_parts(): + raw = _lines(_call("a", {"input_tokens": 5, "output_tokens": 1, "cache_creation_input_tokens": 0}, + "2026-11-14T10:00:00Z"), + _call("a", {"cache_read_input_tokens": 10}, "2026-11-14T10:00:01Z")) + turn = _turns(convert("claude-stream-json", raw).events)[0] + assert turn["input"] == 5 and turn["cache_read"] == 10 and turn["cache_write"] == 0 + assert turn["prompt_tokens"] == 15 + + +def test_a_codex_usage_with_one_side_reports_the_other_as_null(): + raw = _lines({"type": "turn.completed", "usage": {"input_tokens": 6}}, + {"type": "turn.completed", "usage": {"output_tokens": 2}}) + transcript = convert("codex-json", raw) + assert [[t[k] for k in ("input", "output", "prompt_tokens")] for t in _turns(transcript.events)] == [ + [6, None, 6], [None, 2, None]] + assert transcript.input_tokens == 6 and transcript.output_tokens == 2 + + +def test_a_turn_of_a_non_string_parent_tool_use_id_validates_against_the_schema(): + raw = _lines(_call("a", _usage(3, 1, 0, 0), "2026-11-14T10:00:00Z", parent=7)) + turns = _turns(convert("claude-stream-json", raw).events) + assert len(turns) == 1 and turns[0]["agent"] == "7" + assert schema_errors("repo_event.v2", {**turns[0], "v": 2}) == [] + + +def test_a_turn_before_the_first_useful_command_is_not_a_step(): + from dokimi.repos.useful import first_useful + from dokimi.repos.useful_spec import parse_useful + raw = _lines({"type": "turn.completed", "usage": {"input_tokens": 3, "output_tokens": 1}}, + {"type": "item.completed", "item": {"id": "c", "type": "command_execution", "command": "pytest -q", + "aggregated_output": "", "exit_code": 0, "status": "completed"}}) + events = convert("codex-json", raw).events + assert [e["kind"] for e in events][:2] == ["turn", "command"] + assert first_useful(parse_useful({"command": "pytest"}), events)["steps"] == 1 + + +def test_a_codex_turn_with_only_cached_input_sets_the_scope_of_its_totals(): + transcript = convert("codex-json", _lines({"type": "turn.completed", "usage": {"cached_input_tokens": 5}})) + assert (transcript.cache_read_tokens, transcript.usage_scope) == (5, "main") diff --git a/dokimi/tests/test_skill_reach.py b/dokimi/tests/test_skill_reach.py index dee2c1df8..c5f355a49 100644 --- a/dokimi/tests/test_skill_reach.py +++ b/dokimi/tests/test_skill_reach.py @@ -871,8 +871,8 @@ def test_a_skill_reach_trace_is_versioned_and_validates(tmp_path, monkeypatch): for line in p.read_text(encoding="utf-8").splitlines()] assert lines for event in lines: - assert event["v"] == 1 - assert schema_errors("repo_event.v1", event) == [], event + assert event["v"] == 2 + assert schema_errors("repo_event.v2", event) == [], event def test_a_new_run_clears_the_traces_of_an_earlier_one(tmp_path):