From 23a0cae038432ebc0578a4c93232165f28068c39 Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 15:54:34 -0800 Subject: [PATCH 1/6] docs(codex): pin compressed rollout and shared-history storage protocol --- docs/groundtruth/RESEARCH_PLAN.md | 29 ++ groundtruth/codex/groundtruth.html | 2 +- groundtruth/codex/groundtruth.json | 443 +++++++++++++++-- groundtruth/codex/layout.toml | 24 + .../codex/sources/0.149.1.fixtures.json | 24 + groundtruth/codex/sources/0.167.2.writer.json | 451 ++++++++++++++++++ groundtruth/codex/watch.toml | 4 +- 7 files changed, 947 insertions(+), 30 deletions(-) create mode 100644 groundtruth/codex/sources/0.167.2.writer.json diff --git a/docs/groundtruth/RESEARCH_PLAN.md b/docs/groundtruth/RESEARCH_PLAN.md index 77e734b2..da98e157 100644 --- a/docs/groundtruth/RESEARCH_PLAN.md +++ b/docs/groundtruth/RESEARCH_PLAN.md @@ -518,6 +518,35 @@ rolling alphas; track `latest` only. must include compaction; optional state columns and crate reorgs require the schema and `watch.toml` evidence to be refreshed together. +**Storage and lineage closure (2026-11-12).** Pinned `rust-v0.167.2`, commit +`ce21072c90d32c40af7a55b60b2fb35c84a43a6a`, in `sources/0.167.2.writer.json`. +The same rollout can be plain `.jsonl` or `.jsonl.zst`, under dated `sessions/` +or flat `archived_sessions/`. Compression preserves mtime; resume materializes +plain before removing compressed. Read offsets and `history_base.end_byte_offset` +are **decoded JSONL bytes**. Zstd content size is optional and concatenated frames +are valid. The worker's seven-day mtime age, six-hour throttle, two workers and +level 3 are source evidence, not Rosetta assumptions about availability. + +`state_*.sqlite.threads.rollout_path` selects a thread's current rollout, including +reverts whose filename ends `_.jsonl`. A +`history_base.thread_id` names that immutable rollout; its byte and ordinal ends +are exclusive. Resolve recursively with cycle/depth/boundary checks, never by +choosing the newest file for a thread or appending all its branches. Missing +selected files or ancestors leave a transcript incomplete. Native state paths +from a relocated home bind only to recognized managed paths within the enrolled +root. `thread_history_1.sqlite` is a derived, potentially lagging projection and +is not substituted for raw rollouts. Attached children omit inherited parent +context from their displayed history; retained work from the same child remains. + +Rosetta uses bounded decoded header reads for admission, verifies compressed +inputs to an anonymous spool once per capture pass, and journals the selected +view's native headers and exact prefix records together. Original headers track +usage/artifact ownership; copying a parent's visible history never bills it twice. +A representation change verifies the whole committed decoded prefix before +resuming; a content change creates a replacement generation. No Codex files are +materialized or modified. Hermetic storage and live tests cover these mechanisms; +fixtures generated by the native CLI remain pinned to 0.149.1. + **Product admission policy (2026-09-09).** Internal approval reviewers are excluded from the canonical session/timeline/search projection during fresh ingest. Match native `source.subagent.other == "guardian"` or `thread_source == "guardian_review"` diff --git a/groundtruth/codex/groundtruth.html b/groundtruth/codex/groundtruth.html index 9331d678..b72da872 100644 --- a/groundtruth/codex/groundtruth.html +++ b/groundtruth/codex/groundtruth.html @@ -10,4 +10,4 @@ .fname{font-weight:600}.obs{color:var(--mut);font-size:.85rem;text-align:right}.chip{background:var(--chip);color:var(--chipfg);border-radius:4px;padding:.05rem .4rem;font-size:.85rem}.req{color:var(--fix)} .badge{border-radius:4px;padding:.05rem .45rem;font-size:.75rem;text-transform:lowercase}.ev-writer{color:var(--writer);border:1px solid var(--writer)}.ev-fixture{color:var(--fix);border:1px solid var(--fix)}.ev-observed{color:var(--obs);border:1px solid var(--obs)}.ev-research{color:var(--res);border:1px solid var(--res)} details{margin:.25rem 0}summary{cursor:pointer;padding:.35rem .5rem}details.rtype{background:var(--card);border:1px solid var(--line);border-radius:6px;padding:.15rem .4rem;margin:.5rem 0} -.rname{font-weight:700}.variant,.arr{color:var(--mut);font-size:.85rem;margin:.3rem .5rem}code{background:var(--card);padding:.05rem .3rem;border-radius:4px}a{color:var(--chipfg);text-decoration:none}a:hover{text-decoration:underline}

rosetta ground truth

codex 0.149.1 writer

root ~/.codex · generator tools.groundtruth canonical 2

canonical — merged from: sources/0.149.1.writer.json (0.149.1, writer, writer rust-v0.149.1) sha256:4071a3f3e945 · sources/0.150.1.writer.json (0.149.1, writer, writer rust-v0.150.1) sha256:6dec0213bc1c · sources/0.155.1.writer.json (0.155.1, writer, writer main@e7bbc79f) sha256:9a5f252f989e · sources/0.149.1.fixtures.json (0.149.1, fixture) sha256:1a370297cf65

streamglobformatdisciplineevidencetypesrecords
rolloutsessions/*/*/*/rollout-*.jsonljsonlappend_onlywriter1149
state-dbstate_*.sqlite*sqlitedatabasefixture13159
state-dbs*.sqlite*sqlitedatabasefixture1549
writer-locksthread-writer-locks/**/*textrewrite_in_placefixture00
installation-idinstallation_idtextrewrite_in_placefixture00
sandbox-migration.sandbox_migrationtextrewrite_in_placefixture00
configconfig.tomltomlrewrite_in_placefixture00
prompt-historyhistory.jsonljsonlappend_onlyresearch00
authauth.jsonjsonrewrite_in_placeresearch00
logslog/**/*textappend_onlyresearch00
versionversion.jsonjsonrewrite_in_placeresearch00
farm:headless-stdout<farm>/stdout*.jsonljsonlappend_onlyfixture522

rollout sessions/*/*/*/rollout-*.jsonl

Session rollouts under YYYY/MM/DD: the complete event stream. Top-level types (0.149.1): session_meta, event_msg, response_item, world_state, turn_context. Pinned writer evidence adds custom_tool_call/custom_tool_call_output to the fixture-backed response items. token_count carries CUMULATIVE total_token_usage, not per-message deltas. Roles include 'developer'. Can embed compaction history and grow large.

compacted writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: compacted(0)✓
payload object ✓
fieldtypereqobs
messagestring✓
replacement_history array<object>
array of:
fieldtypereqobs
replacement_history_metadata array<object>
array of:
fieldtypereqobs
mcp_resource_origins object
fieldtypereqobs
window_numbernumber
first_window_idstring
previous_window_idstring
window_idstring | number
event_msg fixture ×23
fieldtypereqobs
ordinalnumber✓23
payload object ✓ 23
fieldtypereqobs
collaboration_mode_kindenum: default(4)4
completed_atnumber4
completed_at_msnumber9
duration_msnumber4
info object 5
fieldtypereqobs
last_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
model_context_windownumber✓5
total_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
item object 9
fieldtypereqobs
aggregated_outputstring1
commandarray<string>1
content array<object> 8
array of:
fieldtypereqobs
textstring✓8
text_elementsarray4
typeenum: Text(4) | text(4)✓8
cwdstring1
duration object 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber1
formatted_outputstring1
idstring✓9
parsed_cmd array<object> 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring1
sourceenum: unified_exec_startup(1)1
statusenum: completed(1)1
stderrstring1
stdoutstring1
typeenum: AgentMessage(4) | UserMessage(4) | CommandExecution(1)✓9
last_agent_messagestring4
model_context_windownumber4
rate_limits object 5
fieldtypereqobs
creditsnull✓5
individual_limitnull✓5
limit_idstring✓5
limit_namenull✓5
plan_typenull✓5
primarynull✓5
rate_limit_reached_typenull✓5
secondarynull✓5
spend_control_reachednull✓5
started_atnumber8
started_at_msnumber9
thread_idstring9
thread_settings object 1
fieldtypereqobs
active_permission_profile object ✓ 1
fieldtypereqobs
idstring✓1
approval_policystring✓1
approvals_reviewerstring✓1
collaboration_mode object ✓ 1
fieldtypereqobs
modeenum: default(1)✓1
settings object ✓ 1
fieldtypereqobs
developer_instructionsnull✓1
modelenum: mock-model(1)✓1
reasoning_effortnull✓1
cwdstring✓1
modelenum: mock-model(1)✓1
model_provider_idstring✓1
permission_profile object ✓ 1
fieldtypereqobs
file_system object ✓ 1
fieldtypereqobs
entries array<object> ✓ 1
array of:
fieldtypereqobs
accessstring✓1
path object ✓ 1
fieldtypereqobs
typeenum: special(1)✓1
value object ✓ 1
fieldtypereqobs
kindenum: root(1)✓1
typeenum: restricted(1)✓1
networkstring✓1
typeenum: managed(1)✓1
personalitystring✓1
time_to_first_token_msnumber4
turn_idstring17
typeenum: item_completed(9) | token_count(5) | task_complete(4) | task_started(4) | thread_settings_applied(1)✓23
timestampstring✓23
typeenum: event_msg(23)✓23
inter_agent_communication writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: inter_agent_communication(0)✓
payload object ✓
fieldtypereqobs
idstring
authorstring✓
recipientstring✓
other_recipientsarray<string>
contentstring✓
encrypted_contentstring
internal_chat_message_metadata_passthrough object
fieldtypereqobs
trigger_turnboolean✓
inter_agent_communication_metadata writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: inter_agent_communication_metadata(0)✓
payload object ✓
fieldtypereqobs
trigger_turnboolean✓
realtime_item writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: realtime_item(0)✓
payload object ✓
fieldtypereqobs
idstring✓
realtime_session_idstring✓
typestring✓
response_item fixture ×16
fieldtypereqobs
ordinalnumber✓16
payload object ✓ 16
fieldtypereqobs
argumentsstring1
call_idstring2
content array<object> 14
array of:
fieldtypereqobs
textstring✓17
typeenum: input_text(13) | output_text(4)✓17
idstring✓16
internal_chat_message_metadata_passthrough object ✓ 16
fieldtypereqobs
create_timenumber11
turn_idstring✓16
namestring1
outputstring1
roleenum: user(7) | assistant(4) | developer(3)14
typeenum: message(14) | function_call(1) | function_call_output(1)✓16
timestampstring✓16
typeenum: response_item(16)✓16
security_risk_score writer ×0
fieldtypereqobs
timestampstring✓
typeenum: security_risk_score(0)✓
payload object ✓
fieldtypereqobs
scores object ✓
fieldtypereqobs
sampled_atstring
session_meta fixture ×3
fieldtypereqobs
ordinalnumber✓3
payload object ✓ 3
fieldtypereqobs
base_instructions object ✓ 3
fieldtypereqobs
provenance object ✓ 3
fieldtypereqobs
modelenum: mock-model(3)✓3
typeenum: model(3)✓3
textstring✓3
cli_versionenum: 0.149.1(3)✓3
context_window object ✓ 3
fieldtypereqobs
window_idstring✓3
cwdstring✓3
history_modeenum: paginated(3)✓3
idstring✓3
model_providerenum: mock(3)✓3
originatorstring✓3
session_idstring✓3
sourceenum: exec(3)✓3
thread_sourceenum: user(3)✓3
timestampstring✓3
timestampstring✓3
typeenum: session_meta(3)✓3
token_usage_record writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: token_usage_record(0)✓
payload object ✓
fieldtypereqobs
thread_idstring✓
turn_idstring✓
session_idstring✓
root_turn_idstring✓
response_idstring✓
usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
turn_token_usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
thread_token_usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
turn_context fixture ×4
fieldtypereqobs
ordinalnumber✓4
payload object ✓ 4
fieldtypereqobs
active_permission_profile object 3
fieldtypereqobs
idstring✓3
approval_policystring✓4
approvals_reviewerstring✓4
collaboration_mode object ✓ 4
fieldtypereqobs
modeenum: default(4)✓4
settings object ✓ 4
fieldtypereqobs
developer_instructionsnull✓4
modelenum: mock-model(4)✓4
reasoning_effortnull✓4
current_datestring✓4
cwdstring✓4
modelenum: mock-model(4)✓4
multi_agent_versionenum: v1(4)✓4
permission_profile object ✓ 4
fieldtypereqobs
file_system object 3
fieldtypereqobs
entries array<object> ✓ 3
array of:
fieldtypereqobs
accessstring✓3
path object ✓ 3
fieldtypereqobs
typeenum: special(3)✓3
value object ✓ 3
fieldtypereqobs
kindenum: root(3)✓3
typeenum: restricted(3)✓3
networkstring3
typeenum: managed(3) | disabled(1)✓4
personalitystring✓4
realtime_activeboolean✓4
sandbox_policy object ✓ 4
fieldtypereqobs
typeenum: read-only(3) | danger-full-access(1)✓4
summarystring✓4
timezonestring✓4
turn_idstring✓4
workspace_rootsarray<string>✓4
timestampstring✓4
typeenum: turn_context(4)✓4
world_state fixture ×3
fieldtypereqobs
ordinalnumber✓3
payload object ✓ 3
fieldtypereqobs
fullboolean✓3
state object ✓ 3
fieldtypereqobs
agents_md object ✓ 3
fieldtypereqobs
apps_instructionsboolean✓3
collaboration_mode object ✓ 3
fieldtypereqobs
instructionsstring✓3
modeenum: default(3)✓3
modelenum: mock-model(3)✓3
environments object ✓ 3
fieldtypereqobs
current_datestring✓3
environments object ✓ 3
fieldtypereqobs
local object ✓ 3
fieldtypereqobs
cwdstring✓3
shellstring✓3
statusenum: available(3)✓3
filesystemstring✓3
timezonestring✓3
environments_instructionsboolean✓3
git_attributionboolean✓3
host_skills object ✓ 3
fieldtypereqobs
bodystring✓3
includeInstructionsboolean✓3
modelenum: mock-model(3)✓3
multi_agent_mode object ✓ 3
fieldtypereqobs
orchestrator_skills object ✓ 3
fieldtypereqobs
enabledboolean✓3
includeInstructionsboolean✓3
permissions object ✓ 3
fieldtypereqobs
approved_command_prefixesarray✓3
instructionsstring✓3
personality object ✓ 3
fieldtypereqobs
modelenum: mock-model(3)✓3
personalitystring✓3
plugins_instructionsboolean✓3
realtime object ✓ 3
fieldtypereqobs
activeboolean✓3
skills object ✓ 3
fieldtypereqobs
includeInstructionsboolean✓3
timestampstring✓3
typeenum: world_state(3)✓3

facet: event-msg by type — event_msg.payload (by payload.type)

agent_message writer ×0
fieldtypereqobs
typeenum: agent_message(0)✓
messagestring✓
phasestring
memory_citation object
fieldtypereqobs
agent_reasoning writer ×0
fieldtypereqobs
typeenum: agent_reasoning(0)✓
textstring✓
agent_reasoning_raw_content writer ×0
fieldtypereqobs
typeenum: agent_reasoning_raw_content(0)✓
context_compacted writer ×0
fieldtypereqobs
typeenum: context_compacted(0)✓
entered_review_mode writer ×0
fieldtypereqobs
typeenum: entered_review_mode(0)✓
exec_command_end writer ×0
fieldtypereqobs
typeenum: exec_command_end(0)✓
exited_review_mode writer ×0
fieldtypereqobs
typeenum: exited_review_mode(0)✓
image_generation_end writer ×0
fieldtypereqobs
typeenum: image_generation_end(0)✓
item_completed fixture ×9
fieldtypereqobs
completed_at_msnumber✓9
item object ✓ 9
fieldtypereqobs
aggregated_outputstring1
commandarray<string>1
content array<object> 8
array of:
fieldtypereqobs
textstring✓8
text_elementsarray4
typeenum: Text(4) | text(4)✓8
cwdstring1
duration object 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber1
formatted_outputstring1
idstring✓9
parsed_cmd array<object> 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring1
sourceenum: unified_exec_startup(1)1
statusenum: completed(1)1
stderrstring1
stdoutstring1
typeenum: AgentMessage(4) | UserMessage(4) | CommandExecution(1)✓9
started_at_msnumber✓9
thread_idstring✓9
turn_idstring✓9
typeenum: item_completed(9)✓9
mcp_tool_call_end writer ×0
fieldtypereqobs
typeenum: mcp_tool_call_end(0)✓
patch_apply_end writer ×0
fieldtypereqobs
typeenum: patch_apply_end(0)✓
sub_agent_activity writer ×0
fieldtypereqobs
typeenum: sub_agent_activity(0)✓
task_complete fixture ×4
fieldtypereqobs
completed_atnumber✓4
duration_msnumber✓4
last_agent_messagestring✓4
started_atnumber✓4
time_to_first_token_msnumber✓4
turn_idstring✓4
typeenum: task_complete(4)✓4
task_started fixture ×4
fieldtypereqobs
collaboration_mode_kindenum: default(4)✓4
model_context_windownumber✓4
started_atnumber✓4
turn_idstring✓4
typeenum: task_started(4)✓4
thread_goal_updated writer ×0
fieldtypereqobs
typeenum: thread_goal_updated(0)✓
thread_rolled_back writer ×0
fieldtypereqobs
typeenum: thread_rolled_back(0)✓
num_turnsnumber✓
thread_settings_applied writer ×1
fieldtypereqobs
thread_idstring
thread_settings object ✓
fieldtypereqobs
active_permission_profile object
fieldtypereqobs
idstring1
approval_policy object ✓
variant 1: string
variant 2:
fieldtypereqobs
approvals_reviewerstring✓
collaboration_mode object 1
fieldtypereqobs
modeenum: default(1)✓1
settings object ✓ 1
fieldtypereqobs
developer_instructionsnull✓1
modelenum: mock-model(1)✓1
reasoning_effortnull✓1
cwdstring✓
modelstring✓
model_provider_idstring✓
permission_profile object ✓
fieldtypereqobs
file_system object 1
fieldtypereqobs
entries array<object> ✓ 1
array of:
fieldtypereqobs
accessstring✓1
path object ✓ 1
fieldtypereqobs
typeenum: special(1)✓1
value object ✓ 1
fieldtypereqobs
kindenum: root(1)✓1
typeenum: restricted(1)✓1
networkstring1
typeenum: managed(1)1
personalitystring1
runtime_workspace_rootsarray<string>
service_tierstring
typeenum: thread_settings_applied(1)✓
token_count fixture ×5
fieldtypereqobs
info object ✓ 5
fieldtypereqobs
last_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
model_context_windownumber✓5
total_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
rate_limits object ✓ 5
fieldtypereqobs
creditsnull✓5
individual_limitnull✓5
limit_idstring✓5
limit_namenull✓5
plan_typenull✓5
primarynull✓5
rate_limit_reached_typenull✓5
secondarynull✓5
spend_control_reachednull✓5
typeenum: token_count(5)✓5
turn_aborted writer ×0
fieldtypereqobs
typeenum: turn_aborted(0)✓
reasonstring✓
user_message writer ×0
fieldtypereqobs
typeenum: user_message(0)✓
messagestring✓
imagesarray<string>✓
local_imagesarray<string>✓
text_elements array<object> ✓
array of:
fieldtypereqobs
web_search_end writer ×0
fieldtypereqobs
typeenum: web_search_end(0)✓

facet: response-items by type — response_item.payload (by payload.type)

agent_message writer ×0
fieldtypereqobs
typeenum: agent_message(0)✓
idstring
authorstring✓
recipientstring✓
content array<object> ✓
array of:
fieldtypereqobs
internal_chat_message_metadata_passthrough object
fieldtypereqobs
compaction writer ×0
fieldtypereqobs
typeenum: compaction(0)✓
idstring
encrypted_contentstring✓
context_compaction writer ×0
fieldtypereqobs
typeenum: context_compaction(0)✓
idstring
encrypted_contentstring
custom_tool_call writer ×0
fieldtypereqobs
idstring
statusstring
call_idstring✓
namestring✓
namespacestring
inputstring✓
internal_chat_message_metadata_passthrough object
fieldtypereqobs
typeenum: custom_tool_call(0)✓
custom_tool_call_output writer ×0
fieldtypereqobs
idstring
call_idstring✓
namestring
output object ✓
variant 1: string
variant 2:
array of:
fieldtypereqobs
internal_chat_message_metadata_passthrough object
fieldtypereqobs
typeenum: custom_tool_call_output(0)✓
function_call writer ×1
fieldtypereqobs
argumentsstring✓
call_idstring✓
encrypted_function_argsarray<string>
idstring
internal_chat_message_metadata_passthrough object 1
fieldtypereqobs
turn_idstring✓1
namestring✓
namespacestring
typeenum: function_call(1)✓
function_call_output fixture ×1
fieldtypereqobs
call_idstring✓1
idstring✓1
internal_chat_message_metadata_passthrough object ✓ 1
fieldtypereqobs
create_timenumber✓1
turn_idstring✓1
outputstring✓1
typeenum: function_call_output(1)✓1
image_generation_call writer ×0
fieldtypereqobs
typeenum: image_generation_call(0)✓
idstring
statusstring✓
revised_promptstring
resultstring✓
local_shell_call writer ×0
fieldtypereqobs
typeenum: local_shell_call(0)✓
idstring
call_idstring
statusstring✓
action object ✓
fieldtypereqobs
message writer ×14
fieldtypereqobs
content array<object> ✓
array of:
fieldtypereqobs
textstring17
typeenum: input_text(13) | output_text(4)17
idstring
internal_chat_message_metadata_passthrough object 14
fieldtypereqobs
create_timenumber10
turn_idstring✓14
phasestring
rolestring✓
typeenum: message(14)✓
reasoning writer ×0
fieldtypereqobs
typeenum: reasoning(0)✓
idstring
summary array<object> ✓
array of:
fieldtypereqobs
typeenum: summary_text(0)✓
textstring✓
content array<object>
array of:
fieldtypereqobs
encrypted_contentstring
internal_chat_message_metadata_passthrough object
fieldtypereqobs
tool_search_call writer ×0
fieldtypereqobs
typeenum: tool_search_call(0)✓
idstring
call_idstring
statusstring
executionstring✓
arguments?✓
tool_search_output writer ×0
fieldtypereqobs
typeenum: tool_search_output(0)✓
idstring
call_idstring
statusstring✓
executionstring✓
tools array<object> ✓
array of:
fieldtypereqobs
web_search_call writer ×0
fieldtypereqobs
typeenum: web_search_call(0)✓
idstring
statusstring
action object
fieldtypereqobs

facet: content-blocks by type — response_item.payload.content[] (ContentItem)

input_audio writer ×0
fieldtypereqobs
typeenum: input_audio(0)✓
audio_urlstring✓
input_image writer ×0
fieldtypereqobs
typeenum: input_image(0)✓
image_urlstring✓
detailstring
input_text writer ×0
fieldtypereqobs
typeenum: input_text(0)✓
textstring✓
output_text writer ×0
fieldtypereqobs
typeenum: output_text(0)✓
textstring✓

facet: thread-items by type — event_msg[item_completed].item (by item type)

AgentMessage fixture ×4
fieldtypereqobs
content array<object> ✓ 4
array of:
fieldtypereqobs
textstring✓4
typeenum: Text(4)✓4
idstring✓4
typeenum: AgentMessage(4)✓4
CommandExecution fixture ×1
fieldtypereqobs
aggregated_outputstring✓1
commandarray<string>✓1
cwdstring✓1
duration object ✓ 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber✓1
formatted_outputstring✓1
idstring✓1
parsed_cmd array<object> ✓ 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring✓1
sourceenum: unified_exec_startup(1)✓1
statusenum: completed(1)✓1
stderrstring✓1
stdoutstring✓1
typeenum: CommandExecution(1)✓1
UserMessage fixture ×4
fieldtypereqobs
content array<object> ✓ 4
array of:
fieldtypereqobs
textstring✓4
text_elementsarray✓4
typeenum: text(4)✓4
idstring✓4
typeenum: UserMessage(4)✓4

facet: tool-call-arguments by name — response_item[function_call].arguments (parsed, by tool name)

exec_command fixture ×1
fieldtypereqobs
cmdstring✓1

state-db state_*.sqlite*

Versioned WAL-mode session metadata projection. The threads table supplies the complete current id/cwd/version/time/title inventory; optional generated name takes precedence over a bounded first-prompt fallback. Sidecars are watch hints only. Schema dump: sources/<version>.sqlite-schema.sql.

state_5.sqlite:_sqlx_migrations fixture ×150
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
state_5.sqlite:backfill_state fixture ×3
fieldtypereqobs
idnumber✓
last_success_atnumber
last_watermarkstring
statusstring✓
updated_atnumber✓
state_5.sqlite:external_agent_config_imports fixture ×0
fieldtypereqobs
completed_at_msnumber✓
failuresstring✓
import_idstring✓
provider_idstring
successesstring✓
state_5.sqlite:project_idempotency_keys fixture ×0
fieldtypereqobs
created_at_msnumber✓
keystring✓
project_idstring✓
state_5.sqlite:project_roots fixture ×0
fieldtypereqobs
pathstring✓
positionnumber✓
project_idstring✓
state_5.sqlite:projects fixture ×0
fieldtypereqobs
created_at_msnumber✓
idstring✓
metadatastring✓
namestring✓
positionnumber✓
updated_at_msnumber✓
state_5.sqlite:remote_control_enrollments fixture ×0
fieldtypereqobs
account_idstring✓
app_server_client_namestring✓
environment_idstring✓
remote_control_enablednumber
server_idstring✓
server_namestring✓
updated_atnumber✓
websocket_urlstring✓
state_5.sqlite:rollout_migration_skipped_rollouts fixture ×0
fieldtypereqobs
migration_idstring✓
rollout_modified_at_nsnumber✓
rollout_pathstring✓
rollout_size_bytesnumber✓
skip_reasonstring✓
skipped_atnumber✓
state_5.sqlite:rollout_migration_state fixture ×0
fieldtypereqobs
last_checked_thread_created_atnumber
last_checked_thread_idstring
migration_idstring✓
updated_atnumber✓
state_5.sqlite:thread_dynamic_tools fixture ×0
fieldtypereqobs
defer_loadingnumber✓
descriptionstring✓
input_schemastring✓
namestring✓
namespacestring
positionnumber✓
thread_idstring✓
state_5.sqlite:thread_sections fixture ×3
fieldtypereqobs
appearancestring
idstring✓
namestring✓
state_5.sqlite:thread_spawn_edges fixture ×0
fieldtypereqobs
child_thread_idstring✓
parent_thread_idstring✓
statusstring✓
state_5.sqlite:threads fixture ×3
fieldtypereqobs
agent_nicknamestring
agent_pathstring
agent_rolestring
approval_modestring✓
archivednumber✓
archived_atnumber
cli_versionstring✓
created_atnumber✓
created_at_msnumber
cwdstring✓
first_user_messagestring✓
git_branchstring
git_origin_urlstring
git_shastring
has_user_eventnumber✓
history_modestring✓
idstring✓
is_pinnednumber✓
memory_modestring✓
modelstring
model_providerstring✓
namestring
previewstring✓
project_idstring
reasoning_effortstring
recency_atnumber✓
recency_at_msnumber✓
rollout_pathstring✓
sandbox_policy object ✓
fieldtypereqobs
file_system object 2
fieldtypereqobs
entries array<object> ✓ 2
array of:
fieldtypereqobs
accessstring✓2
path object ✓ 2
fieldtypereqobs
typeenum: special(2)✓2
value object ✓ 2
fieldtypereqobs
kindenum: root(2)✓2
typeenum: restricted(2)✓2
networkstring2
typeenum: managed(2) | disabled(1)✓3
section_entered_at_msnumber
section_positionnumber
sourcestring✓
thread_section_idstring
thread_sourcestring
titlestring✓
tokens_usednumber✓
updated_atnumber✓
updated_at_msnumber

state-dbs *.sqlite*

Inventory for the remaining version-suffixed WAL-mode stores: goals_1, logs_2, memories_1, queue_1, thread_history_1 (+ -wal/-shm). thread_history_1 is Codex's own rollout projection with byte-offset cursors; rollouts remain timeline truth. The earlier state-db stream claims state_*.sqlite first.

goals_1.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
goals_1.sqlite:thread_goal_continuation_deferrals fixture ×0
fieldtypereqobs
thread_idstring✓
goals_1.sqlite:thread_goals fixture ×0
fieldtypereqobs
created_at_msnumber✓
goal_idstring✓
objectivestring✓
statusstring✓
thread_idstring✓
time_used_secondsnumber✓
token_budgetnumber
tokens_usednumber✓
updated_at_msnumber✓
logs_2.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
logs_2.sqlite:logs fixture ×0
fieldtypereqobs
estimated_bytesnumber✓
feedback_log_bodystring
filestring
idnumber✓
levelstring✓
linenumber
module_pathstring
process_uuidstring
targetstring✓
thread_idstring
tsnumber✓
ts_nanosnumber✓
memories_1.sqlite:_sqlx_migrations fixture ×3
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
memories_1.sqlite:jobs fixture ×0
fieldtypereqobs
finished_atnumber
input_watermarknumber
job_keystring✓
kindstring✓
last_errorstring
last_success_watermarknumber
lease_untilnumber
ownership_tokenstring
retry_atnumber
retry_remainingnumber✓
started_atnumber
statusstring✓
worker_idstring
memories_1.sqlite:stage1_outputs fixture ×0
fieldtypereqobs
generated_atnumber✓
last_usagenumber
raw_memorystring✓
rollout_slugstring
rollout_summarystring✓
selected_for_phase2number✓
selected_for_phase2_source_updated_atnumber
source_updated_atnumber✓
thread_idstring✓
usage_countnumber
queue_1.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
queue_1.sqlite:queued_items fixture ×0
fieldtypereqobs
created_at_msnumber✓
idstring✓
payload_jsonstring✓
queue_ordernumber✓
thread_idstring✓
updated_at_msnumber✓
queue_1.sqlite:queued_thread_revisions fixture ×0
fieldtypereqobs
revisionnumber✓
thread_idstring✓
thread_history_1.sqlite:_sqlx_migrations fixture ×12
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
thread_history_1.sqlite:thread_history_projection_state fixture ×3
fieldtypereqobs
next_rollout_byte_offsetnumber✓
next_rollout_ordinalnumber✓
thread_idstring✓
thread_history_1.sqlite:thread_items fixture ×9
fieldtypereqobs
created_at_msnumber✓
item_idstring✓
item_json object ✓
fieldtypereqobs
aggregatedOutputstring1
clientIdnull4
commandstring1
commandActions array<object> 1
array of:
fieldtypereqobs
commandstring✓1
typeenum: unknown(1)✓1
content array<object> 4
array of:
fieldtypereqobs
textstring✓4
text_elementsarray✓4
typeenum: text(4)✓4
cwdstring1
deliverynull4
durationMsnumber1
exitCodenumber1
idstring✓9
memoryCitationnull4
phasenull4
pluginIdnull1
processIdstring1
scriptPathnull1
sourceenum: unifiedExecStartup(1)1
statusenum: completed(1)1
textstring4
typeenum: agentMessage(4) | userMessage(4) | commandExecution(1)✓9
item_typestring✓
rollout_ordinalnumber✓
thread_idstring✓
turn_idstring✓
updated_at_ordinalnumber✓
thread_history_1.sqlite:thread_turns fixture ×4
fieldtypereqobs
completed_atnumber
duration_msnumber
error_jsonstring
final_agent_item_idstring
first_user_item_idstring
rollout_byte_offsetnumber
rollout_end_byte_offsetnumber
rollout_end_ordinalnumber
rollout_ordinalnumber✓
started_atnumber
statusstring✓
thread_idstring✓
turn_idstring✓

farm:headless-stdout <farm>/stdout*.jsonl

Headless stream-json stdout — corroboration surface captured by the farm, not part of the agent data root.

item.completed fixture ×9
fieldtypereqobs
item object ✓ 9
fieldtypereqobs
aggregated_outputstring1
commandstring1
exit_codenumber1
idstring✓9
messagestring4
statusenum: completed(1)1
textstring4
typeenum: agent_message(4) | error(4) | command_execution(1)✓9
typeenum: item.completed(9)✓9
item.started fixture ×1
fieldtypereqobs
item object ✓ 1
fieldtypereqobs
aggregated_outputstring✓1
commandstring✓1
exit_codenull✓1
idstring✓1
statusenum: in_progress(1)✓1
typeenum: command_execution(1)✓1
typeenum: item.started(1)✓1
thread.started fixture ×4
fieldtypereqobs
thread_idstring✓4
typeenum: thread.started(4)✓4
turn.completed fixture ×4
fieldtypereqobs
typeenum: turn.completed(4)✓4
usage object ✓ 4
fieldtypereqobs
cache_write_input_tokensnumber✓4
cached_input_tokensnumber✓4
input_tokensnumber✓4
output_tokensnumber✓4
reasoning_output_tokensnumber✓4
turn.started fixture ×4
fieldtypereqobs
typeenum: turn.started(4)✓4
+.rname{font-weight:700}.variant,.arr{color:var(--mut);font-size:.85rem;margin:.3rem .5rem}code{background:var(--card);padding:.05rem .3rem;border-radius:4px}a{color:var(--chipfg);text-decoration:none}a:hover{text-decoration:underline}

rosetta ground truth

codex 0.149.1 writer

root ~/.codex · generator tools.groundtruth canonical 2

canonical — merged from: sources/0.149.1.writer.json (0.149.1, writer, writer rust-v0.149.1) sha256:4071a3f3e945 · sources/0.150.1.writer.json (0.149.1, writer, writer rust-v0.150.1) sha256:6dec0213bc1c · sources/0.155.1.writer.json (0.155.1, writer, writer main@e7bbc79f) sha256:9a5f252f989e · sources/0.167.2.writer.json (0.167.2, writer, writer rust-v0.167.2@ce21072c90d32c40af7a55b60b2fb35c84a43a6a) sha256:745af4d01714 · sources/0.149.1.fixtures.json (0.149.1, fixture) sha256:f78063b85966

streamglobformatdisciplineevidencetypesrecords
rolloutsessions/*/*/*/rollout-*.jsonljsonlappend_onlywriter1149
rollout-compressedsessions/*/*/*/rollout-*.jsonl.zstjsonlappend_onlywriter10
archived-rolloutarchived_sessions/rollout-*.jsonljsonlappend_onlywriter10
archived-rollout-compressedarchived_sessions/rollout-*.jsonl.zstjsonlappend_onlywriter10
state-dbstate_*.sqlite*sqlitedatabasefixture13159
state-dbs*.sqlite*sqlitedatabasefixture1549
writer-locksthread-writer-locks/**/*textrewrite_in_placefixture00
installation-idinstallation_idtextrewrite_in_placefixture00
sandbox-migration.sandbox_migrationtextrewrite_in_placefixture00
configconfig.tomltomlrewrite_in_placefixture00
prompt-historyhistory.jsonljsonlappend_onlyresearch00
authauth.jsonjsonrewrite_in_placeresearch00
logslog/**/*textappend_onlyresearch00
versionversion.jsonjsonrewrite_in_placeresearch00
farm:headless-stdout<farm>/stdout*.jsonljsonlappend_onlyfixture522

rollout sessions/*/*/*/rollout-*.jsonl

Session rollouts under YYYY/MM/DD: the complete event stream. Top-level types (0.149.1): session_meta, event_msg, response_item, world_state, turn_context. Pinned writer evidence adds custom_tool_call/custom_tool_call_output to the fixture-backed response items. token_count carries CUMULATIVE total_token_usage, not per-message deltas. Roles include 'developer'. Can embed compaction history and grow large.

compacted writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: compacted(0)✓
payload object ✓
fieldtypereqobs
messagestring✓
replacement_history array<object>
array of:
fieldtypereqobs
replacement_history_metadata array<object>
array of:
fieldtypereqobs
mcp_resource_origins object
fieldtypereqobs
window_numbernumber
first_window_idstring
previous_window_idstring
window_idstring | number
event_msg fixture ×23
fieldtypereqobs
ordinalnumber✓23
payload object ✓ 23
fieldtypereqobs
collaboration_mode_kindenum: default(4)4
completed_atnumber4
completed_at_msnumber9
duration_msnumber4
info object 5
fieldtypereqobs
last_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
model_context_windownumber✓5
total_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
item object 9
fieldtypereqobs
aggregated_outputstring1
commandarray<string>1
content array<object> 8
array of:
fieldtypereqobs
textstring✓8
text_elementsarray4
typeenum: Text(4) | text(4)✓8
cwdstring1
duration object 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber1
formatted_outputstring1
idstring✓9
parsed_cmd array<object> 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring1
sourceenum: unified_exec_startup(1)1
statusenum: completed(1)1
stderrstring1
stdoutstring1
typeenum: AgentMessage(4) | UserMessage(4) | CommandExecution(1)✓9
last_agent_messagestring4
model_context_windownumber4
rate_limits object 5
fieldtypereqobs
creditsnull✓5
individual_limitnull✓5
limit_idstring✓5
limit_namenull✓5
plan_typenull✓5
primarynull✓5
rate_limit_reached_typenull✓5
secondarynull✓5
spend_control_reachednull✓5
started_atnumber8
started_at_msnumber9
thread_idstring9
thread_settings object 1
fieldtypereqobs
active_permission_profile object ✓ 1
fieldtypereqobs
idstring✓1
approval_policystring✓1
approvals_reviewerstring✓1
collaboration_mode object ✓ 1
fieldtypereqobs
modeenum: default(1)✓1
settings object ✓ 1
fieldtypereqobs
developer_instructionsnull✓1
modelenum: mock-model(1)✓1
reasoning_effortnull✓1
cwdstring✓1
modelenum: mock-model(1)✓1
model_provider_idstring✓1
permission_profile object ✓ 1
fieldtypereqobs
file_system object ✓ 1
fieldtypereqobs
entries array<object> ✓ 1
array of:
fieldtypereqobs
accessstring✓1
path object ✓ 1
fieldtypereqobs
typeenum: special(1)✓1
value object ✓ 1
fieldtypereqobs
kindenum: root(1)✓1
typeenum: restricted(1)✓1
networkstring✓1
typeenum: managed(1)✓1
personalitystring✓1
time_to_first_token_msnumber4
turn_idstring17
typeenum: item_completed(9) | token_count(5) | task_complete(4) | task_started(4) | thread_settings_applied(1)✓23
timestampstring✓23
typeenum: event_msg(23)✓23
inter_agent_communication writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: inter_agent_communication(0)✓
payload object ✓
fieldtypereqobs
idstring
authorstring✓
recipientstring✓
other_recipientsarray<string>
contentstring✓
encrypted_contentstring
internal_chat_message_metadata_passthrough object
fieldtypereqobs
trigger_turnboolean✓
inter_agent_communication_metadata writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: inter_agent_communication_metadata(0)✓
payload object ✓
fieldtypereqobs
trigger_turnboolean✓
realtime_item writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: realtime_item(0)✓
payload object ✓
fieldtypereqobs
idstring✓
realtime_session_idstring✓
typestring✓
response_item fixture ×16
fieldtypereqobs
ordinalnumber✓16
payload object ✓ 16
fieldtypereqobs
argumentsstring1
call_idstring2
content array<object> 14
array of:
fieldtypereqobs
textstring✓17
typeenum: input_text(13) | output_text(4)✓17
idstring✓16
internal_chat_message_metadata_passthrough object ✓ 16
fieldtypereqobs
create_timenumber11
turn_idstring✓16
namestring1
outputstring1
roleenum: user(7) | assistant(4) | developer(3)14
typeenum: message(14) | function_call(1) | function_call_output(1)✓16
timestampstring✓16
typeenum: response_item(16)✓16
security_risk_score writer ×0
fieldtypereqobs
timestampstring✓
typeenum: security_risk_score(0)✓
payload object ✓
fieldtypereqobs
scores object ✓
fieldtypereqobs
sampled_atstring
session_meta writer ×3
fieldtypereqobs
ordinalnumber
payload object ✓
fieldtypereqobs
attached_threadboolean
base_instructions object 3
fieldtypereqobs
provenance object ✓ 3
fieldtypereqobs
modelenum: mock-model(3)✓3
typeenum: model(3)✓3
textstring✓3
cli_versionenum: 0.149.1(3)3
context_window object 3
fieldtypereqobs
window_idstring✓3
cwdstring3
forked_from_idstring
forked_from_ordinal_exclusivenumber
history_base object
fieldtypereqobs
thread_idstring✓
end_ordinal_exclusivenumber✓
end_byte_offsetnumber✓
history_modestring
idstring✓
model_providerenum: mock(3)3
originatorstring3
parent_thread_idstring
session_idstring3
sourceenum: exec(3)3
subagent_history_start_ordinalnumber
thread_sourceenum: user(3)3
timestampstring3
timestampstring3
typeenum: session_meta(3)✓
token_usage_record writer ×0
fieldtypereqobs
timestampstring✓
ordinalnumber
typeenum: token_usage_record(0)✓
payload object ✓
fieldtypereqobs
thread_idstring✓
turn_idstring✓
session_idstring✓
root_turn_idstring✓
response_idstring✓
usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
turn_token_usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
thread_token_usage object ✓
fieldtypereqobs
input_tokensnumber✓
cached_input_tokensnumber✓
cache_write_input_tokensnumber✓
output_tokensnumber✓
reasoning_output_tokensnumber✓
total_tokensnumber✓
turn_context fixture ×4
fieldtypereqobs
ordinalnumber✓4
payload object ✓ 4
fieldtypereqobs
active_permission_profile object 3
fieldtypereqobs
idstring✓3
approval_policystring✓4
approvals_reviewerstring✓4
collaboration_mode object ✓ 4
fieldtypereqobs
modeenum: default(4)✓4
settings object ✓ 4
fieldtypereqobs
developer_instructionsnull✓4
modelenum: mock-model(4)✓4
reasoning_effortnull✓4
current_datestring✓4
cwdstring✓4
modelenum: mock-model(4)✓4
multi_agent_versionenum: v1(4)✓4
permission_profile object ✓ 4
fieldtypereqobs
file_system object 3
fieldtypereqobs
entries array<object> ✓ 3
array of:
fieldtypereqobs
accessstring✓3
path object ✓ 3
fieldtypereqobs
typeenum: special(3)✓3
value object ✓ 3
fieldtypereqobs
kindenum: root(3)✓3
typeenum: restricted(3)✓3
networkstring3
typeenum: managed(3) | disabled(1)✓4
personalitystring✓4
realtime_activeboolean✓4
sandbox_policy object ✓ 4
fieldtypereqobs
typeenum: read-only(3) | danger-full-access(1)✓4
summarystring✓4
timezonestring✓4
turn_idstring✓4
workspace_rootsarray<string>✓4
timestampstring✓4
typeenum: turn_context(4)✓4
world_state fixture ×3
fieldtypereqobs
ordinalnumber✓3
payload object ✓ 3
fieldtypereqobs
fullboolean✓3
state object ✓ 3
fieldtypereqobs
agents_md object ✓ 3
fieldtypereqobs
apps_instructionsboolean✓3
collaboration_mode object ✓ 3
fieldtypereqobs
instructionsstring✓3
modeenum: default(3)✓3
modelenum: mock-model(3)✓3
environments object ✓ 3
fieldtypereqobs
current_datestring✓3
environments object ✓ 3
fieldtypereqobs
local object ✓ 3
fieldtypereqobs
cwdstring✓3
shellstring✓3
statusenum: available(3)✓3
filesystemstring✓3
timezonestring✓3
environments_instructionsboolean✓3
git_attributionboolean✓3
host_skills object ✓ 3
fieldtypereqobs
bodystring✓3
includeInstructionsboolean✓3
modelenum: mock-model(3)✓3
multi_agent_mode object ✓ 3
fieldtypereqobs
orchestrator_skills object ✓ 3
fieldtypereqobs
enabledboolean✓3
includeInstructionsboolean✓3
permissions object ✓ 3
fieldtypereqobs
approved_command_prefixesarray✓3
instructionsstring✓3
personality object ✓ 3
fieldtypereqobs
modelenum: mock-model(3)✓3
personalitystring✓3
plugins_instructionsboolean✓3
realtime object ✓ 3
fieldtypereqobs
activeboolean✓3
skills object ✓ 3
fieldtypereqobs
includeInstructionsboolean✓3
timestampstring✓3
typeenum: world_state(3)✓3

facet: event-msg by type — event_msg.payload (by payload.type)

agent_message writer ×0
fieldtypereqobs
typeenum: agent_message(0)✓
messagestring✓
phasestring
memory_citation object
fieldtypereqobs
agent_reasoning writer ×0
fieldtypereqobs
typeenum: agent_reasoning(0)✓
textstring✓
agent_reasoning_raw_content writer ×0
fieldtypereqobs
typeenum: agent_reasoning_raw_content(0)✓
context_compacted writer ×0
fieldtypereqobs
typeenum: context_compacted(0)✓
entered_review_mode writer ×0
fieldtypereqobs
typeenum: entered_review_mode(0)✓
exec_command_end writer ×0
fieldtypereqobs
typeenum: exec_command_end(0)✓
exited_review_mode writer ×0
fieldtypereqobs
typeenum: exited_review_mode(0)✓
image_generation_end writer ×0
fieldtypereqobs
typeenum: image_generation_end(0)✓
item_completed fixture ×9
fieldtypereqobs
completed_at_msnumber✓9
item object ✓ 9
fieldtypereqobs
aggregated_outputstring1
commandarray<string>1
content array<object> 8
array of:
fieldtypereqobs
textstring✓8
text_elementsarray4
typeenum: Text(4) | text(4)✓8
cwdstring1
duration object 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber1
formatted_outputstring1
idstring✓9
parsed_cmd array<object> 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring1
sourceenum: unified_exec_startup(1)1
statusenum: completed(1)1
stderrstring1
stdoutstring1
typeenum: AgentMessage(4) | UserMessage(4) | CommandExecution(1)✓9
started_at_msnumber✓9
thread_idstring✓9
turn_idstring✓9
typeenum: item_completed(9)✓9
mcp_tool_call_end writer ×0
fieldtypereqobs
typeenum: mcp_tool_call_end(0)✓
patch_apply_end writer ×0
fieldtypereqobs
typeenum: patch_apply_end(0)✓
sub_agent_activity writer ×0
fieldtypereqobs
typeenum: sub_agent_activity(0)✓
task_complete fixture ×4
fieldtypereqobs
completed_atnumber✓4
duration_msnumber✓4
last_agent_messagestring✓4
started_atnumber✓4
time_to_first_token_msnumber✓4
turn_idstring✓4
typeenum: task_complete(4)✓4
task_started fixture ×4
fieldtypereqobs
collaboration_mode_kindenum: default(4)✓4
model_context_windownumber✓4
started_atnumber✓4
turn_idstring✓4
typeenum: task_started(4)✓4
thread_goal_updated writer ×0
fieldtypereqobs
typeenum: thread_goal_updated(0)✓
thread_rolled_back writer ×0
fieldtypereqobs
typeenum: thread_rolled_back(0)✓
num_turnsnumber✓
thread_settings_applied writer ×1
fieldtypereqobs
thread_idstring
thread_settings object ✓
fieldtypereqobs
active_permission_profile object
fieldtypereqobs
idstring1
approval_policy object ✓
variant 1: string
variant 2:
fieldtypereqobs
approvals_reviewerstring✓
collaboration_mode object 1
fieldtypereqobs
modeenum: default(1)✓1
settings object ✓ 1
fieldtypereqobs
developer_instructionsnull✓1
modelenum: mock-model(1)✓1
reasoning_effortnull✓1
cwdstring✓
modelstring✓
model_provider_idstring✓
permission_profile object ✓
fieldtypereqobs
file_system object 1
fieldtypereqobs
entries array<object> ✓ 1
array of:
fieldtypereqobs
accessstring✓1
path object ✓ 1
fieldtypereqobs
typeenum: special(1)✓1
value object ✓ 1
fieldtypereqobs
kindenum: root(1)✓1
typeenum: restricted(1)✓1
networkstring1
typeenum: managed(1)1
personalitystring1
runtime_workspace_rootsarray<string>
service_tierstring
typeenum: thread_settings_applied(1)✓
token_count fixture ×5
fieldtypereqobs
info object ✓ 5
fieldtypereqobs
last_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
model_context_windownumber✓5
total_token_usage object ✓ 5
fieldtypereqobs
cache_write_input_tokensnumber✓5
cached_input_tokensnumber✓5
input_tokensnumber✓5
output_tokensnumber✓5
reasoning_output_tokensnumber✓5
total_tokensnumber✓5
rate_limits object ✓ 5
fieldtypereqobs
creditsnull✓5
individual_limitnull✓5
limit_idstring✓5
limit_namenull✓5
plan_typenull✓5
primarynull✓5
rate_limit_reached_typenull✓5
secondarynull✓5
spend_control_reachednull✓5
typeenum: token_count(5)✓5
turn_aborted writer ×0
fieldtypereqobs
typeenum: turn_aborted(0)✓
reasonstring✓
user_message writer ×0
fieldtypereqobs
typeenum: user_message(0)✓
messagestring✓
imagesarray<string>✓
local_imagesarray<string>✓
text_elements array<object> ✓
array of:
fieldtypereqobs
web_search_end writer ×0
fieldtypereqobs
typeenum: web_search_end(0)✓

facet: response-items by type — response_item.payload (by payload.type)

agent_message writer ×0
fieldtypereqobs
typeenum: agent_message(0)✓
idstring
authorstring✓
recipientstring✓
content array<object> ✓
array of:
fieldtypereqobs
internal_chat_message_metadata_passthrough object
fieldtypereqobs
compaction writer ×0
fieldtypereqobs
typeenum: compaction(0)✓
idstring
encrypted_contentstring✓
context_compaction writer ×0
fieldtypereqobs
typeenum: context_compaction(0)✓
idstring
encrypted_contentstring
custom_tool_call writer ×0
fieldtypereqobs
idstring
statusstring
call_idstring✓
namestring✓
namespacestring
inputstring✓
internal_chat_message_metadata_passthrough object
fieldtypereqobs
typeenum: custom_tool_call(0)✓
custom_tool_call_output writer ×0
fieldtypereqobs
idstring
call_idstring✓
namestring
output object ✓
variant 1: string
variant 2:
array of:
fieldtypereqobs
internal_chat_message_metadata_passthrough object
fieldtypereqobs
typeenum: custom_tool_call_output(0)✓
function_call writer ×1
fieldtypereqobs
argumentsstring✓
call_idstring✓
encrypted_function_argsarray<string>
idstring
internal_chat_message_metadata_passthrough object 1
fieldtypereqobs
turn_idstring✓1
namestring✓
namespacestring
typeenum: function_call(1)✓
function_call_output fixture ×1
fieldtypereqobs
call_idstring✓1
idstring✓1
internal_chat_message_metadata_passthrough object ✓ 1
fieldtypereqobs
create_timenumber✓1
turn_idstring✓1
outputstring✓1
typeenum: function_call_output(1)✓1
image_generation_call writer ×0
fieldtypereqobs
typeenum: image_generation_call(0)✓
idstring
statusstring✓
revised_promptstring
resultstring✓
local_shell_call writer ×0
fieldtypereqobs
typeenum: local_shell_call(0)✓
idstring
call_idstring
statusstring✓
action object ✓
fieldtypereqobs
message writer ×14
fieldtypereqobs
content array<object> ✓
array of:
fieldtypereqobs
textstring17
typeenum: input_text(13) | output_text(4)17
idstring
internal_chat_message_metadata_passthrough object 14
fieldtypereqobs
create_timenumber10
turn_idstring✓14
phasestring
rolestring✓
typeenum: message(14)✓
reasoning writer ×0
fieldtypereqobs
typeenum: reasoning(0)✓
idstring
summary array<object> ✓
array of:
fieldtypereqobs
typeenum: summary_text(0)✓
textstring✓
content array<object>
array of:
fieldtypereqobs
encrypted_contentstring
internal_chat_message_metadata_passthrough object
fieldtypereqobs
tool_search_call writer ×0
fieldtypereqobs
typeenum: tool_search_call(0)✓
idstring
call_idstring
statusstring
executionstring✓
arguments?✓
tool_search_output writer ×0
fieldtypereqobs
typeenum: tool_search_output(0)✓
idstring
call_idstring
statusstring✓
executionstring✓
tools array<object> ✓
array of:
fieldtypereqobs
web_search_call writer ×0
fieldtypereqobs
typeenum: web_search_call(0)✓
idstring
statusstring
action object
fieldtypereqobs

facet: content-blocks by type — response_item.payload.content[] (ContentItem)

input_audio writer ×0
fieldtypereqobs
typeenum: input_audio(0)✓
audio_urlstring✓
input_image writer ×0
fieldtypereqobs
typeenum: input_image(0)✓
image_urlstring✓
detailstring
input_text writer ×0
fieldtypereqobs
typeenum: input_text(0)✓
textstring✓
output_text writer ×0
fieldtypereqobs
typeenum: output_text(0)✓
textstring✓

facet: thread-items by type — event_msg[item_completed].item (by item type)

AgentMessage fixture ×4
fieldtypereqobs
content array<object> ✓ 4
array of:
fieldtypereqobs
textstring✓4
typeenum: Text(4)✓4
idstring✓4
typeenum: AgentMessage(4)✓4
CommandExecution fixture ×1
fieldtypereqobs
aggregated_outputstring✓1
commandarray<string>✓1
cwdstring✓1
duration object ✓ 1
fieldtypereqobs
nanosnumber✓1
secsnumber✓1
exit_codenumber✓1
formatted_outputstring✓1
idstring✓1
parsed_cmd array<object> ✓ 1
array of:
fieldtypereqobs
cmdstring✓1
typeenum: unknown(1)✓1
process_idstring✓1
sourceenum: unified_exec_startup(1)✓1
statusenum: completed(1)✓1
stderrstring✓1
stdoutstring✓1
typeenum: CommandExecution(1)✓1
UserMessage fixture ×4
fieldtypereqobs
content array<object> ✓ 4
array of:
fieldtypereqobs
textstring✓4
text_elementsarray✓4
typeenum: text(4)✓4
idstring✓4
typeenum: UserMessage(4)✓4

facet: tool-call-arguments by name — response_item[function_call].arguments (parsed, by tool name)

exec_command fixture ×1
fieldtypereqobs
cmdstring✓1

rollout-compressed sessions/*/*/*/rollout-*.jsonl.zst

Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.

session_meta writer ×0
fieldtypereqobs
typeenum: session_meta(0)✓
ordinalnumber
payload object ✓
fieldtypereqobs
idstring✓
history_modestring
history_base object
fieldtypereqobs
thread_idstring✓
end_ordinal_exclusivenumber✓
end_byte_offsetnumber✓
forked_from_idstring
forked_from_ordinal_exclusivenumber
subagent_history_start_ordinalnumber
parent_thread_idstring
attached_threadboolean

archived-rollout archived_sessions/rollout-*.jsonl

Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.

session_meta writer ×0
fieldtypereqobs
typeenum: session_meta(0)✓
ordinalnumber
payload object ✓
fieldtypereqobs
idstring✓
history_modestring
history_base object
fieldtypereqobs
thread_idstring✓
end_ordinal_exclusivenumber✓
end_byte_offsetnumber✓
forked_from_idstring
forked_from_ordinal_exclusivenumber
subagent_history_start_ordinalnumber
parent_thread_idstring
attached_threadboolean

archived-rollout-compressed archived_sessions/rollout-*.jsonl.zst

Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.

session_meta writer ×0
fieldtypereqobs
typeenum: session_meta(0)✓
ordinalnumber
payload object ✓
fieldtypereqobs
idstring✓
history_modestring
history_base object
fieldtypereqobs
thread_idstring✓
end_ordinal_exclusivenumber✓
end_byte_offsetnumber✓
forked_from_idstring
forked_from_ordinal_exclusivenumber
subagent_history_start_ordinalnumber
parent_thread_idstring
attached_threadboolean

state-db state_*.sqlite*

Versioned WAL-mode session metadata projection. The threads table supplies the complete current id/cwd/version/time/title inventory; optional generated name takes precedence over a bounded first-prompt fallback. Sidecars are watch hints only. Schema dump: sources/<version>.sqlite-schema.sql.

state_5.sqlite:_sqlx_migrations fixture ×150
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
state_5.sqlite:backfill_state fixture ×3
fieldtypereqobs
idnumber✓
last_success_atnumber
last_watermarkstring
statusstring✓
updated_atnumber✓
state_5.sqlite:external_agent_config_imports fixture ×0
fieldtypereqobs
completed_at_msnumber✓
failuresstring✓
import_idstring✓
provider_idstring
successesstring✓
state_5.sqlite:project_idempotency_keys fixture ×0
fieldtypereqobs
created_at_msnumber✓
keystring✓
project_idstring✓
state_5.sqlite:project_roots fixture ×0
fieldtypereqobs
pathstring✓
positionnumber✓
project_idstring✓
state_5.sqlite:projects fixture ×0
fieldtypereqobs
created_at_msnumber✓
idstring✓
metadatastring✓
namestring✓
positionnumber✓
updated_at_msnumber✓
state_5.sqlite:remote_control_enrollments fixture ×0
fieldtypereqobs
account_idstring✓
app_server_client_namestring✓
environment_idstring✓
remote_control_enablednumber
server_idstring✓
server_namestring✓
updated_atnumber✓
websocket_urlstring✓
state_5.sqlite:rollout_migration_skipped_rollouts fixture ×0
fieldtypereqobs
migration_idstring✓
rollout_modified_at_nsnumber✓
rollout_pathstring✓
rollout_size_bytesnumber✓
skip_reasonstring✓
skipped_atnumber✓
state_5.sqlite:rollout_migration_state fixture ×0
fieldtypereqobs
last_checked_thread_created_atnumber
last_checked_thread_idstring
migration_idstring✓
updated_atnumber✓
state_5.sqlite:thread_dynamic_tools fixture ×0
fieldtypereqobs
defer_loadingnumber✓
descriptionstring✓
input_schemastring✓
namestring✓
namespacestring
positionnumber✓
thread_idstring✓
state_5.sqlite:thread_sections fixture ×3
fieldtypereqobs
appearancestring
idstring✓
namestring✓
state_5.sqlite:thread_spawn_edges fixture ×0
fieldtypereqobs
child_thread_idstring✓
parent_thread_idstring✓
statusstring✓
state_5.sqlite:threads fixture ×3
fieldtypereqobs
agent_nicknamestring
agent_pathstring
agent_rolestring
approval_modestring✓
archivednumber✓
archived_atnumber
cli_versionstring✓
created_atnumber✓
created_at_msnumber
cwdstring✓
first_user_messagestring✓
git_branchstring
git_origin_urlstring
git_shastring
has_user_eventnumber✓
history_modestring✓
idstring✓
is_pinnednumber✓
memory_modestring✓
modelstring
model_providerstring✓
namestring
previewstring✓
project_idstring
reasoning_effortstring
recency_atnumber✓
recency_at_msnumber✓
rollout_pathstring✓
sandbox_policy object ✓
fieldtypereqobs
file_system object 2
fieldtypereqobs
entries array<object> ✓ 2
array of:
fieldtypereqobs
accessstring✓2
path object ✓ 2
fieldtypereqobs
typeenum: special(2)✓2
value object ✓ 2
fieldtypereqobs
kindenum: root(2)✓2
typeenum: restricted(2)✓2
networkstring2
typeenum: managed(2) | disabled(1)✓3
section_entered_at_msnumber
section_positionnumber
sourcestring✓
thread_section_idstring
thread_sourcestring
titlestring✓
tokens_usednumber✓
updated_atnumber✓
updated_at_msnumber

state-dbs *.sqlite*

Inventory for the remaining version-suffixed WAL-mode stores: goals_1, logs_2, memories_1, queue_1, thread_history_1 (+ -wal/-shm). thread_history_1 is Codex's own rollout projection with byte-offset cursors; rollouts remain timeline truth. The earlier state-db stream claims state_*.sqlite first.

goals_1.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
goals_1.sqlite:thread_goal_continuation_deferrals fixture ×0
fieldtypereqobs
thread_idstring✓
goals_1.sqlite:thread_goals fixture ×0
fieldtypereqobs
created_at_msnumber✓
goal_idstring✓
objectivestring✓
statusstring✓
thread_idstring✓
time_used_secondsnumber✓
token_budgetnumber
tokens_usednumber✓
updated_at_msnumber✓
logs_2.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
logs_2.sqlite:logs fixture ×0
fieldtypereqobs
estimated_bytesnumber✓
feedback_log_bodystring
filestring
idnumber✓
levelstring✓
linenumber
module_pathstring
process_uuidstring
targetstring✓
thread_idstring
tsnumber✓
ts_nanosnumber✓
memories_1.sqlite:_sqlx_migrations fixture ×3
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
memories_1.sqlite:jobs fixture ×0
fieldtypereqobs
finished_atnumber
input_watermarknumber
job_keystring✓
kindstring✓
last_errorstring
last_success_watermarknumber
lease_untilnumber
ownership_tokenstring
retry_atnumber
retry_remainingnumber✓
started_atnumber
statusstring✓
worker_idstring
memories_1.sqlite:stage1_outputs fixture ×0
fieldtypereqobs
generated_atnumber✓
last_usagenumber
raw_memorystring✓
rollout_slugstring
rollout_summarystring✓
selected_for_phase2number✓
selected_for_phase2_source_updated_atnumber
source_updated_atnumber✓
thread_idstring✓
usage_countnumber
queue_1.sqlite:_sqlx_migrations fixture ×6
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
queue_1.sqlite:queued_items fixture ×0
fieldtypereqobs
created_at_msnumber✓
idstring✓
payload_jsonstring✓
queue_ordernumber✓
thread_idstring✓
updated_at_msnumber✓
queue_1.sqlite:queued_thread_revisions fixture ×0
fieldtypereqobs
revisionnumber✓
thread_idstring✓
thread_history_1.sqlite:_sqlx_migrations fixture ×12
fieldtypereqobs
checksum?✓
descriptionstring✓
execution_timenumber✓
installed_on?✓
successboolean✓
versionnumber✓
thread_history_1.sqlite:thread_history_projection_state fixture ×3
fieldtypereqobs
next_rollout_byte_offsetnumber✓
next_rollout_ordinalnumber✓
thread_idstring✓
thread_history_1.sqlite:thread_items fixture ×9
fieldtypereqobs
created_at_msnumber✓
item_idstring✓
item_json object ✓
fieldtypereqobs
aggregatedOutputstring1
clientIdnull4
commandstring1
commandActions array<object> 1
array of:
fieldtypereqobs
commandstring✓1
typeenum: unknown(1)✓1
content array<object> 4
array of:
fieldtypereqobs
textstring✓4
text_elementsarray✓4
typeenum: text(4)✓4
cwdstring1
deliverynull4
durationMsnumber1
exitCodenumber1
idstring✓9
memoryCitationnull4
phasenull4
pluginIdnull1
processIdstring1
scriptPathnull1
sourceenum: unifiedExecStartup(1)1
statusenum: completed(1)1
textstring4
typeenum: agentMessage(4) | userMessage(4) | commandExecution(1)✓9
item_typestring✓
rollout_ordinalnumber✓
thread_idstring✓
turn_idstring✓
updated_at_ordinalnumber✓
thread_history_1.sqlite:thread_turns fixture ×4
fieldtypereqobs
completed_atnumber
duration_msnumber
error_jsonstring
final_agent_item_idstring
first_user_item_idstring
rollout_byte_offsetnumber
rollout_end_byte_offsetnumber
rollout_end_ordinalnumber
rollout_ordinalnumber✓
started_atnumber
statusstring✓
thread_idstring✓
turn_idstring✓

farm:headless-stdout <farm>/stdout*.jsonl

Headless stream-json stdout — corroboration surface captured by the farm, not part of the agent data root.

item.completed fixture ×9
fieldtypereqobs
item object ✓ 9
fieldtypereqobs
aggregated_outputstring1
commandstring1
exit_codenumber1
idstring✓9
messagestring4
statusenum: completed(1)1
textstring4
typeenum: agent_message(4) | error(4) | command_execution(1)✓9
typeenum: item.completed(9)✓9
item.started fixture ×1
fieldtypereqobs
item object ✓ 1
fieldtypereqobs
aggregated_outputstring✓1
commandstring✓1
exit_codenull✓1
idstring✓1
statusenum: in_progress(1)✓1
typeenum: command_execution(1)✓1
typeenum: item.started(1)✓1
thread.started fixture ×4
fieldtypereqobs
thread_idstring✓4
typeenum: thread.started(4)✓4
turn.completed fixture ×4
fieldtypereqobs
typeenum: turn.completed(4)✓4
usage object ✓ 4
fieldtypereqobs
cache_write_input_tokensnumber✓4
cached_input_tokensnumber✓4
input_tokensnumber✓4
output_tokensnumber✓4
reasoning_output_tokensnumber✓4
turn.started fixture ×4
fieldtypereqobs
typeenum: turn.started(4)✓4
diff --git a/groundtruth/codex/groundtruth.json b/groundtruth/codex/groundtruth.json index 15f435b3..ca4d1ea5 100644 --- a/groundtruth/codex/groundtruth.json +++ b/groundtruth/codex/groundtruth.json @@ -30,12 +30,20 @@ "sha256": "9a5f252f989eeaef0b0ede7ef29e2e15994ee1b56079deac5454176d031437ca", "writer_version": "main@e7bbc79f" }, + { + "generator": "writer-source openai/codex pinned release; storage and lineage audit 2026-11-12", + "agent_version": "0.167.2", + "evidence": "writer", + "path": "sources/0.167.2.writer.json", + "sha256": "745af4d0171448e74f240855caf8f0f3fcdb9e24a9ad8e8dc8715b644aec3fb4", + "writer_version": "rust-v0.167.2@ce21072c90d32c40af7a55b60b2fb35c84a43a6a" + }, { "generator": "tools.groundtruth farm 1", "agent_version": "0.149.1", "evidence": "fixture", "path": "sources/0.149.1.fixtures.json", - "sha256": "1a370297cf6553f137943591fe04542a40f99bd8f6ecc1f47b902593b3e9b2bd" + "sha256": "f78063b85966c03408614b157ec26a2a4a86284c86d7c98d521096f877d31343" } ], "streams": [ @@ -1295,13 +1303,18 @@ "shape": { "kind": "number" }, - "required": true, - "observed": 3 + "required": false }, "payload": { "shape": { "kind": "object", "fields": { + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + }, "base_instructions": { "shape": { "kind": "object", @@ -1344,7 +1357,7 @@ } } }, - "required": true, + "required": false, "observed": 3 }, "cli_version": { @@ -1354,7 +1367,7 @@ "0.149.1": 3 } }, - "required": true, + "required": false, "observed": 3 }, "context_window": { @@ -1370,32 +1383,65 @@ } } }, - "required": true, + "required": false, "observed": 3 }, "cwd": { "shape": { "kind": "string" }, - "required": true, + "required": false, "observed": 3 }, - "history_mode": { + "forked_from_id": { "shape": { - "kind": "string", - "values": { - "paginated": 3 + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } } + } + }, + "history_mode": { + "shape": { + "kind": "string" }, - "required": true, - "observed": 3 + "required": false }, "id": { "shape": { "kind": "string" }, - "required": true, - "observed": 3 + "required": true }, "model_provider": { "shape": { @@ -1404,21 +1450,27 @@ "mock": 3 } }, - "required": true, + "required": false, "observed": 3 }, "originator": { "shape": { "kind": "string" }, - "required": true, + "required": false, "observed": 3 }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, "session_id": { "shape": { "kind": "string" }, - "required": true, + "required": false, "observed": 3 }, "source": { @@ -1428,9 +1480,15 @@ "exec": 3 } }, - "required": true, + "required": false, "observed": 3 }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, "thread_source": { "shape": { "kind": "string", @@ -1438,26 +1496,25 @@ "user": 3 } }, - "required": true, + "required": false, "observed": 3 }, "timestamp": { "shape": { "kind": "string" }, - "required": true, + "required": false, "observed": 3 } } }, - "required": true, - "observed": 3 + "required": true }, "timestamp": { "shape": { "kind": "string" }, - "required": true, + "required": false, "observed": 3 }, "type": { @@ -1467,12 +1524,11 @@ "session_meta": 3 } }, - "required": true, - "observed": 3 + "required": true } } }, - "evidence": "fixture" + "evidence": "writer" }, "token_usage_record": { "observed": 0, @@ -4716,6 +4772,339 @@ } ] }, + { + "id": "rollout-compressed", + "glob": "sessions/*/*/*/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + }, + "evidence": "writer" + } + } + }, + { + "id": "archived-rollout", + "glob": "archived_sessions/rollout-*.jsonl", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + }, + "evidence": "writer" + } + } + }, + { + "id": "archived-rollout-compressed", + "glob": "archived_sessions/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + }, + "evidence": "writer" + } + } + }, { "id": "state-db", "glob": "state_*.sqlite*", diff --git a/groundtruth/codex/layout.toml b/groundtruth/codex/layout.toml index 7307f078..7ea31238 100644 --- a/groundtruth/codex/layout.toml +++ b/groundtruth/codex/layout.toml @@ -10,6 +10,30 @@ discipline = "append_only" evidence = "fixture" notes = "Session rollouts under YYYY/MM/DD: the complete event stream. Top-level types (0.149.1): session_meta, event_msg, response_item, world_state, turn_context. Pinned writer evidence adds custom_tool_call/custom_tool_call_output to the fixture-backed response items. token_count carries CUMULATIVE total_token_usage, not per-message deltas. Roles include 'developer'. Can embed compaction history and grow large." +[[streams]] +id = "rollout-compressed" +glob = "sessions/*/*/*/rollout-*.jsonl.zst" +file_format = "jsonl" +discipline = "append_only" +evidence = "writer" +notes = "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + +[[streams]] +id = "archived-rollout" +glob = "archived_sessions/rollout-*.jsonl" +file_format = "jsonl" +discipline = "append_only" +evidence = "writer" +notes = "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + +[[streams]] +id = "archived-rollout-compressed" +glob = "archived_sessions/rollout-*.jsonl.zst" +file_format = "jsonl" +discipline = "append_only" +evidence = "writer" +notes = "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + [[streams]] id = "state-db" glob = "state_*.sqlite*" diff --git a/groundtruth/codex/sources/0.149.1.fixtures.json b/groundtruth/codex/sources/0.149.1.fixtures.json index 2f7554a2..18809dcd 100644 --- a/groundtruth/codex/sources/0.149.1.fixtures.json +++ b/groundtruth/codex/sources/0.149.1.fixtures.json @@ -3104,6 +3104,30 @@ } ] }, + { + "id": "rollout-compressed", + "glob": "sessions/*/*/*/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + }, + { + "id": "archived-rollout", + "glob": "archived_sessions/rollout-*.jsonl", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + }, + { + "id": "archived-rollout-compressed", + "glob": "archived_sessions/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Physical representation of rollout. rust-v0.167.2: plain wins when both exist; zstd offsets are decoded bytes; archive relocation preserves logical identity. Only exact jsonl/jsonl.zst suffixes are transcripts. Temporary migration, compression, and materialization files are excluded." + }, { "id": "state-db", "glob": "state_*.sqlite*", diff --git a/groundtruth/codex/sources/0.167.2.writer.json b/groundtruth/codex/sources/0.167.2.writer.json new file mode 100644 index 00000000..b993e7bb --- /dev/null +++ b/groundtruth/codex/sources/0.167.2.writer.json @@ -0,0 +1,451 @@ +{ + "format": "rosetta-groundtruth/1", + "agent": "codex", + "agent_version": "0.167.2", + "writer_version": "rust-v0.167.2@ce21072c90d32c40af7a55b60b2fb35c84a43a6a", + "evidence": "writer", + "generator": "writer-source openai/codex pinned release; storage and lineage audit 2026-11-12", + "data_root": "~/.codex", + "streams": [ + { + "id": "rollout", + "glob": "sessions/*/*/*/rollout-*.jsonl", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + } + } + } + }, + { + "id": "rollout-compressed", + "glob": "sessions/*/*/*/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + } + } + } + }, + { + "id": "archived-rollout", + "glob": "archived_sessions/rollout-*.jsonl", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + } + } + } + }, + { + "id": "archived-rollout-compressed", + "glob": "archived_sessions/rollout-*.jsonl.zst", + "file_format": "jsonl", + "discipline": "append_only", + "evidence": "writer", + "notes": "Pinned rust-v0.167.2. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/compression.rs: cold mtime >=7 days; run throttle 6 hours; 2 workers; zstd level 3; verified no-clobber publication, original removed last. Resume materializes to a temporary file and publishes plain before removing compressed; plain is preferred. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/rollout/src/seekable_reader.rs: all offsets are decoded JSONL; old frames can omit size and streams can have concatenated frames. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_rollout_resolver.rs: state threads.rollout_path selects the current rollout, even when only .zst exists; paginated missing selection never falls back to older history. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_lineage.rs: history_base.thread_id is an immutable rollout ID, not necessarily the stable session_meta.id; prefix ends are exclusive, recursive, bounded by decoded byte and ordinal. Revert writes a new threadID_rolloutID filename and changes the DB selection. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/thread_history_materialization.rs: thread_history SQLite is a rebuildable, potentially incomplete projection, not a raw transcript source. https://github.com/openai/codex/blob/ce21072c90d32c40af7a55b60b2fb35c84a43a6a/codex-rs/thread-store/src/local/rollout_migration.rs: legacy to paginated can rewrite content under unchanged mtime; encoding equivalence must verify decoded content.", + "discriminator": "type", + "record_types": { + "session_meta": { + "observed": 0, + "shape": { + "kind": "object", + "fields": { + "type": { + "shape": { + "kind": "string", + "values": { + "session_meta": 0 + } + }, + "required": true + }, + "ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "payload": { + "shape": { + "kind": "object", + "fields": { + "id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "history_mode": { + "shape": { + "kind": "string" + }, + "required": false + }, + "history_base": { + "required": false, + "shape": { + "kind": "object", + "fields": { + "thread_id": { + "shape": { + "kind": "string" + }, + "required": true + }, + "end_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": true + }, + "end_byte_offset": { + "shape": { + "kind": "number" + }, + "required": true + } + } + } + }, + "forked_from_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "forked_from_ordinal_exclusive": { + "shape": { + "kind": "number" + }, + "required": false + }, + "subagent_history_start_ordinal": { + "shape": { + "kind": "number" + }, + "required": false + }, + "parent_thread_id": { + "shape": { + "kind": "string" + }, + "required": false + }, + "attached_thread": { + "required": false, + "shape": { + "kind": "boolean" + } + } + } + }, + "required": true + } + } + } + } + } + } + ] +} diff --git a/groundtruth/codex/watch.toml b/groundtruth/codex/watch.toml index 624e67ff..58ea5c38 100644 --- a/groundtruth/codex/watch.toml +++ b/groundtruth/codex/watch.toml @@ -16,13 +16,13 @@ tier = "A" [release] channel = "github-releases" # openai/codex/releases.atom; npm @openai/codex secondary id = "openai/codex" -surveyed = "0.150.1" # 2026-08-28; track `latest` only, ignore alphas +surveyed = "0.167.2" # storage/lineage writer audit, 2026-11-12 pinned = "0.149.1" # fixtures + groundtruth doc committed (2026-08-25) changelog = "https://github.com/openai/codex/releases" [source] repo = "https://github.com/openai/codex" -persistence_paths = ["codex-rs/history/src/", "codex-rs/rollout/src/"] +persistence_paths = ["codex-rs/history/src/", "codex-rs/rollout/src/", "codex-rs/thread-store/src/local/", "codex-rs/protocol/src/protocol.rs"] [fixture] headless = "codex exec --json" From f9d88fb633a750d7aeedcc5540b7a997fa0b2708 Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 15:54:34 -0800 Subject: [PATCH 2/6] feat(capture): read stable decoded transcript inputs across representations --- crates/capture/src/input.rs | 473 ++++++++++++++++++ crates/capture/src/lib.rs | 9 + crates/capture/src/tailer.rs | 212 ++++---- crates/capture/src/tailer/checkpoint_bytes.rs | 73 +++ crates/capture/src/tailer/verification.rs | 39 ++ crates/capture/tests/decoded_input.rs | 176 +++++++ 6 files changed, 876 insertions(+), 106 deletions(-) create mode 100644 crates/capture/src/input.rs create mode 100644 crates/capture/src/tailer/checkpoint_bytes.rs create mode 100644 crates/capture/src/tailer/verification.rs create mode 100644 crates/capture/tests/decoded_input.rs diff --git a/crates/capture/src/input.rs b/crates/capture/src/input.rs new file mode 100644 index 00000000..30643511 --- /dev/null +++ b/crates/capture/src/input.rs @@ -0,0 +1,473 @@ +//! Read-only physical representations of a logical decoded transcript. +//! +//! A compressed input is verified to EOF into an anonymous seekable spool +//! once per pass, never once per tail batch. Native files are never restored, +//! renamed or locked. All ranges, limits and cursors refer to decoded bytes. + +use std::fs::File; +use std::io::{Read, Seek, Write}; +use std::path::{Component, Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use serde::{Deserialize, Serialize}; + +use crate::tailer::{FileIdentity, file_identity}; +use crate::{CaptureError, io_error}; + +const BUFFER_BYTES: usize = 128 * 1024; +/// A bad frame cannot consume unbounded scratch space. This is a decoded +/// per-source limit, independent of the normal per-record framing limit. +pub const MAX_DECODED_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum InputEncoding { + Plain, + Zstd, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InputCandidate { + pub path: String, + pub encoding: InputEncoding, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InputSegment { + /// Preference order; a present but invalid preferred file never falls + /// back to an older representation. + pub candidates: Vec, + pub start: u64, + pub end: Option, + /// Bounded header evidence protects a planned lineage from migration + /// between metadata admission and capture. + pub header_hash: Option<[u8; 32]>, + /// Optional native record-index boundary, interpreted by the adapter. + pub end_record_index: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InputSource { + /// Exact bounded native header, when a selected view must configure its + /// normalizer before replaying ancestors. Validated against the selected + /// segment's header before this spool can be published. + pub prefix: Vec, + pub segments: Vec, +} + +impl InputSource { + pub fn single(candidates: Vec) -> Self { + Self { + prefix: Vec::new(), + segments: vec![InputSegment { + candidates, + start: 0, + end: None, + header_hash: None, + end_record_index: None, + }], + } + } + + pub fn headers_match(&self, root: &Path) -> Result { + for segment in &self.segments { + if let Some(expected) = segment.header_hash { + let source = Self::single(segment.candidates.clone()); + if source + .head(root, 32 * 1024 * 1024)? + .is_none_or(|head| *blake3::hash(&head.bytes).as_bytes() != expected) + { + return Ok(false); + } + } + } + Ok(true) + } + + pub fn claims(&self, hint: &str) -> bool { + self.segments.iter().any(|segment| { + segment.candidates.iter().any(|candidate| { + candidate.path == hint + || candidate + .path + .strip_prefix(hint) + .is_some_and(|s| s.starts_with('/')) + }) + }) + } + + /// Content-free gate. Includes inode, size, nanosecond mtime and ctime, + /// plus the range plan, so relocation and timestamp-preserving atomic + /// replacement nominate verification without decoding quiet sources. + pub fn stamp(&self, root: &Path) -> Result, CaptureError> { + if self.segments.is_empty() { + return Ok(None); + } + let mut hash = blake3::Hasher::new(); + hash.update(&serde_json::to_vec(&self.segments).expect("input plan serializes")); + hash.update(&self.prefix); + for segment in &self.segments { + let Some((candidate, path, file)) = open_candidate(root, &segment.candidates)? else { + return Ok(None); + }; + hash_metadata(&mut hash, candidate, &path, &file)?; + } + Ok(Some(hash.finalize().to_hex().to_string())) + } + + /// Read only the first complete record, with a hard decoded bound. No + /// content-size header is assumed: old and concatenated frames work too. + pub fn head(&self, root: &Path, limit: usize) -> Result, CaptureError> { + let Some(segment) = self.segments.first() else { + return Ok(None); + }; + let Some((candidate, path, file)) = open_candidate(root, &segment.candidates)? else { + return Ok(None); + }; + let metadata = file + .metadata() + .map_err(|e| io_error("reading input metadata", &path, e))?; + let len = (candidate.encoding == InputEncoding::Plain).then_some(metadata.len()); + let modified_ms = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |t| t.as_millis() as i64); + let reader = decoder(file, candidate.encoding, &path)?; + let mut reader = std::io::BufReader::new(reader.take(limit as u64 + 1)); + let mut bytes = Vec::new(); + std::io::BufRead::read_until(&mut reader, b'\n', &mut bytes).map_err(|e| { + decode_error("reading decoded input header", &path, candidate.encoding, e) + })?; + if bytes.len() > limit { + return Err(CaptureError::LimitExceeded( + "decoded transcript header".into(), + )); + } + Ok(Some(InputHead { + bytes, + len, + modified_ms, + })) + } + + pub fn open( + &self, + root: &Path, + cancelled: impl Fn() -> bool, + ) -> Result, CaptureError> { + if self.segments.is_empty() { + return Ok(None); + } + let mut opened = Vec::new(); + let mut stamp = blake3::Hasher::new(); + stamp.update(&serde_json::to_vec(&self.segments).expect("input plan serializes")); + stamp.update(&self.prefix); + for segment in &self.segments { + if cancelled() { + return Err(cancel_error(root)); + } + let Some((candidate, path, file)) = open_candidate(root, &segment.candidates)? else { + return Ok(None); + }; + hash_metadata(&mut stamp, candidate, &path, &file)?; + opened.push((segment, candidate, path, file)); + } + let stamp = stamp.finalize().to_hex().to_string(); + // Preserve old inode checkpoints for an ordinary uncompressed file. + if let [(segment, candidate, path, file)] = opened.as_slice() + && candidate.encoding == InputEncoding::Plain + && self.prefix.is_empty() + && segment.start == 0 + && segment.end.is_none() + { + let identity = file_identity( + path, + &file + .metadata() + .map_err(|e| io_error("reading input metadata", path, e))?, + ); + let expected = segment.header_hash; + let (_, _, path, mut file) = opened.pop().expect("one input"); + if let Some(expected) = expected { + let mut header = Vec::new(); + let mut reader = std::io::BufReader::new((&mut file).take(32 * 1024 * 1024 + 1)); + std::io::BufRead::read_until(&mut reader, b'\n', &mut header) + .map_err(|e| io_error("verifying pinned header", &path, e))?; + if *blake3::hash(&header).as_bytes() != expected { + return Err(invalid(&path, "transcript header changed during capture")); + } + file.rewind() + .map_err(|e| io_error("rewinding transcript", &path, e))?; + } + return Ok(Some(OpenedInput { + file, + path, + identity, + stamp, + boundaries: Vec::new(), + cancel: None, + })); + } + let mut spool = + tempfile::tempfile().map_err(|e| io_error("creating decoded spool", root, e))?; + if self.prefix.len() > 32 * 1024 * 1024 { + return Err(CaptureError::LimitExceeded( + "transcript prefix exceeds 32 MiB".into(), + )); + } + spool + .write_all(&self.prefix) + .map_err(|e| io_error("writing selected header", root, e))?; + let mut total = self.prefix.len() as u64; + let mut boundaries = Vec::new(); + for (segment, candidate, path, file) in opened { + let monitor = file + .try_clone() + .map_err(|e| io_error("pinning transcript", &path, e))?; + let before = monitor + .metadata() + .map_err(|e| io_error("reading input metadata", &path, e))?; + let mut reader = decoder(file, candidate.encoding, &path)?; + let mut boundary_record = Vec::new(); + let mut buffer = vec![0; BUFFER_BYTES]; + let mut offset = 0_u64; + let mut last = None; + let mut header = blake3::Hasher::new(); + let mut in_header = true; + let mut header_len = 0_usize; + loop { + if cancelled() { + return Err(cancel_error(&path)); + } + let count = reader.read(&mut buffer).map_err(|e| { + decode_error("decoding transcript", &path, candidate.encoding, e) + })?; + if count == 0 { + break; + } + let next = offset + .checked_add(count as u64) + .ok_or_else(|| CaptureError::LimitExceeded("decoded offset overflow".into()))?; + if next > MAX_DECODED_BYTES { + return Err(CaptureError::LimitExceeded( + "decoded transcript exceeds 64 GiB".into(), + )); + } + if in_header && segment.header_hash.is_some() { + let end = buffer[..count] + .iter() + .position(|b| *b == b'\n') + .map(|i| i + 1); + let header_bytes = &buffer[..end.unwrap_or(count)]; + header_len += header_bytes.len(); + if header_len > 32 * 1024 * 1024 { + return Err(CaptureError::LimitExceeded( + "decoded transcript header".into(), + )); + } + header.update(header_bytes); + if end.is_some() { + in_header = false; + if Some(*header.finalize().as_bytes()) != segment.header_hash { + return Err(invalid(&path, "transcript header changed during capture")); + } + } + } + let start = offset.max(segment.start); + let end = next.min(segment.end.unwrap_or(u64::MAX)); + if start < end { + let bytes = &buffer[(start - offset) as usize..(end - offset) as usize]; + total = total.checked_add(bytes.len() as u64).ok_or_else(|| { + CaptureError::LimitExceeded("decoded source overflow".into()) + })?; + if total > MAX_DECODED_BYTES { + return Err(CaptureError::LimitExceeded( + "composed transcript exceeds 64 GiB".into(), + )); + } + spool + .write_all(bytes) + .map_err(|e| io_error("writing decoded spool", &path, e))?; + last = bytes.last().copied(); + if segment.end_record_index.is_some() { + // Keep only the final complete record of this fixed + // range, bounded like every projected native record. + for chunk in bytes.split_inclusive(|b| *b == b'\n') { + if boundary_record.ends_with(b"\n") { + boundary_record.clear(); + } + if boundary_record.len() + chunk.len() > 32 * 1024 * 1024 { + return Err(CaptureError::LimitExceeded( + "referenced boundary record exceeds 32 MiB".into(), + )); + } + boundary_record.extend_from_slice(chunk); + } + } + } + offset = next; + // Bounded plain prefixes need not read a parent's later + // growth. Zstd is always drained to validate its checksum. + if candidate.encoding == InputEncoding::Plain + && segment.end.is_some_and(|end| offset >= end) + { + break; + } + } + let after = monitor + .metadata() + .map_err(|e| io_error("rechecking decoded input", &path, e))?; + if after.len() < before.len() + || (after.len() == before.len() && after.modified().ok() != before.modified().ok()) + || (candidate.encoding == InputEncoding::Zstd && after.len() != before.len()) + { + return Err(invalid(&path, "transcript changed while decoding")); + } + if let Some(expected) = segment.end_record_index { + boundaries.push((expected, boundary_record)); + } + if offset < segment.start || segment.end.is_some_and(|end| offset < end) { + return Err(invalid(&path, "referenced decoded range is truncated")); + } + if segment.end.is_some() && last != Some(b'\n') && segment.end != Some(segment.start) { + return Err(invalid( + &path, + "referenced range does not end at a complete record", + )); + } + if segment.header_hash.is_some() && in_header { + return Err(invalid(&path, "referenced header is incomplete")); + } + } + spool + .rewind() + .map_err(|e| io_error("rewinding decoded spool", root, e))?; + // The temporary inode is never durable identity. A different physical + // stamp asks the tailer to verify the committed decoded prefix. + Ok(Some(OpenedInput { + file: spool, + path: root.to_path_buf(), + identity: FileIdentity::Path(stamp.as_bytes().to_vec()), + stamp, + boundaries, + cancel: None, + })) + } +} + +pub struct InputHead { + pub bytes: Vec, + pub len: Option, + pub modified_ms: i64, +} + +#[derive(Debug)] +pub struct OpenedInput { + pub file: File, + pub path: PathBuf, + pub identity: FileIdentity, + pub stamp: String, + pub boundaries: Vec<(u64, Vec)>, + pub cancel: Option>, +} + +fn decoder( + file: File, + encoding: InputEncoding, + path: &Path, +) -> Result, CaptureError> { + match encoding { + InputEncoding::Plain => Ok(Box::new(file)), + InputEncoding::Zstd => zstd::stream::read::Decoder::new(file) + .map(|r| Box::new(r) as Box) + .map_err(|e| decode_error("opening zstd transcript", path, InputEncoding::Zstd, e)), + } +} + +fn open_candidate<'a>( + root: &Path, + candidates: &'a [InputCandidate], +) -> Result, CaptureError> { + for candidate in candidates { + let relative = Path::new(&candidate.path); + if relative.is_absolute() + || relative + .components() + .any(|c| !matches!(c, Component::Normal(_))) + { + return Err(CaptureError::InvalidConfig( + "input candidate must be root-relative".into(), + )); + } + let path = root.join(relative); + match File::open(&path) { + Ok(file) => return Ok(Some((candidate, path, file))), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(io_error("opening transcript representation", &path, e)), + } + } + Ok(None) +} + +fn hash_metadata( + hash: &mut blake3::Hasher, + candidate: &InputCandidate, + path: &Path, + file: &File, +) -> Result<(), CaptureError> { + let metadata = file + .metadata() + .map_err(|e| io_error("reading input metadata", path, e))?; + hash.update(candidate.path.as_bytes()); + hash.update( + format!( + "{:?}:{}:{:?}", + file_identity(path, &metadata), + metadata.len(), + metadata.modified().ok() + ) + .as_bytes(), + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + hash.update(&metadata.ctime().to_le_bytes()); + hash.update(&metadata.ctime_nsec().to_le_bytes()); + } + Ok(()) +} + +fn invalid(path: &Path, message: &str) -> CaptureError { + io_error( + "validating decoded transcript", + path, + std::io::Error::new(std::io::ErrorKind::InvalidData, message), + ) +} +fn cancel_error(path: &Path) -> CaptureError { + io_error( + "decoding transcript", + path, + std::io::Error::new(std::io::ErrorKind::Interrupted, "capture cancelled"), + ) +} + +// zstd reports malformed frame/checksum failures as ErrorKind::Other with +// no OS errno. Classify these as data faults, while retaining filesystem IO +// errors (permission, storage failure, etc.) as actual operational failures. +fn decode_error( + operation: &'static str, + path: &Path, + encoding: InputEncoding, + error: std::io::Error, +) -> CaptureError { + let error = if encoding == InputEncoding::Zstd + && error.kind() == std::io::ErrorKind::Other + && error.raw_os_error().is_none() + { + std::io::Error::new(std::io::ErrorKind::InvalidData, error) + } else { + error + }; + io_error(operation, path, error) +} diff --git a/crates/capture/src/lib.rs b/crates/capture/src/lib.rs index 65b0aab6..02e2f8fb 100644 --- a/crates/capture/src/lib.rs +++ b/crates/capture/src/lib.rs @@ -8,6 +8,7 @@ pub mod discovery; pub mod document; +pub mod input; pub mod ledger; mod recompress; pub mod segment; @@ -95,3 +96,11 @@ pub(crate) fn io_error( source, } } + +impl CaptureError { + /// A native source exists but cannot yet supply a complete decoded view. + /// This is not EOF and must not activate an empty replacement. + pub fn incomplete_input(&self) -> bool { + matches!(self, Self::Io { source, .. } if matches!(source.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof)) + } +} diff --git a/crates/capture/src/tailer.rs b/crates/capture/src/tailer.rs index a1f2596d..44ffb2e8 100644 --- a/crates/capture/src/tailer.rs +++ b/crates/capture/src/tailer.rs @@ -19,6 +19,11 @@ use std::path::Path; use crate::{CaptureError, io_error}; +mod checkpoint_bytes; +mod verification; +use checkpoint_bytes::ByteReader; +use verification::verify_chain; + const CHECKPOINT_MAGIC: &[u8] = b"RSTC"; const CHECKPOINT_VERSION: u8 = 2; /// Oldest checkpoint version `decode` still accepts. Version 1 predates the @@ -445,16 +450,84 @@ impl Tailer { } Err(error) => return Err(io_error("opening", path, error)), }; + let metadata = file + .metadata() + .map_err(|e| io_error("reading metadata for", path, e))?; + let identity = file_identity(path, &metadata); + self.read_handle( + &mut file, + path, + identity, + previous, + force_replay, + max_read_bytes, + end, + false, + None, + ) + } + + /// A pinned decoded input. Path replacement cannot invalidate this open + /// snapshot. Representation changes verify the entire committed prefix + /// before continuing the same generation; older chain-less cursors replay. + pub fn read_input( + &self, + input: &mut crate::input::OpenedInput, + previous: Option<&TailCheckpoint>, + force_replay: bool, + end: Option, + ) -> Result { + self.read_handle( + &mut input.file, + &input.path, + input.identity.clone(), + previous, + force_replay, + u64::MAX, + end, + true, + input.cancel.as_deref(), + ) + } + + #[allow(clippy::too_many_arguments)] + fn read_handle( + &self, + file: &mut File, + path: &Path, + identity: FileIdentity, + previous: Option<&TailCheckpoint>, + force_replay: bool, + max_read_bytes: u64, + end: Option, + pinned: bool, + cancel: Option<&std::sync::atomic::AtomicBool>, + ) -> Result { let mut accounting = ReadAccounting::new(max_read_bytes); let initial_metadata = file .metadata() .map_err(|error| io_error("reading metadata for", path, error))?; let physical_len = initial_metadata.len(); - let identity = file_identity(path, &initial_metadata); + let mut equivalent = None; + if pinned + && let Some(previous) = previous + && previous.identity != identity + && previous.committed_chain.is_some() + && physical_len >= previous.committed_offset + && matches!( + verify_chain(file, path, previous, cancel)?, + CommittedVerification::Verified + ) + { + let mut rebound = previous.clone(); + rebound.identity = identity.clone(); + equivalent = Some(rebound); + } + let previous = equivalent.as_ref().or(previous); let (generation, start_offset, transition) = self.resume_point( - &mut file, + file, path, physical_len, &identity, @@ -485,7 +558,7 @@ impl Tailer { }; let framed = self.frame( - &mut file, + file, path, start_offset, resume_chain, @@ -493,8 +566,7 @@ impl Tailer { generation, &mut accounting, )?; - let prefix = - self.prefix_anchor(&mut file, path, framed.committed_offset, &mut accounting)?; + let prefix = self.prefix_anchor(file, path, framed.committed_offset, &mut accounting)?; // Stability recheck against both the open handle and the path: a // shrink or identity change under our feet means the snapshot cannot @@ -503,19 +575,21 @@ impl Tailer { let handle_metadata = file .metadata() .map_err(|error| io_error("rechecking metadata for", path, error))?; - let path_metadata = match std::fs::metadata(path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if handle_metadata.len() < snapshot_len { + return Ok(TailRead::Retry); + } + if !pinned { + let path_metadata = match std::fs::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(TailRead::Retry); + } + Err(error) => return Err(io_error("rechecking path for", path, error)), + }; + if file_identity(path, &path_metadata) != identity || path_metadata.len() < snapshot_len + { return Ok(TailRead::Retry); } - Err(error) => return Err(io_error("rechecking path for", path, error)), - }; - if file_identity(path, &handle_metadata) != identity - || file_identity(path, &path_metadata) != identity - || handle_metadata.len() < snapshot_len - || path_metadata.len() < snapshot_len - { - return Ok(TailRead::Retry); } let checkpoint = TailCheckpoint { @@ -571,26 +645,23 @@ impl Tailer { if metadata.len() < previous.committed_offset { return Ok(CommittedVerification::Truncated); } - let mut remaining = previous.committed_offset; - let mut chain = CONTENT_CHAIN_BASIS; - let mut buffer = [0_u8; READ_BUFFER_BYTES]; - while remaining > 0 { - let wanted = usize::try_from(remaining.min(READ_BUFFER_BYTES as u64)) - .expect("bounded read length always fits usize"); - let count = file - .read(&mut buffer[..wanted]) - .map_err(|error| io_error("verifying", path, error))?; - if count == 0 { - return Ok(CommittedVerification::Truncated); - } - chain = chain_bytes(chain, &buffer[..count]); - remaining -= count as u64; + verify_chain(&mut file, path, previous, None) + } + + pub fn verify_input( + &self, + input: &mut crate::input::OpenedInput, + previous: &TailCheckpoint, + ) -> Result { + if previous.committed_chain.is_none() && previous.identity != input.identity { + return Ok(CommittedVerification::IdentityChanged); } - Ok(match previous.committed_chain { - Some(expected) if expected == chain => CommittedVerification::Verified, - Some(_) => CommittedVerification::Mismatch, - None => CommittedVerification::Established(chain), - }) + verify_chain( + &mut input.file, + &input.path, + previous, + input.cancel.as_deref(), + ) } #[allow(clippy::too_many_arguments)] @@ -945,77 +1016,6 @@ fn next_generation(current: u64) -> Result { .ok_or_else(|| CaptureError::InvalidCheckpoint("tail generation overflowed".to_string())) } -struct ByteReader<'a> { - remaining: &'a [u8], -} - -impl<'a> ByteReader<'a> { - /// Accept any version in `versions`, returning the one found so the - /// caller can decode the fields that version carries. - fn versioned( - bytes: &'a [u8], - magic: &[u8], - versions: std::ops::RangeInclusive, - ) -> Result<(Self, u8), CaptureError> { - let Some(rest) = bytes.strip_prefix(magic) else { - return Err(CaptureError::InvalidCheckpoint( - "checkpoint magic does not match".to_string(), - )); - }; - let mut reader = Self { remaining: rest }; - let version = reader.byte()?; - if !versions.contains(&version) { - return Err(CaptureError::InvalidCheckpoint( - "unsupported checkpoint version".to_string(), - )); - } - Ok((reader, version)) - } - - fn finish(self) -> Result<(), CaptureError> { - if self.remaining.is_empty() { - Ok(()) - } else { - Err(CaptureError::InvalidCheckpoint( - "checkpoint contains trailing bytes".to_string(), - )) - } - } - - fn byte(&mut self) -> Result { - Ok(self.take(1)?[0]) - } - - fn u64(&mut self) -> Result { - Ok(u64::from_be_bytes(self.take(8)?.try_into().map_err( - |_| CaptureError::InvalidCheckpoint("truncated u64".to_string()), - )?)) - } - - fn len_u64(&mut self) -> Result { - usize::try_from(self.u64()?).map_err(|_| { - CaptureError::InvalidCheckpoint("checkpoint length is too large".to_string()) - }) - } - - fn hash(&mut self) -> Result<[u8; HASH_BYTES], CaptureError> { - self.take(HASH_BYTES)? - .try_into() - .map_err(|_| CaptureError::InvalidCheckpoint("truncated hash".to_string())) - } - - fn take(&mut self, count: usize) -> Result<&'a [u8], CaptureError> { - if count > self.remaining.len() { - return Err(CaptureError::InvalidCheckpoint( - "checkpoint is truncated".to_string(), - )); - } - let (value, rest) = self.remaining.split_at(count); - self.remaining = rest; - Ok(value) - } -} - #[cfg(test)] mod cut_tests; diff --git a/crates/capture/src/tailer/checkpoint_bytes.rs b/crates/capture/src/tailer/checkpoint_bytes.rs new file mode 100644 index 00000000..c2ff2db9 --- /dev/null +++ b/crates/capture/src/tailer/checkpoint_bytes.rs @@ -0,0 +1,73 @@ +//! Bounded binary reader for durable tail checkpoints. +use super::*; + +pub(super) struct ByteReader<'a> { + remaining: &'a [u8], +} + +impl<'a> ByteReader<'a> { + /// Accept any version in `versions`, returning the one found so the + /// caller can decode the fields that version carries. + pub(super) fn versioned( + bytes: &'a [u8], + magic: &[u8], + versions: std::ops::RangeInclusive, + ) -> Result<(Self, u8), CaptureError> { + let Some(rest) = bytes.strip_prefix(magic) else { + return Err(CaptureError::InvalidCheckpoint( + "checkpoint magic does not match".to_string(), + )); + }; + let mut reader = Self { remaining: rest }; + let version = reader.byte()?; + if !versions.contains(&version) { + return Err(CaptureError::InvalidCheckpoint( + "unsupported checkpoint version".to_string(), + )); + } + Ok((reader, version)) + } + + pub(super) fn finish(self) -> Result<(), CaptureError> { + if self.remaining.is_empty() { + Ok(()) + } else { + Err(CaptureError::InvalidCheckpoint( + "checkpoint contains trailing bytes".to_string(), + )) + } + } + + pub(super) fn byte(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + pub(super) fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes(self.take(8)?.try_into().map_err( + |_| CaptureError::InvalidCheckpoint("truncated u64".to_string()), + )?)) + } + + pub(super) fn len_u64(&mut self) -> Result { + usize::try_from(self.u64()?).map_err(|_| { + CaptureError::InvalidCheckpoint("checkpoint length is too large".to_string()) + }) + } + + pub(super) fn hash(&mut self) -> Result<[u8; HASH_BYTES], CaptureError> { + self.take(HASH_BYTES)? + .try_into() + .map_err(|_| CaptureError::InvalidCheckpoint("truncated hash".to_string())) + } + + pub(super) fn take(&mut self, count: usize) -> Result<&'a [u8], CaptureError> { + if count > self.remaining.len() { + return Err(CaptureError::InvalidCheckpoint( + "checkpoint is truncated".to_string(), + )); + } + let (value, rest) = self.remaining.split_at(count); + self.remaining = rest; + Ok(value) + } +} diff --git a/crates/capture/src/tailer/verification.rs b/crates/capture/src/tailer/verification.rs new file mode 100644 index 00000000..fa30e091 --- /dev/null +++ b/crates/capture/src/tailer/verification.rs @@ -0,0 +1,39 @@ +//! Full decoded-prefix verification for audits and representation transitions. +use super::*; + +pub(super) fn verify_chain( + file: &mut File, + path: &Path, + previous: &TailCheckpoint, + cancel: Option<&std::sync::atomic::AtomicBool>, +) -> Result { + file.rewind() + .map_err(|e| io_error("rewinding verification", path, e))?; + let mut remaining = previous.committed_offset; + let mut chain = CONTENT_CHAIN_BASIS; + let mut buffer = [0_u8; READ_BUFFER_BYTES]; + while remaining > 0 { + if cancel.is_some_and(|flag| flag.load(std::sync::atomic::Ordering::Relaxed)) { + return Err(io_error( + "verifying decoded prefix", + path, + std::io::Error::new(std::io::ErrorKind::Interrupted, "capture cancelled"), + )); + } + let wanted = usize::try_from(remaining.min(READ_BUFFER_BYTES as u64)) + .expect("bounded read length always fits usize"); + let count = file + .read(&mut buffer[..wanted]) + .map_err(|error| io_error("verifying", path, error))?; + if count == 0 { + return Ok(CommittedVerification::Truncated); + } + chain = chain_bytes(chain, &buffer[..count]); + remaining -= count as u64; + } + Ok(match previous.committed_chain { + Some(expected) if expected == chain => CommittedVerification::Verified, + Some(_) => CommittedVerification::Mismatch, + None => CommittedVerification::Established(chain), + }) +} diff --git a/crates/capture/tests/decoded_input.rs b/crates/capture/tests/decoded_input.rs new file mode 100644 index 00000000..7a5dcab8 --- /dev/null +++ b/crates/capture/tests/decoded_input.rs @@ -0,0 +1,176 @@ +use rosetta_capture::input::{InputCandidate, InputEncoding, InputSource}; +use rosetta_capture::{TailConfig, TailRead, Tailer, Transition}; +use std::io::{Read, Write}; +use tempfile::TempDir; + +fn source() -> InputSource { + InputSource::single(vec![ + InputCandidate { + path: "source.jsonl".into(), + encoding: InputEncoding::Plain, + }, + InputCandidate { + path: "source.jsonl.zst".into(), + encoding: InputEncoding::Zstd, + }, + ]) +} +fn batch(read: TailRead) -> rosetta_capture::TailBatch { + match read { + TailRead::Batch(b) => b, + other => panic!("{other:?}"), + } +} + +#[test] +fn decoded_offsets_and_generation_survive_encoding_changes() { + let dir = TempDir::new().unwrap(); + let plan = source(); + let bytes = b"{\"a\":1}\r\n{\"b\":2}\nunfinished"; + std::fs::write( + dir.path().join("source.jsonl.zst"), + zstd::encode_all(&bytes[..], 3).unwrap(), + ) + .unwrap(); + let tailer = Tailer::new(TailConfig::json_lines()).unwrap(); + let mut cold = plan.open(dir.path(), || false).unwrap().unwrap(); + let first = batch(tailer.read_input(&mut cold, None, false, None).unwrap()); + assert_eq!(first.checkpoint.committed_offset, 17); + assert_eq!(first.checkpoint.incomplete_suffix_len, 10); + std::fs::write(dir.path().join("source.jsonl"), bytes).unwrap(); + let mut hot = plan.open(dir.path(), || false).unwrap().unwrap(); + let resumed = batch( + tailer + .read_input(&mut hot, Some(&first.checkpoint), false, None) + .unwrap(), + ); + assert_eq!(resumed.transition, Transition::Continued); + assert!(resumed.items.is_empty()); + assert_eq!(resumed.checkpoint.generation, first.checkpoint.generation); + std::fs::OpenOptions::new() + .append(true) + .open(dir.path().join("source.jsonl")) + .unwrap() + .write_all(b"\n") + .unwrap(); + let appended = batch( + tailer + .read_input(&mut hot, Some(&resumed.checkpoint), false, None) + .unwrap(), + ); + assert_eq!(appended.items.len(), 1); + assert_eq!(appended.checkpoint.committed_offset, bytes.len() as u64 + 1); +} + +#[test] +fn open_handle_survives_unlink_without_switching_sources_mid_pass() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("source.jsonl"), b"first\nsecond\n").unwrap(); + let mut opened = source().open(dir.path(), || false).unwrap().unwrap(); + std::fs::remove_file(dir.path().join("source.jsonl")).unwrap(); + std::fs::write( + dir.path().join("source.jsonl.zst"), + zstd::encode_all(&b"other\n"[..], 3).unwrap(), + ) + .unwrap(); + let tailer = Tailer::new(TailConfig { + max_records_per_batch: 1, + ..TailConfig::json_lines() + }) + .unwrap(); + let first = batch(tailer.read_input(&mut opened, None, false, None).unwrap()); + let second = batch( + tailer + .read_input(&mut opened, Some(&first.checkpoint), false, None) + .unwrap(), + ); + assert_eq!(second.transition, Transition::Continued); + assert_eq!(second.checkpoint.committed_offset, 13); +} + +#[test] +fn replacement_checks_the_entire_committed_prefix() { + let dir = TempDir::new().unwrap(); + let mut bytes = b"original\n".to_vec(); + bytes.extend(vec![b'x'; 9000]); + bytes.push(b'\n'); + std::fs::write(dir.path().join("source.jsonl"), &bytes).unwrap(); + let tailer = Tailer::new(TailConfig::json_lines()).unwrap(); + let mut opened = source().open(dir.path(), || false).unwrap().unwrap(); + let first = batch(tailer.read_input(&mut opened, None, false, None).unwrap()); + bytes[..8].copy_from_slice(b"replaced"); + std::fs::write( + dir.path().join("source.jsonl.zst"), + zstd::encode_all(&bytes[..], 3).unwrap(), + ) + .unwrap(); + std::fs::remove_file(dir.path().join("source.jsonl")).unwrap(); + let mut changed = source().open(dir.path(), || false).unwrap().unwrap(); + let next = batch( + tailer + .read_input(&mut changed, Some(&first.checkpoint), false, None) + .unwrap(), + ); + assert!(next.transition.starts_new_generation()); + assert_eq!(next.checkpoint.generation, first.checkpoint.generation + 1); +} + +#[test] +fn unknown_size_multiframe_and_checksum_failures() { + let dir = TempDir::new().unwrap(); + // encode_all uses an unknown-size streaming encoder. + let first = zstd::encode_all(&b"one\n"[..], 3).unwrap(); + let second = zstd::encode_all(&b"two\n"[..], 3).unwrap(); + let both = [first.clone(), second].concat(); + std::fs::write(dir.path().join("source.jsonl.zst"), &both).unwrap(); + let mut opened = source().open(dir.path(), || false).unwrap().unwrap(); + let mut decoded = String::new(); + opened.file.read_to_string(&mut decoded).unwrap(); + assert_eq!(decoded, "one\ntwo\n"); + std::fs::write(dir.path().join("source.jsonl.zst"), &both[..both.len() - 2]).unwrap(); + assert!(source().open(dir.path(), || false).is_err()); + std::fs::write(dir.path().join("source.jsonl.zst"), first).unwrap(); + assert!(source().open(dir.path(), || true).is_err()); +} + +#[test] +fn plain_precedence_never_hides_corrupt_preferred_content() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("source.jsonl"), b"plain\n").unwrap(); + std::fs::write(dir.path().join("source.jsonl.zst"), b"corrupt").unwrap(); + let before = source().stamp(dir.path()).unwrap(); + let mut opened = source().open(dir.path(), || false).unwrap().unwrap(); + let mut decoded = String::new(); + opened.file.read_to_string(&mut decoded).unwrap(); + assert_eq!(decoded, "plain\n"); + assert_eq!(before, source().stamp(dir.path()).unwrap()); + std::fs::remove_file(dir.path().join("source.jsonl")).unwrap(); + assert!(source().open(dir.path(), || false).is_err()); +} + +#[test] +fn a_chain_less_legacy_checkpoint_cannot_adopt_a_new_representation() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("source.jsonl"), b"one\ntwo\n").unwrap(); + let tailer = Tailer::new(TailConfig::json_lines()).unwrap(); + let mut input = source().open(dir.path(), || false).unwrap().unwrap(); + let mut previous = batch(tailer.read_input(&mut input, None, false, None).unwrap()).checkpoint; + previous.committed_chain = None; + std::fs::write( + dir.path().join("source.jsonl.zst"), + zstd::encode_all(&b"one\ntwo\n"[..], 3).unwrap(), + ) + .unwrap(); + std::fs::remove_file(dir.path().join("source.jsonl")).unwrap(); + let mut input = source().open(dir.path(), || false).unwrap().unwrap(); + assert_eq!( + tailer.verify_input(&mut input, &previous).unwrap(), + rosetta_capture::CommittedVerification::IdentityChanged + ); + let next = batch( + tailer + .read_input(&mut input, Some(&previous), false, None) + .unwrap(), + ); + assert!(next.transition.starts_new_generation()); +} From 8a355f1c92aa7b12d6e3f8c9648a1afc2728331e Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 15:54:35 -0800 Subject: [PATCH 3/6] feat(codex): ingest compressed archives and selected shared-history views --- Cargo.lock | 2 + crates/adapter-support/Cargo.toml | 1 + crates/adapter-support/src/adapter.rs | 44 ++ crates/adapter-support/src/lib.rs | 2 + crates/adapter-support/src/transcripts.rs | 23 + crates/adapters/codex/src/adapter.rs | 68 +- crates/adapters/codex/src/lib.rs | 1 + crates/adapters/codex/src/normalize.rs | 30 + crates/adapters/codex/src/raw/rollout.rs | 4 + crates/adapters/codex/src/spec.rs | 7 + crates/adapters/codex/src/storage.rs | 136 ++++ crates/adapters/codex/src/usage.rs | 14 + crates/pipeline/Cargo.toml | 1 + crates/pipeline/src/capture/pool/cut_tests.rs | 1 + crates/pipeline/src/capture/pool/tests.rs | 1 + crates/pipeline/src/checkpoint.rs | 5 +- crates/pipeline/src/closed/tests.rs | 2 + crates/pipeline/src/closed/trees/tests.rs | 5 + crates/pipeline/src/closed/walk_order.rs | 6 +- .../pipeline/src/closed/walk_order/tests.rs | 2 + crates/pipeline/src/discover.rs | 10 +- crates/pipeline/src/early_tests.rs | 2 + crates/pipeline/src/early_trees_tests.rs | 1 + crates/pipeline/src/lib.rs | 8 +- crates/pipeline/src/live.rs | 67 +- crates/pipeline/src/meta.rs | 145 ++-- crates/pipeline/src/meta/inventory.rs | 89 ++- crates/pipeline/src/meta/transcripts.rs | 299 ++++++++ crates/pipeline/src/meta_journal.rs | 67 ++ crates/pipeline/src/plan.rs | 1 + crates/pipeline/src/replay.rs | 46 +- crates/pipeline/src/sched.rs | 1 + crates/pipeline/src/skim.rs | 40 +- crates/pipeline/src/stream.rs | 2 +- crates/pipeline/src/stream/pass.rs | 114 ++- crates/pipeline/src/streamed/ordinals.rs | 1 + crates/pipeline/src/streamed/slices_tests.rs | 1 + crates/pipeline/tests/codex_storage.rs | 674 ++++++++++++++++++ crates/pipeline/tests/exclusions.rs | 5 +- crates/pipeline/tests/ledger_only.rs | 8 +- crates/rosettad/src/ingest_metrics/tests.rs | 1 + docs/ARCHITECTURE.md | 48 +- 42 files changed, 1841 insertions(+), 144 deletions(-) create mode 100644 crates/adapter-support/src/transcripts.rs create mode 100644 crates/adapters/codex/src/storage.rs create mode 100644 crates/pipeline/src/meta/transcripts.rs create mode 100644 crates/pipeline/src/meta_journal.rs create mode 100644 crates/pipeline/tests/codex_storage.rs diff --git a/Cargo.lock b/Cargo.lock index 4badc65f..f4ef8b2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1441,6 +1441,7 @@ dependencies = [ "image", "percent-encoding", "pulldown-cmark", + "rosetta-capture", "rosetta-core", "rosetta-media", "serde", @@ -1529,6 +1530,7 @@ dependencies = [ "serde_json", "tempfile", "walkdir", + "zstd", ] [[package]] diff --git a/crates/adapter-support/Cargo.toml b/crates/adapter-support/Cargo.toml index c40b8f9e..fe25cf35 100644 --- a/crates/adapter-support/Cargo.toml +++ b/crates/adapter-support/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] +rosetta-capture = { path = "../capture" } rosetta-media = { path = "../media" } base64 = { workspace = true } blake3 = { workspace = true } diff --git a/crates/adapter-support/src/adapter.rs b/crates/adapter-support/src/adapter.rs index 98d052d2..ae3a3dfa 100644 --- a/crates/adapter-support/src/adapter.rs +++ b/crates/adapter-support/src/adapter.rs @@ -14,6 +14,16 @@ use rosetta_core::{EvidenceOrder, IngestOp, SourceIdentity}; use crate::GroundTruthDoc; +pub use rosetta_capture::input::{InputCandidate, InputEncoding, InputSource}; + +/// Physical representations of one logical transcript. Paths are relative +/// to the runtime root; the stable key survives compression and relocation. +#[derive(Debug, Clone)] +pub struct TranscriptLocation { + pub logical_path: String, + pub source: InputSource, +} + /// A stream parser the generic pipeline knows how to frame. Layout inventory /// is intentionally broader than this list: an adapter must opt a stream into /// normalization instead of every matching file being treated as a transcript. @@ -256,6 +266,40 @@ impl NormalizeCtx { /// One implementation per agent. Object-safe: the pipeline holds a registry /// of `&'static dyn AgentAdapter`. pub trait AgentAdapter: Send + Sync { + /// Native position of a range's final record; capture remains format-blind. + fn transcript_record_index(&self, _record: &[u8]) -> Option { + None + } + + fn transcript_header( + &self, + _path: &str, + _first: &[u8], + ) -> Result, String> { + Ok(None) + } + + /// A promise that snapshot records emit no canonical operations. Such + /// shared metadata sources can precede independently publishable trees. + fn snapshot_is_metadata_only(&self, stream: &str) -> bool { + self.snapshot_plan(stream) + .is_some_and(|plan| plan.record_queries.is_empty()) + } + + fn transcript_selection( + &self, + _stream: &str, + _row: &serde_json::Value, + ) -> Option { + None + } + + /// Opt in to representation-independent, decoded-byte capture. Native + /// naming rules stay in the adapter; capture only opens declared paths. + fn transcript_location(&self, _path: &str) -> Option { + None + } + /// Agent kind, e.g. "claude-code". fn kind(&self) -> &'static str; diff --git a/crates/adapter-support/src/lib.rs b/crates/adapter-support/src/lib.rs index 3fb56e37..2d7c1559 100644 --- a/crates/adapter-support/src/lib.rs +++ b/crates/adapter-support/src/lib.rs @@ -159,3 +159,5 @@ impl GroundTruthDoc { pub fn glob_match(pattern: &str, path: &str) -> bool { Glob::new(pattern).matches(path) } + +pub mod transcripts; diff --git a/crates/adapter-support/src/transcripts.rs b/crates/adapter-support/src/transcripts.rs new file mode 100644 index 00000000..cd094d35 --- /dev/null +++ b/crates/adapter-support/src/transcripts.rs @@ -0,0 +1,23 @@ +//! Pure native metadata for resolving shared transcript prefixes. + +#[derive(Debug, Clone)] +pub struct TranscriptReference { + pub source_id: String, + pub end_offset: u64, + pub end_ordinal: u64, +} + +#[derive(Debug, Clone)] +pub struct TranscriptHeader { + pub source_id: String, + pub session_id: String, + pub base: Option, + pub start_ordinal: u64, + pub paginated: bool, +} + +#[derive(Debug, Clone)] +pub struct TranscriptSelection { + /// Native path, absolute or relative to the runtime root. + pub path: String, +} diff --git a/crates/adapters/codex/src/adapter.rs b/crates/adapters/codex/src/adapter.rs index 44097e16..37f1ee3e 100644 --- a/crates/adapters/codex/src/adapter.rs +++ b/crates/adapters/codex/src/adapter.rs @@ -18,6 +18,40 @@ use crate::time::rfc3339_ms; pub struct CodexAdapter; impl AgentAdapter for CodexAdapter { + fn transcript_record_index(&self, record: &[u8]) -> Option { + serde_json::from_slice::(record) + .ok()? + .get("ordinal")? + .as_u64() + } + + fn transcript_header( + &self, + path: &str, + first: &[u8], + ) -> Result, String> { + crate::storage::header(path, first) + } + fn transcript_selection( + &self, + stream: &str, + row: &Value, + ) -> Option { + if stream != spec::STATE_DB { + return None; + } + Some(rosetta_adapter_support::transcripts::TranscriptSelection { + path: crate::storage::selection_path(row.get("rollout_path")?.as_str()?), + }) + } + + fn transcript_location( + &self, + path: &str, + ) -> Option { + crate::storage::location(path) + } + fn kind(&self) -> &'static str { "codex" } @@ -28,21 +62,29 @@ impl AgentAdapter for CodexAdapter { fn stream_driver(&self, stream_id: &str) -> Option { match stream_id { - spec::ROLLOUT => Some(StreamDriver::JsonLines), + spec::ROLLOUT + | "rollout-compressed" + | "archived-rollout" + | "archived-rollout-compressed" => Some(StreamDriver::JsonLines), spec::STATE_DB => Some(StreamDriver::SqliteSnapshot), _ => None, } } fn census_session_streams(&self) -> Option<&'static [&'static str]> { - Some(&[spec::ROLLOUT]) + Some(&[ + spec::ROLLOUT, + "rollout-compressed", + "archived-rollout", + "archived-rollout-compressed", + ]) } /// A rollout writes its own session, except a guardian review's usage, /// which lands on the parent its header names: metadata links the /// guardian to that parent, so both are one tree. The state database is - /// shared by every tree but declares no record queries, so its members - /// emit no ops (`snapshot_plan`). + /// shared by every tree; its native selector records emit no canonical + /// ops, preserving closure (`snapshot_plan`, `ingest_line`). fn cold_admission(&self) -> rosetta_adapter_support::ColdAdmission { rosetta_adapter_support::ColdAdmission::RootTree } @@ -61,7 +103,7 @@ impl AgentAdapter for CodexAdapter { // renditions made after import (2026-11-01): legacy bundle refs stay // valid, so re-streaming every source is not worth it; a rebuild // converts them (docs/ARCHITECTURE.md). - 21 + 22 } fn html_artifact_hints( @@ -84,6 +126,10 @@ impl AgentAdapter for CodexAdapter { } } + fn snapshot_is_metadata_only(&self, stream: &str) -> bool { + stream == spec::STATE_DB + } + fn snapshot_plan(&self, stream_id: &str) -> Option { (stream_id == spec::STATE_DB).then(|| SnapshotPlan { // `SELECT *` is intentional compatibility machinery: the base @@ -91,7 +137,12 @@ impl AgentAdapter for CodexAdapter { // columns. `snapshot_session` inspects whichever optional columns // this version exposes without emitting schema-dependent SQL. sessions_sql: "SELECT * FROM threads ORDER BY created_at, id".to_owned(), - record_queries: Vec::new(), + // Preserve the native selection with the archive. It projects no + // rows, but keeps the authoritative pointer available offline. + record_queries: vec![rosetta_adapter_support::adapter::SnapshotQuery { + name: "thread".into(), + sql: "SELECT * FROM threads WHERE id = ?1".into(), + }], }) } @@ -184,7 +235,7 @@ impl AgentAdapter for CodexAdapter { /// opening record, and the file's mtime already gives the pipeline the /// session's `updated_at`. fn skim(&self, stream_id: &str, stat: &FileStat, head: &[u8], _tail: &[u8]) -> SkimResult { - if stream_id != spec::ROLLOUT { + if !spec::is_rollout(stream_id) { return SkimResult::default(); } @@ -234,6 +285,9 @@ impl AgentAdapter for CodexAdapter { } fn ingest_line(&self, ctx: &mut NormalizeCtx, stream_id: &str, line: &[u8]) -> LineOutcome { + if stream_id == spec::STATE_DB { + return LineOutcome::parsed(Vec::new()); + } // Owning the state for the call keeps it out of the borrow of `ctx`, // which every row needs mutably for its ordinal. One restore point, so // no projection path can lose it. diff --git a/crates/adapters/codex/src/lib.rs b/crates/adapters/codex/src/lib.rs index 3ac0b2a1..21c73a7f 100644 --- a/crates/adapters/codex/src/lib.rs +++ b/crates/adapters/codex/src/lib.rs @@ -22,6 +22,7 @@ pub mod normalize; pub mod raw; mod session_policy; pub mod spec; +mod storage; pub mod time; pub mod tool; mod usage; diff --git a/crates/adapters/codex/src/normalize.rs b/crates/adapters/codex/src/normalize.rs index d0e1dcc6..e8f7d10e 100644 --- a/crates/adapters/codex/src/normalize.rs +++ b/crates/adapters/codex/src/normalize.rs @@ -89,6 +89,12 @@ pub struct CodexState { /// configures the file. #[serde(default)] header_seen: bool, + #[serde(default)] + pub(crate) shared_history: bool, + #[serde(default)] + attached_thread: bool, + #[serde(default)] + pub(crate) origin_thread: Option, /// Current execution cwd used only to resolve relative tool arguments. /// The canonical session cwd is selected independently during meta ingest. pub(crate) tool_cwd: Option, @@ -191,6 +197,13 @@ pub fn ingest_line( return LineOutcome::parsed(Vec::new()); } + if state.shared_history + && state.attached_thread + && state.origin_thread != state.thread_id + && !matches!(record.payload, RolloutPayload::SessionMeta(_)) + { + return LineOutcome::parsed(Vec::new()); + } let mut ops = Vec::new(); // Permissive parsing means novel fields land in the flatten maps instead of // failing the line — that is the whole point, but it must still be visible. @@ -207,6 +220,14 @@ pub fn ingest_line( } let status = project(ctx, state, stream_id, &record, ts, &mut ops); + if state.shared_history && state.origin_thread != state.thread_id { + ops.retain(|op| { + !matches!( + op, + IngestOp::HtmlArtifact(_) | IngestOp::HtmlArtifactContent(_) + ) + }); + } LineOutcome { ops, status } } @@ -311,8 +332,17 @@ fn session_meta( ts: Option, ) { if state.header_seen { + if state.shared_history { + state.origin_thread = Some(meta.id.clone()); + } return; } + state.attached_thread = meta.attached_thread.unwrap_or(false); + state.shared_history = meta + .history_base + .as_ref() + .is_some_and(|base| !base.is_null()); + state.origin_thread = Some(meta.id.clone()); state.header_seen = true; ctx.agent_version = Some(meta.cli_version.clone()); state.thread_id = Some(meta.id.clone()); diff --git a/crates/adapters/codex/src/raw/rollout.rs b/crates/adapters/codex/src/raw/rollout.rs index 3e6ec37e..8b8d7741 100644 --- a/crates/adapters/codex/src/raw/rollout.rs +++ b/crates/adapters/codex/src/raw/rollout.rs @@ -225,6 +225,10 @@ pub struct SessionMeta { #[serde(skip_serializing_if = "Option::is_none")] pub history_base: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub attached_thread: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub forked_from_ordinal_exclusive: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dynamic_tools: Option, #[serde(skip_serializing_if = "Option::is_none")] pub subagent_history_start_ordinal: Option, diff --git a/crates/adapters/codex/src/spec.rs b/crates/adapters/codex/src/spec.rs index 94e9ed41..1c004371 100644 --- a/crates/adapters/codex/src/spec.rs +++ b/crates/adapters/codex/src/spec.rs @@ -81,6 +81,13 @@ struct FacetView { record_types: BTreeMap, } +pub fn is_rollout(id: &str) -> bool { + matches!( + id, + ROLLOUT | "rollout-compressed" | "archived-rollout" | "archived-rollout-compressed" + ) +} + #[cfg(test)] mod tests { use super::{ROLLOUT, STATE_DB, declared_event_subtypes, declared_response_items, doc}; diff --git a/crates/adapters/codex/src/storage.rs b/crates/adapters/codex/src/storage.rs new file mode 100644 index 00000000..8186a12e --- /dev/null +++ b/crates/adapters/codex/src/storage.rs @@ -0,0 +1,136 @@ +//! Rollout storage naming, pinned to rust-v0.167.2 in ground truth. +//! Pure descriptions only: filesystem and decoder work belong to capture. + +use rosetta_adapter_support::adapter::{ + InputCandidate, InputEncoding, InputSource, TranscriptLocation, +}; + +pub fn location(path: &str) -> Option { + let plain = path.strip_suffix(".zst").unwrap_or(path); + if !plain.ends_with(".jsonl") { + return None; + } + let name = plain.rsplit('/').next()?; + let rest = name.strip_prefix("rollout-")?; + let timestamp = rest.get(..19)?; + if !timestamp.is_ascii() { + return None; + } + if timestamp.as_bytes().get(4) != Some(&b'-') + || timestamp.as_bytes().get(7) != Some(&b'-') + || rest.as_bytes().get(19) != Some(&b'-') + { + return None; + } + let active = format!( + "sessions/{}/{}/{}/{}", + ×tamp[..4], + ×tamp[5..7], + ×tamp[8..10], + name + ); + let archived = format!("archived_sessions/{name}"); + if plain != active && plain != archived { + return None; + } + let candidates = [ + (&active, InputEncoding::Plain), + (&archived, InputEncoding::Plain), + (&active, InputEncoding::Zstd), + (&archived, InputEncoding::Zstd), + ] + .into_iter() + .map(|(path, encoding)| InputCandidate { + path: if encoding == InputEncoding::Zstd { + format!("{path}.zst") + } else { + path.clone() + }, + encoding, + }) + .collect(); + Some(TranscriptLocation { + logical_path: active, + source: InputSource::single(candidates), + }) +} + +pub fn header( + path: &str, + first: &[u8], +) -> Result, String> { + use rosetta_adapter_support::transcripts::{TranscriptHeader, TranscriptReference}; + let Some(location) = location(path) else { + return Ok(None); + }; + let text = std::str::from_utf8(first).map_err(|e| e.to_string())?; + let crate::raw::ParsedLine::Known(record) = + crate::raw::parse_line(text).map_err(|e| e.to_string())? + else { + return Ok(None); + }; + let crate::raw::rollout::RolloutPayload::SessionMeta(meta) = record.payload else { + return Ok(None); + }; + let filename = location + .logical_path + .rsplit('/') + .next() + .ok_or("missing rollout filename")?; + let ids = filename + .strip_prefix("rollout-") + .and_then(|s| s.get(20..)) + .and_then(|s| s.strip_suffix(".jsonl")) + .ok_or("invalid rollout filename")?; + let (thread, source_id) = ids.split_once('_').unwrap_or((ids, ids)); + if thread != meta.id { + return Err("rollout filename and header name different threads".into()); + } + let base = meta + .history_base + .as_ref() + .filter(|base| !base.is_null()) + .map(|base| { + Ok::<_, String>(TranscriptReference { + source_id: base + .get("thread_id") + .and_then(serde_json::Value::as_str) + .ok_or("missing history base rollout id")? + .to_owned(), + end_offset: base + .get("end_byte_offset") + .and_then(serde_json::Value::as_u64) + .ok_or("missing history base offset")?, + end_ordinal: base + .get("end_ordinal_exclusive") + .and_then(serde_json::Value::as_u64) + .ok_or("missing history base ordinal")?, + }) + }) + .transpose()?; + Ok(Some(TranscriptHeader { + source_id: source_id.to_owned(), + session_id: meta.id, + base, + start_ordinal: record.ordinal.unwrap_or(0), + paginated: meta.history_mode.as_deref() == Some("paginated"), + })) +} + +/// A copied/relocated CODEX_HOME can retain old absolute SQLite paths. Bind +/// only a recognized managed suffix inside the enrolled root; never read the +/// old absolute location or substitute a different rollout for this thread. +pub fn selection_path(path: &str) -> String { + if location(path).is_some() { + return path.to_owned(); + } + for marker in ["/sessions/", "/archived_sessions/"] { + if let Some(at) = path.rfind(marker) { + let relative = &path[at + 1..]; + if location(relative).is_some() { + return relative.to_owned(); + } + } + } + path.to_owned() +} diff --git a/crates/adapters/codex/src/usage.rs b/crates/adapters/codex/src/usage.rs index 558536ba..745dc632 100644 --- a/crates/adapters/codex/src/usage.rs +++ b/crates/adapters/codex/src/usage.rs @@ -54,6 +54,11 @@ pub(crate) fn token_count( return; }; let last = RawUsage::parse(&info["last_token_usage"]); + if state.shared_history && state.origin_thread != state.thread_id { + state.prev_total = Some(total); + state.last_record_usage = None; + return; + } if let Some(record) = state.last_record_usage.take() && last == Some(record) { @@ -509,6 +514,15 @@ impl ReplayGate { /// boundary, a fork source, a parent, or a spawn descriptor. A plain /// session written by any CLI is open from its first record (CODEX-8). pub(crate) fn for_session(meta: &SessionMeta, ts: Option) -> Self { + // Shared history retains original timestamps; the old copied-head + // heuristic does not apply. Native headers identify each range's owner. + if meta + .history_base + .as_ref() + .is_some_and(|base| !base.is_null()) + { + return Self::Open; + } let copied_history = meta.forked_from_id.is_some() || meta.parent_thread_id.is_some() || thread_spawn(&meta.source); diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index 71607ea2..6099d29d 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -17,6 +17,7 @@ pulldown-cmark.workspace = true percent-encoding.workspace = true [dev-dependencies] +zstd.workspace = true base64.workspace = true rosetta-media = { path = "../media" } rusqlite.workspace = true diff --git a/crates/pipeline/src/capture/pool/cut_tests.rs b/crates/pipeline/src/capture/pool/cut_tests.rs index a81b1c60..1c22350a 100644 --- a/crates/pipeline/src/capture/pool/cut_tests.rs +++ b/crates/pipeline/src/capture/pool/cut_tests.rs @@ -26,6 +26,7 @@ fn runtime() -> RuntimeRow { fn unit(index: u32, size: u64, cut_len: Option) -> UnitSpec { UnitSpec { file: SessionFile { + input: None, ledger_only: false, relative_path: format!("projects/-w-cut/s{index}.jsonl"), stream_id: "transcript".into(), diff --git a/crates/pipeline/src/capture/pool/tests.rs b/crates/pipeline/src/capture/pool/tests.rs index 38c6d0d3..e2f4fc8a 100644 --- a/crates/pipeline/src/capture/pool/tests.rs +++ b/crates/pipeline/src/capture/pool/tests.rs @@ -34,6 +34,7 @@ fn runtime_row() -> RuntimeRow { fn file(path: &str, size: u64) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.to_owned(), stream_id: "transcript".to_owned(), diff --git a/crates/pipeline/src/checkpoint.rs b/crates/pipeline/src/checkpoint.rs index fcda3fe9..dda0d625 100644 --- a/crates/pipeline/src/checkpoint.rs +++ b/crates/pipeline/src/checkpoint.rs @@ -38,8 +38,9 @@ pub struct PipeCheckpoint { pub cumulative: Option, /// Adapter-owned, versioned by `normalizer_version` and opaque to pipeline. pub adapter_state: Option, - /// Snapshot sources only: the db file-set stamp at the last successful - /// pass — equal stamps skip even the copy. Additive with a default so + /// Physical input stamp at the last successful pass: a DB file set or + /// a representation-independent transcript's source set. Equal stamps + /// skip copying/decoding quiet inputs. Additive with a default so /// pre-existing blobs still decode. #[serde(default)] pub snapshot_stamp: Option, diff --git a/crates/pipeline/src/closed/tests.rs b/crates/pipeline/src/closed/tests.rs index e18452e3..16b41fc7 100644 --- a/crates/pipeline/src/closed/tests.rs +++ b/crates/pipeline/src/closed/tests.rs @@ -404,6 +404,7 @@ fn eager_cursors_and_roster_keys_match_the_lazy_dense_walk_across_a_restart_and_ #[test] fn close_file_skips_an_unread_source_and_refuses_what_only_a_bug_leaves() { let file = SessionFile { + input: None, ledger_only: false, relative_path: "projects/-w/close.jsonl".into(), stream_id: "transcript".into(), @@ -514,6 +515,7 @@ fn close_file_skips_an_unread_source_and_refuses_what_only_a_bug_leaves() { #[test] fn a_frontier_that_never_closed_names_its_lane() { let file = |path: &str| SessionFile { + input: None, ledger_only: false, relative_path: path.into(), stream_id: "transcript".into(), diff --git a/crates/pipeline/src/closed/trees/tests.rs b/crates/pipeline/src/closed/trees/tests.rs index 2fc992a7..3639139b 100644 --- a/crates/pipeline/src/closed/trees/tests.rs +++ b/crates/pipeline/src/closed/trees/tests.rs @@ -8,6 +8,7 @@ use super::*; /// A transcript of `session` with census size and time, and no cut. fn transcript(path: &str, session: &str, size: u64, modified_ms: i64) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.to_owned(), stream_id: "transcript".to_owned(), @@ -29,6 +30,7 @@ fn transcript(path: &str, session: &str, size: u64, modified_ms: i64) -> Session /// A transcript as the skim leaves it: census facts plus the skim's cut. fn skimmed(path: &str, session: &str, census: (u64, i64), cut: (u64, i64)) -> SessionFile { SessionFile { + input: None, cut_len: Some(cut.0), cut_modified_ms: Some(cut.1), ..transcript(path, session, census.0, census.1) @@ -38,6 +40,7 @@ fn skimmed(path: &str, session: &str, census: (u64, i64), cut: (u64, i64)) -> Se /// A ledger-only internal source whose metadata names `parent`. fn guardian(path: &str, session: &str, parent: &str) -> SessionFile { SessionFile { + input: None, ledger_only: true, tree_parent: Some(parent.to_owned()), ..transcript(path, session, 1, 1) @@ -47,6 +50,7 @@ fn guardian(path: &str, session: &str, parent: &str) -> SessionFile { /// A virtual member `#` of a snapshot database. fn member(physical: &str, session: &str, size: u64, modified_ms: i64) -> SessionFile { SessionFile { + input: None, stream_id: "state-db".to_owned(), file_format: FileFormat::Sqlite, discipline: FileDiscipline::Database, @@ -58,6 +62,7 @@ fn member(physical: &str, session: &str, size: u64, modified_ms: i64) -> Session /// A virtual member `#` of a sidecar document. fn document(physical: &str, session: &str, size: u64, modified_ms: i64) -> SessionFile { SessionFile { + input: None, stream_id: "subagent-metadata".to_owned(), file_format: FileFormat::Json, discipline: FileDiscipline::RewriteInPlace, diff --git a/crates/pipeline/src/closed/walk_order.rs b/crates/pipeline/src/closed/walk_order.rs index 2e0c2e8f..bd806678 100644 --- a/crates/pipeline/src/closed/walk_order.rs +++ b/crates/pipeline/src/closed/walk_order.rs @@ -221,7 +221,7 @@ impl WalkPlan<'_> { } /// [`walk_plan`] for `adapter`'s roster: a snapshot member is hoistable only -/// when its stream reads no per-session record queries. +/// when the adapter promises its records emit no canonical operations. pub(crate) fn walk_plan_for<'a>( adapter: &dyn AgentAdapter, files: &'a [SessionFile], @@ -230,9 +230,7 @@ pub(crate) fn walk_plan_for<'a>( ) -> WalkPlan<'a> { plan(files, trees, opts, &|file: &SessionFile| { file.driver == StreamDriver::SqliteSnapshot - && adapter - .snapshot_plan(&file.stream_id) - .is_some_and(|plan| plan.record_queries.is_empty()) + && adapter.snapshot_is_metadata_only(&file.stream_id) }) } diff --git a/crates/pipeline/src/closed/walk_order/tests.rs b/crates/pipeline/src/closed/walk_order/tests.rs index 342f19a2..9aba8d81 100644 --- a/crates/pipeline/src/closed/walk_order/tests.rs +++ b/crates/pipeline/src/closed/walk_order/tests.rs @@ -42,6 +42,7 @@ impl Rng { fn transcript(path: &str, session: &str, size: u64, modified_ms: i64) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.to_owned(), stream_id: "transcript".to_owned(), @@ -70,6 +71,7 @@ fn member(physical: &str, session: &str, driver: StreamDriver, modified_ms: i64) ), }; SessionFile { + input: None, stream_id: stream.to_owned(), file_format: format, discipline, diff --git a/crates/pipeline/src/discover.rs b/crates/pipeline/src/discover.rs index 7596524e..77d373da 100644 --- a/crates/pipeline/src/discover.rs +++ b/crates/pipeline/src/discover.rs @@ -155,16 +155,24 @@ pub fn discover_root( let mut newest_modified_ms = 0; let session_streams = adapter.census_session_streams(); let mut session_count = session_streams.map(|_| 0); + let mut logical_streams = std::collections::BTreeSet::new(); + let mut logical_sessions = std::collections::BTreeSet::new(); for file in &report.files { file_count += 1; total_bytes += file.size; newest_modified_ms = newest_modified_ms.max(file.modified_ms); if let Some(stream) = inventory_stream(layout, file) { - if adapter.stream_driver(&stream.id).is_some() { + let logical = adapter + .transcript_location(&file.relative_path) + .map_or_else(|| file.relative_path.clone(), |l| l.logical_path); + if adapter.stream_driver(&stream.id).is_some() + && logical_streams.insert(logical.clone()) + { streamable_count += 1; streamable_bytes += file.size; } if session_streams.is_some_and(|streams| streams.contains(&stream.id.as_str())) + && logical_sessions.insert(logical) && let Some(count) = &mut session_count { *count += 1; diff --git a/crates/pipeline/src/early_tests.rs b/crates/pipeline/src/early_tests.rs index d272c793..52dba565 100644 --- a/crates/pipeline/src/early_tests.rs +++ b/crates/pipeline/src/early_tests.rs @@ -11,6 +11,7 @@ use crate::live::PollEvidence; fn file(path: &str, session_id: &str) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.into(), stream_id: "transcript".into(), @@ -258,6 +259,7 @@ fn without_early_tools_every_fact_stays_in_its_batch() { /// thread never becomes a canonical session. fn ledger_only_file(path: &str, session_id: &str) -> SessionFile { SessionFile { + input: None, ledger_only: true, ..file(path, session_id) } diff --git a/crates/pipeline/src/early_trees_tests.rs b/crates/pipeline/src/early_trees_tests.rs index b019469f..55a8c204 100644 --- a/crates/pipeline/src/early_trees_tests.rs +++ b/crates/pipeline/src/early_trees_tests.rs @@ -16,6 +16,7 @@ use crate::live::PollEvidence; fn file(path: &str, session_id: &str) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.into(), stream_id: "transcript".into(), diff --git a/crates/pipeline/src/lib.rs b/crates/pipeline/src/lib.rs index 43a3c73d..53c02b41 100644 --- a/crates/pipeline/src/lib.rs +++ b/crates/pipeline/src/lib.rs @@ -30,6 +30,8 @@ pub use html_artifacts::ArtifactRecovery; pub mod instrument; pub mod live; pub mod meta; +mod meta_journal; +pub use meta_journal::LedgerMetadata; pub mod plan; pub mod progress; mod project; @@ -83,9 +85,9 @@ pub use streamed::{ /// Version of the canonical-op envelope stored in ledger meta frames. Replay /// routes explicitly by this value instead of assuming today's row JSON. -/// Meta payload v2 makes the adapter-selected primary cwd explicit on the -/// session and removes streamed workspace observations from the projection. -pub const LEDGER_META_SCHEMA_VERSION: u32 = 2; +/// Metadata v3 journals required-source readiness beside canonical metadata. +/// Readers retain support for the legacy operation arrays (v0 and v2). +pub const LEDGER_META_SCHEMA_VERSION: u32 = 3; /// The largest record the pipeline normalizes. A longer JSONL record is still /// captured and ledgered whole, up to what one raw ledger frame holds diff --git a/crates/pipeline/src/live.rs b/crates/pipeline/src/live.rs index 538b3514..9d581bcf 100644 --- a/crates/pipeline/src/live.rs +++ b/crates/pipeline/src/live.rs @@ -101,7 +101,8 @@ pub fn map_hint( let nominated: Vec = files .iter() .filter(|file| { - file.relative_path == hinted + file.input.as_ref().is_some_and(|input| input.claims(hinted)) + || file.relative_path == hinted // macOS FSEvents commonly reports only the containing // directory. An ancestor hint nominates its admitted children; // their cursor anchors still decide whether any source work is @@ -113,7 +114,14 @@ pub fn map_hint( .collect(); if !nominated.is_empty() { return HintMapping { - needs_admission: snapshot.is_some(), + needs_admission: snapshot.is_some() + || files.iter().any(|file| { + nominated.contains(&file.relative_path) + && file + .input + .as_ref() + .is_some_and(|input| input.segments.is_empty()) + }), nominated, }; } @@ -175,13 +183,22 @@ pub struct PollEvidence { normalizer_version: Option, /// `None` before the first framed batch, and without a usable checkpoint. observed_len: Option, + input_stamp: Option<[u8; 32]>, } impl PollEvidence { + fn stamp(text: Option<&str>) -> Option<[u8; 32]> { + text.and_then(|text| blake3::Hash::from_hex(text).ok()) + .map(|hash| *hash.as_bytes()) + } + /// The evidence of one cursor checkpoint as [`PipeCheckpoint::decode`] /// yields it; `None` is an absent or expired checkpoint. pub fn of(checkpoint: Option<&PipeCheckpoint>) -> Self { Self { + input_stamp: Self::stamp( + checkpoint.and_then(|checkpoint| checkpoint.snapshot_stamp.as_deref()), + ), normalizer_version: checkpoint.map(|checkpoint| checkpoint.normalizer_version), observed_len: checkpoint .and_then(PipeCheckpoint::tail_checkpoint) @@ -484,10 +501,32 @@ impl<'a> LiveRunner<'a> { &mut self, store: &mut Store, mut changed: BTreeSet, - needs_admission: bool, + mut needs_admission: bool, audit: bool, instrument: &Instrument, ) -> Result<(), PipelineError> { + if !needs_admission { + for file in self + .files + .iter() + .filter(|file| changed.contains(&file.relative_path)) + { + if let Some(input) = &file.input { + match input.headers_match(&self.root) { + Ok(true) => {} + Ok(false) => { + needs_admission = true; + break; + } + Err(error) if error.incomplete_input() => { + needs_admission = true; + break; + } + Err(error) => return Err(error.into()), + } + } + } + } if needs_admission { instrument.time(Metric::LiveAdmitNanos, || { self.admit(store, &mut changed, instrument) @@ -679,6 +718,14 @@ impl<'a> LiveRunner<'a> { |path| layout.matching_stream(path).is_some_and(driven), ) })?; + let mut walk = walk; + let mut seen = BTreeSet::new(); + walk.files.retain_mut(|file| { + if let Some(location) = self.adapter.transcript_location(&file.relative_path) { + file.relative_path = location.logical_path; + } + seen.insert(file.relative_path.clone()) + }); instrument.add(Metric::LivePollFiles, walk.files.len() as u64); // Index once per poll instead of scanning the whole admitted roster // for every physical file. Preserve the old first-match behavior if @@ -696,7 +743,17 @@ impl<'a> LiveRunner<'a> { })?; let known = &mut self.files[index]; let touched = known.modified_ms != walked.modified_ms; - if evidence.behind(normalizer_version, walked.size) || touched { + let behind = if let Some(input) = &known.input { + if input.segments.is_empty() && (touched || known.size != walked.size) { + *needs_admission = true; + } + evidence.normalizer_version != Some(normalizer_version) + || evidence.input_stamp + != PollEvidence::stamp(input.stamp(&self.root)?.as_deref()) + } else { + evidence.behind(normalizer_version, walked.size) || touched + }; + if behind { known.size = walked.size; known.modified_ms = walked.modified_ms; changed.insert(known.relative_path.clone()); @@ -783,7 +840,7 @@ impl<'a> LiveRunner<'a> { let known = self .files .iter() - .any(|old| old.relative_path == file.relative_path); + .any(|old| old.relative_path == file.relative_path && old.input == file.input); if !known { changed.insert(file.relative_path.clone()); } diff --git a/crates/pipeline/src/meta.rs b/crates/pipeline/src/meta.rs index 115d8aee..077eb15a 100644 --- a/crates/pipeline/src/meta.rs +++ b/crates/pipeline/src/meta.rs @@ -22,11 +22,14 @@ mod exclusions; use exclusions::CwdAdmission; mod inventory; mod recheck; +mod transcripts; pub use recheck::{AdmissionCheck, AdmissionSources}; /// One source file the stream phase will tail, with the ids minted here. #[derive(Debug, Clone)] pub struct SessionFile { + /// RAM-only decoded source plan. Raw records remain self-contained in the ledger. + pub input: Option, /// Capture and checkpoint this source without admitting a public session. pub ledger_only: bool, pub relative_path: String, @@ -50,8 +53,8 @@ pub struct SessionFile { /// root-tree build reads it up to; `skim::Cut` says what has none. pub cut_len: Option, /// RAM only. The mtime (epoch ms) at that fstat, which root trees rank - /// by; `modified_ms` is the older census's. `None` exactly when - /// `cut_len` is. + /// by; `modified_ms` is the older census's. An encoded or composed input + /// can have this stamp without a cheaply known decoded `cut_len`. pub cut_modified_ms: Option, } @@ -98,6 +101,7 @@ pub struct MetaReport { #[derive(Debug, Default)] struct MetaOutput { + selections: BTreeMap, projects: Vec, /// First-merge order. Grows only through [`MetaOutput::merge_session`], /// which keeps `session_positions` in step. @@ -199,14 +203,16 @@ pub fn meta_ingest_inventory_into( runtime.discovered_at = 0; } } - let semantic = serde_json::to_vec(&semantic_ops).map_err(|error| { - PipelineError::Config(format!("serializing semantic meta key: {error}")) - })?; + let semantic = serde_json::to_vec(&crate::LedgerMetadata::new(semantic_ops, &files)) + .map_err(|error| { + PipelineError::Config(format!("serializing semantic meta key: {error}")) + })?; let semantic_key = blake3::hash(&semantic).to_hex().to_string(); instrument.ledger_write(ledger, |ledger| -> Result<(), PipelineError> { - let json = serde_json::to_string(&ops).map_err(|error| { - PipelineError::Config(format!("serializing meta ops for the ledger: {error}")) - })?; + let json = serde_json::to_string(&crate::LedgerMetadata::new(ops.clone(), &files)) + .map_err(|error| { + PipelineError::Config(format!("serializing meta ops for the ledger: {error}")) + })?; ledger.append_meta_checkpoint_if_changed( crate::LEDGER_META_SCHEMA_VERSION, &json, @@ -256,11 +262,22 @@ pub(crate) fn plan_inventory( }; report.walk_truncated = walk_truncated; + let mut seen = BTreeSet::new(); + let inventory: Vec<_> = inventory + .iter() + .filter(|file| { + let key = adapter + .transcript_location(&file.relative_path) + .map_or_else(|| file.relative_path.clone(), |l| l.logical_path); + seen.insert(key) + }) + .cloned() + .collect(); inventory::read_inventory( adapter, runtime, root, - inventory, + &inventory, opts, instrument, threads, @@ -268,6 +285,17 @@ pub(crate) fn plan_inventory( &mut output, )?; + transcripts::resolve( + adapter, + runtime, + root, + store, + &inventory, + opts, + &mut report, + &mut output, + )?; + // A snapshot source that disappears whole keeps every member's last // projected generation, like a deleted JSONL transcript keeps its rows; // its virtual cursors simply see no further activity (product decision, @@ -680,6 +708,7 @@ fn admit_identified( .map(|native| ids::session_id(&runtime.id, native)) .filter(|parent| parent != &session_id); output.files.push(SessionFile { + input: None, ledger_only: true, relative_path: source.relative_path, stream_id: source.stream_id, @@ -693,7 +722,7 @@ fn admit_identified( agent_version: skim.agent_version, source_missing: false, tree_parent, - cut_len: source.cut.map(|cut| cut.len), + cut_len: source.cut.and_then(|cut| cut.len), cut_modified_ms: source.cut.map(|cut| cut.modified_ms), }); return; @@ -826,6 +855,7 @@ fn admit_identified( output.merge_session(row); output.files.push(SessionFile { + input: None, ledger_only: false, relative_path: source.relative_path, stream_id: source.stream_id, @@ -839,7 +869,7 @@ fn admit_identified( agent_version: skim.agent_version, source_missing: false, tree_parent: None, - cut_len: source.cut.map(|cut| cut.len), + cut_len: source.cut.and_then(|cut| cut.len), cut_modified_ms: source.cut.map(|cut| cut.modified_ms), }); if output.admission.is_empty() { @@ -869,6 +899,7 @@ fn canonical_project_hint(runtime_id: &str, hint: ProjectHint) -> Option#`); an unidentified skim is only counted. #[allow(clippy::too_many_arguments)] fn admit_skim( + adapter: &dyn AgentAdapter, runtime: &RuntimeRow, report: &mut MetaReport, output: &mut MetaOutput, @@ -900,6 +931,12 @@ fn admit_skim( } else { file.relative_path.clone() }; + let location = (!member) + .then(|| adapter.transcript_location(&relative_path)) + .flatten(); + let relative_path = location + .as_ref() + .map_or(relative_path, |l| l.logical_path.clone()); admit_identified( runtime, output, @@ -917,87 +954,15 @@ fn admit_skim( native_session_id, instrument, ); -} - -/// Meta admission for a `SqliteSnapshot` stream: enumerate the database's -/// sessions via the adapter's declared plan and admit each as a VIRTUAL -/// source (`#`) — its own cursor, its own generation -/// swaps, its own ledger entries, exactly like a file. -#[allow(clippy::too_many_arguments)] -fn admit_database( - adapter: &dyn AgentAdapter, - runtime: &RuntimeRow, - root: &Path, - stream: &rosetta_adapter_support::StreamSpec, - driver: StreamDriver, - db_relative: &str, - size: u64, - modified_ms: i64, - report: &mut MetaReport, - output: &mut MetaOutput, - instrument: &Instrument, -) -> Result<(), PipelineError> { - let Some(plan) = adapter.snapshot_plan(&stream.id) else { - return Err(PipelineError::Config(format!( - "{} drives stream {} as SqliteSnapshot but declares no snapshot plan", - adapter.kind(), - stream.id - ))); - }; - let outcome = instrument.time(Metric::SkimNanos, || { - rosetta_capture::snapshot_db(&root.join(db_relative)) - })?; - let snapshot = match outcome { - rosetta_capture::SnapshotOutcome::Ready(snapshot) => snapshot, - // A racing writer or vanished db is the stream phase's problem. - rosetta_capture::SnapshotOutcome::Missing | rosetta_capture::SnapshotOutcome::Unstable => { - report.files_vanished += 1; - instrument.add(Metric::FilesVanished, 1); - return Ok(()); + if let Some(location) = location + && let Some(file) = output.files.last_mut() + { + file.input = Some(location.source); + // Alternate physical encodings use the logical stream's normalizer. + if let Some(stream) = adapter.layout().matching_stream(&file.relative_path) { + file.stream_id = stream.id.clone(); } - }; - report.files_skimmed += 1; - instrument.add(Metric::FilesSkimmed, 1); - - for row in snapshot.query_rows(&plan.sessions_sql, None)? { - let value: serde_json::Value = serde_json::from_str(&row).map_err(|error| { - PipelineError::Config(format!("sessions_sql row is not an object: {error}")) - })?; - let skim = adapter.snapshot_session(&stream.id, &value); - let Some(native) = skim.session_id.clone() else { - if output.admission.excludes_cwd(skim.cwd.as_deref()) { - continue; - } - report.files_unidentified += 1; - instrument.add(Metric::FilesUnidentified, 1); - continue; - }; - let updated = skim.updated_at.unwrap_or(modified_ms); - admit_identified( - runtime, - output, - AdmitSource { - relative_path: format!("{db_relative}#{native}"), - stream_id: stream.id.clone(), - file_format: stream.file_format, - discipline: stream.discipline, - driver, - size, - modified_ms: updated, - cut: None, - }, - skim, - native, - instrument, - ); } - - // A member no longer returned by sessions_sql keeps its last projected - // generation, exactly as a deleted JSONL transcript keeps its rows: an - // agent pruning its own store (Hermes 0.21.4 retains 90 days by default) - // must not erase Rosetta's history (product decision, 2026-09-24). The - // cursor simply sees no further activity. - Ok(()) } #[cfg(test)] diff --git a/crates/pipeline/src/meta/inventory.rs b/crates/pipeline/src/meta/inventory.rs index a06790bd..8e6e8bb4 100644 --- a/crates/pipeline/src/meta/inventory.rs +++ b/crates/pipeline/src/meta/inventory.rs @@ -163,6 +163,7 @@ pub(super) fn read_inventory( crate::skim::Skimmed::Transcript(skim, cut) => { report.files_skimmed += 1; admit_skim( + adapter, runtime, report, output, @@ -185,8 +186,8 @@ pub(super) fn read_inventory( report.files_skimmed += 1; for skim in skims { admit_skim( - runtime, report, output, file, stream, driver, skim, true, None, - instrument, + adapter, runtime, report, output, file, stream, driver, skim, true, + None, instrument, ); } } @@ -198,3 +199,87 @@ pub(super) fn read_inventory( Ok(()) } + +/// Meta admission for a `SqliteSnapshot` stream: enumerate the database's +/// sessions via the adapter's declared plan and admit each as a VIRTUAL +/// source (`#`) — its own cursor, its own generation +/// swaps, its own ledger entries, exactly like a file. +#[allow(clippy::too_many_arguments)] +fn admit_database( + adapter: &dyn AgentAdapter, + runtime: &RuntimeRow, + root: &Path, + stream: &rosetta_adapter_support::StreamSpec, + driver: StreamDriver, + db_relative: &str, + size: u64, + modified_ms: i64, + report: &mut MetaReport, + output: &mut MetaOutput, + instrument: &Instrument, +) -> Result<(), PipelineError> { + let Some(plan) = adapter.snapshot_plan(&stream.id) else { + return Err(PipelineError::Config(format!( + "{} drives stream {} as SqliteSnapshot but declares no snapshot plan", + adapter.kind(), + stream.id + ))); + }; + let outcome = instrument.time(Metric::SkimNanos, || { + rosetta_capture::snapshot_db(&root.join(db_relative)) + })?; + let snapshot = match outcome { + rosetta_capture::SnapshotOutcome::Ready(snapshot) => snapshot, + // A racing writer or vanished db is the stream phase's problem. + rosetta_capture::SnapshotOutcome::Missing | rosetta_capture::SnapshotOutcome::Unstable => { + report.files_vanished += 1; + instrument.add(Metric::FilesVanished, 1); + return Ok(()); + } + }; + report.files_skimmed += 1; + instrument.add(Metric::FilesSkimmed, 1); + + for row in snapshot.query_rows(&plan.sessions_sql, None)? { + let value: serde_json::Value = serde_json::from_str(&row).map_err(|error| { + PipelineError::Config(format!("sessions_sql row is not an object: {error}")) + })?; + let skim = adapter.snapshot_session(&stream.id, &value); + let Some(native) = skim.session_id.clone() else { + if output.admission.excludes_cwd(skim.cwd.as_deref()) { + continue; + } + report.files_unidentified += 1; + instrument.add(Metric::FilesUnidentified, 1); + continue; + }; + if let Some(selection) = adapter.transcript_selection(&stream.id, &value) { + output.selections.insert(native.clone(), selection); + } + let updated = skim.updated_at.unwrap_or(modified_ms); + admit_identified( + runtime, + output, + AdmitSource { + relative_path: format!("{db_relative}#{native}"), + stream_id: stream.id.clone(), + file_format: stream.file_format, + discipline: stream.discipline, + driver, + size, + modified_ms: updated, + cut: None, + }, + skim, + native, + instrument, + ); + } + + // A member no longer returned by sessions_sql keeps its last projected + // generation, exactly as a deleted JSONL transcript keeps its rows: an + // agent pruning its own store (Hermes 0.21.4 retains 90 days by default) + // must not erase Rosetta's history (product decision, 2026-09-24). The + // cursor simply sees no further activity. + Ok(()) +} diff --git a/crates/pipeline/src/meta/transcripts.rs b/crates/pipeline/src/meta/transcripts.rs new file mode 100644 index 00000000..7f33d41d --- /dev/null +++ b/crates/pipeline/src/meta/transcripts.rs @@ -0,0 +1,299 @@ +//! Plan logical transcripts from bounded native headers and DB selectors. +//! Plans are transient. Their selected header and inherited raw records are +//! captured together in one source generation, making offline replay closed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use rosetta_adapter_support::AgentAdapter; +use rosetta_adapter_support::transcripts::TranscriptHeader; +use rosetta_capture::input::{InputSegment, InputSource}; +use rosetta_core::rows::RuntimeRow; +use rosetta_store::{Store, queries}; + +use super::{MetaOutput, MetaReport, SessionFile}; +use crate::{IngestOptions, PipelineError}; + +struct Node { + file: SessionFile, + header: TranscriptHeader, + header_len: u64, + header_hash: [u8; 32], +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn resolve( + adapter: &dyn AgentAdapter, + runtime: &RuntimeRow, + root: &Path, + store: &Store, + inventory: &[rosetta_capture::ScannedFile], + opts: &IngestOptions, + report: &mut MetaReport, + output: &mut MetaOutput, +) -> Result<(), PipelineError> { + let candidates: BTreeMap<_, _> = inventory + .iter() + .filter_map(|file| { + adapter + .transcript_location(&file.relative_path) + .map(|l| (l.logical_path, l.source)) + }) + .collect(); + if candidates.is_empty() && output.selections.is_empty() { + return Ok(()); + } + let cursors = queries::cursor_sources_with_prefix(store.conn(), &runtime.id, "")?; + for (path, stream, session) in &cursors { + if let Some(input) = candidates.get(path) + && !output.files.iter().any(|file| file.relative_path == *path) + && let Some(row) = queries::session(store.conn(), session)? + { + let facts = inventory.iter().find(|file| { + adapter + .transcript_location(&file.relative_path) + .is_some_and(|location| location.logical_path == *path) + }); + output.files.push(SessionFile { + input: Some(input.clone()), + ledger_only: false, + relative_path: path.clone(), + stream_id: stream.clone(), + file_format: rosetta_adapter_support::FileFormat::Jsonl, + discipline: rosetta_adapter_support::FileDiscipline::AppendOnly, + driver: rosetta_adapter_support::StreamDriver::JsonLines, + session_id: session.clone(), + native_session_id: row.native_session_id, + size: facts.map_or(0, |file| file.size), + modified_ms: facts.map_or(row.updated_at, |file| file.modified_ms), + agent_version: None, + source_missing: false, + tree_parent: None, + cut_len: None, + cut_modified_ms: None, + }); + } + } + let mut nodes = BTreeMap::new(); + let mut groups: BTreeMap> = BTreeMap::new(); + for file in &output.files { + opts.check_cancelled()?; + let Some(input) = &file.input else { + continue; + }; + let head = match input.head(root, crate::MAX_PROJECTED_RECORD_BYTES) { + Ok(Some(head)) => head, + Ok(None) => { + groups.entry(file.native_session_id.clone()).or_default(); + continue; + } + Err(error) if error.incomplete_input() => { + groups.entry(file.native_session_id.clone()).or_default(); + continue; + } + Err(error) => return Err(error.into()), + }; + let header = match adapter.transcript_header(&file.relative_path, &head.bytes) { + Ok(Some(header)) => header, + Ok(None) => continue, + Err(_) => { + groups.entry(file.native_session_id.clone()).or_default(); + report.files_unidentified += 1; + continue; + } + }; + groups + .entry(header.session_id.clone()) + .or_default() + .push(header.source_id.clone()); + if nodes + .insert( + header.source_id.clone(), + Node { + file: file.clone(), + header, + header_len: head.bytes.len() as u64, + header_hash: *blake3::hash(&head.bytes).as_bytes(), + }, + ) + .is_some() + { + return Err(PipelineError::Config( + "ambiguous immutable transcript identity".into(), + )); + } + } + // Reuse an older plain-path cursor, including after its native file was + // archived or removed. This keeps upgrades and later reverts in one + // logical source rather than merging abandoned branches into a session. + let mut retained = Vec::new(); + let mut handled = BTreeSet::new(); + for (native, selection) in &output.selections { + groups.entry(native.clone()).or_default(); + let _ = selection; + } + for (native, ids) in &groups { + opts.check_cancelled()?; + let selected = if let Some(selection) = output.selections.get(native) { + let path = Path::new(&selection.path); + let relative = if path.is_absolute() { + path.strip_prefix(root).ok() + } else { + Some(path) + }; + relative + .and_then(Path::to_str) + .and_then(|path| adapter.transcript_location(path)) + .and_then(|location| { + ids.iter().find_map(|id| { + let node = &nodes[id]; + (node.file.relative_path == location.logical_path).then_some(node) + }) + }) + } else if ids.len() == 1 { + nodes.get(&ids[0]) + } else { + // Multiple branches without an authoritative selector are + // ambiguous. Never guess "newest" and expose discarded history. + None + }; + let template = selected + .map(|n| &n.file) + .or_else(|| ids.first().map(|id| &nodes[id].file)) + .or_else(|| output.files.iter().find(|f| f.native_session_id == *native)); + let Some(template) = template else { + continue; + }; + let mut file = template.clone(); + handled.insert(native.clone()); + file.driver = rosetta_adapter_support::StreamDriver::JsonLines; + file.file_format = rosetta_adapter_support::FileFormat::Jsonl; + file.discipline = rosetta_adapter_support::FileDiscipline::AppendOnly; + // A missing selected source still contributes an incomplete transcript + // obligation, so its metadata-only DB member cannot mark it Indexed. + if file.input.is_none() + && let Some(selection) = output.selections.get(native) + { + let path = Path::new(&selection.path); + if let Some(location) = path + .strip_prefix(root) + .ok() + .or(Some(path)) + .and_then(Path::to_str) + .and_then(|p| adapter.transcript_location(p)) + { + file.relative_path = location.logical_path; + if let Some(stream) = adapter.layout().matching_stream(&file.relative_path) { + file.stream_id = stream.id.clone(); + } + } else { + file.relative_path = format!("missing-transcript/{native}"); + } + } + let existing = cursors.iter().find(|(path, _, session)| { + session == &file.session_id && adapter.transcript_location(path).is_some() + }); + file.relative_path = existing + .map(|(path, _, _)| path.clone()) + .unwrap_or_else(|| { + ids.iter() + .map(|id| &nodes[id].file.relative_path) + .min() + .cloned() + .unwrap_or_else(|| file.relative_path.clone()) + }); + file.input = Some(match selected { + Some(node) => compose(node, &nodes, root).unwrap_or_else(|_| InputSource { + prefix: Vec::new(), + segments: vec![], + }), + None => InputSource { + prefix: Vec::new(), + segments: vec![], + }, + }); + if selected.is_some_and(|node| node.header.base.is_some()) { + // This is a new virtual byte stream; its ancestor cuts are + // explicit and its own file is pinned when capture starts. + file.cut_len = None; + } + retained.push(file); + } + output + .files + .retain(|file| file.input.is_none() || !handled.contains(&file.native_session_id)); + output.files.extend(retained); + Ok(()) +} + +fn segment(node: &Node, start: u64, end: Option) -> InputSegment { + let mut segment = node.file.input.as_ref().expect("decoded source").segments[0].clone(); + segment.start = start; + segment.end = end; + segment.header_hash = Some(node.header_hash); + segment +} + +fn compose( + selected: &Node, + nodes: &BTreeMap, + root: &Path, +) -> Result { + if selected.header.base.is_none() { + return Ok(InputSource { + prefix: Vec::new(), + segments: vec![segment(selected, 0, None)], + }); + } + let mut seen = BTreeSet::new(); + let mut chain = Vec::new(); + let mut node = selected; + let mut end = None; + let mut end_record_index = None; + loop { + if chain.len() >= 128 + || !seen.insert(node.header.source_id.clone()) + || !node.header.paginated + { + return Err(()); + } + let mut part = segment(node, 0, end); + part.end_record_index = end_record_index; + chain.push(part); + let Some(base) = &node.header.base else { + break; + }; + if base.end_ordinal == 0 + || base.end_offset == 0 + || node.header.start_ordinal != base.end_ordinal + { + return Err(()); + } + node = nodes.get(&base.source_id).ok_or(())?; + if base.end_offset < node.header_len || base.end_ordinal <= node.header.start_ordinal { + return Err(()); + } + end = Some(base.end_offset); + end_record_index = Some(base.end_ordinal); + } + chain.reverse(); + // The selected header configures the normalizer first. Each original + // header then marks the owner of its following records. These are native + // bytes, not a synthetic protocol; the ledger is self-contained. + let prefix = selected + .file + .input + .as_ref() + .ok_or(())? + .head(root, crate::MAX_PROJECTED_RECORD_BYTES) + .map_err(|_| ())? + .ok_or(())? + .bytes; + if *blake3::hash(&prefix).as_bytes() != selected.header_hash { + return Err(()); + } + Ok(InputSource { + prefix, + segments: chain, + }) +} diff --git a/crates/pipeline/src/meta_journal.rs b/crates/pipeline/src/meta_journal.rs new file mode 100644 index 00000000..9d5c05d9 --- /dev/null +++ b/crates/pipeline/src/meta_journal.rs @@ -0,0 +1,67 @@ +//! Versioned metadata journal. Required source identifiers preserve an +//! incomplete transcript's readiness across native-file deletion and rebuild. +use crate::{PipelineError, SessionFile}; +use rosetta_core::IngestOp; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SourceRequirement { + pub path: String, + pub session_id: String, + pub available: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct LedgerMetadata { + pub ops: Vec, + pub required_sources: Vec, +} +impl LedgerMetadata { + pub fn decode(payload: &str) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Compatible { + Legacy(Vec), + Current(LedgerMetadata), + } + match serde_json::from_str(payload) + .map_err(|e| PipelineError::Config(format!("undecodable meta batch in ledger: {e}")))? + { + Compatible::Legacy(ops) => Ok(Self { + ops, + required_sources: vec![], + }), + Compatible::Current(meta) => Ok(meta), + } + } + pub(crate) fn new(ops: Vec, files: &[SessionFile]) -> Self { + let required_sources = files + .iter() + .filter(|f| !f.ledger_only) + .filter_map(|file| { + file.input.as_ref().map(|input| SourceRequirement { + path: file.relative_path.clone(), + session_id: file.session_id.clone(), + available: !input.segments.is_empty(), + }) + }) + .collect(); + Self { + ops, + required_sources, + } + } +} + +pub(crate) fn apply_requirements( + requirements: &[SourceRequirement], + present: &BTreeSet, + health: &mut BTreeMap, +) { + for required in requirements { + if !required.available || !present.contains(&required.path) { + health.insert(required.session_id.clone(), false); + } + } +} diff --git a/crates/pipeline/src/plan.rs b/crates/pipeline/src/plan.rs index ce06b11c..eccf19a9 100644 --- a/crates/pipeline/src/plan.rs +++ b/crates/pipeline/src/plan.rs @@ -162,6 +162,7 @@ mod tests { fn file(path: &str, session_id: &str, size: u64) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.into(), stream_id: "transcript".into(), diff --git a/crates/pipeline/src/replay.rs b/crates/pipeline/src/replay.rs index 5913c0ec..1d4bf620 100644 --- a/crates/pipeline/src/replay.rs +++ b/crates/pipeline/src/replay.rs @@ -107,6 +107,7 @@ struct ElectionCache { only_dirty: HashSet, unsealed: BTreeMap, admission: ReplayAdmission, + required_sources: Vec, } struct ElectionState { @@ -116,6 +117,7 @@ struct ElectionState { only_dirty: HashSet, unsealed: BTreeMap, admission: ReplayAdmission, + required_sources: Vec, } fn segment_stamps(ledger_dir: &Path) -> Result, PipelineError> { @@ -162,7 +164,7 @@ fn load_election_cache( ) -> Option { let payload = std::fs::read(path).ok()?; let cache: ElectionCache = serde_json::from_slice(&payload).ok()?; - if cache.schema != 3 + if cache.schema != 4 || cache.normalizer_version != normalizer_version || cache.segments != segments { @@ -179,6 +181,7 @@ fn load_election_cache( only_dirty: cache.only_dirty, unsealed: cache.unsealed, admission: cache.admission, + required_sources: cache.required_sources, }) } @@ -200,7 +203,7 @@ fn save_election_cache( })?; } let cache = ElectionCache { - schema: 3, + schema: 4, normalizer_version, segments, visit_segments: state.visit.segments, @@ -211,6 +214,7 @@ fn save_election_cache( only_dirty: state.only_dirty.clone(), unsealed: state.unsealed.clone(), admission: state.admission.clone(), + required_sources: state.required_sources.clone(), }; let payload = serde_json::to_vec(&cache).map_err(|error| { PipelineError::Config(format!( @@ -633,6 +637,7 @@ pub fn rebuild_runtime_with_options( let mut latest_seals: BTreeMap = BTreeMap::new(); let mut raw_generations: BTreeMap<(String, u64), SourceDef> = BTreeMap::new(); let mut admission = AdmissionScan::default(); + let mut required_sources = Vec::new(); let visit = visit_election::(ledger_dir, instrument, |entry| { if options .cancel @@ -677,7 +682,16 @@ pub fn rebuild_runtime_with_options( *slot = (generation, source); } } - LedgerEntry::Meta { .. } | LedgerEntry::Document { .. } => {} + LedgerEntry::Meta { + schema_version, + payload, + } => { + if schema_version == crate::LEDGER_META_SCHEMA_VERSION { + required_sources = + crate::LedgerMetadata::decode(&payload)?.required_sources; + } + } + LedgerEntry::Document { .. } => {} } Ok(()) })?; @@ -725,6 +739,7 @@ pub fn rebuild_runtime_with_options( latest_seals, only_dirty, unsealed, + required_sources, }; if let (Some(path), Some(segments)) = (options.election_cache.as_deref(), cache_segments.clone()) @@ -740,6 +755,7 @@ pub fn rebuild_runtime_with_options( only_dirty, unsealed, admission, + required_sources, } = election; report.segments = first.segments; report.torn_segments = first.torn_segments; @@ -776,14 +792,12 @@ pub fn rebuild_runtime_with_options( schema_version, payload, } => { - if !matches!(schema_version, 0 | crate::LEDGER_META_SCHEMA_VERSION) { + if !matches!(schema_version, 0 | 2 | crate::LEDGER_META_SCHEMA_VERSION) { return Err(PipelineError::Config(format!( "unsupported ledger meta schema version {schema_version}" ))); } - let mut ops: Vec = serde_json::from_str(&payload).map_err(|error| { - PipelineError::Config(format!("undecodable meta batch in ledger: {error}")) - })?; + let mut ops = crate::LedgerMetadata::decode(&payload)?.ops; admission.filter_meta(&mut ops); for op in &mut ops { if let IngestOp::SessionMeta(row) = op { @@ -1037,6 +1051,7 @@ pub fn rebuild_runtime_with_options( // sessions whose every sealed source is currently clean are Indexed. // Meta-only sessions are absent from this map and retain Pending. let mut session_health: BTreeMap = BTreeMap::new(); + let required_present = latest_seals.keys().cloned().collect(); for (path, (_, source, known_unclean)) in latest_seals { let clean = if known_unclean { false @@ -1052,6 +1067,11 @@ pub fn rebuild_runtime_with_options( .and_modify(|healthy| *healthy &= clean) .or_insert(clean); } + crate::meta_journal::apply_requirements( + &required_sources, + &required_present, + &mut session_health, + ); for (session_id, clean) in session_health { let next = if clean { StreamState::Indexed @@ -1125,14 +1145,12 @@ pub fn prepare_runtime( schema_version, payload, } => { - if !matches!(schema_version, 0 | crate::LEDGER_META_SCHEMA_VERSION) { + if !matches!(schema_version, 0 | 2 | crate::LEDGER_META_SCHEMA_VERSION) { return Err(PipelineError::Config(format!( "unsupported ledger meta schema version {schema_version}" ))); } - let mut ops = serde_json::from_str(&payload).map_err(|error| { - PipelineError::Config(format!("undecodable meta batch in ledger: {error}")) - })?; + let mut ops = crate::LedgerMetadata::decode(&payload)?.ops; election.admission.filter_meta(&mut ops); meta_ops.push(ops); report.meta_batches += 1; @@ -1337,6 +1355,7 @@ pub fn prepare_runtime( .iter() .map(|file| (file.source.relative_path.as_str(), file)) .collect::>(); + let required_present = election.latest_seals.keys().cloned().collect(); for (path, (generation, source, known_unclean)) in election.latest_seals { let clean = !known_unclean && prepared_by_path.get(path.as_str()).is_some_and(|file| { @@ -1348,6 +1367,11 @@ pub fn prepare_runtime( .or_insert(clean); } + crate::meta_journal::apply_requirements( + &election.required_sources, + &required_present, + &mut session_health, + ); Ok(PreparedRuntime { report, meta_ops, diff --git a/crates/pipeline/src/sched.rs b/crates/pipeline/src/sched.rs index f4a63873..32fe747c 100644 --- a/crates/pipeline/src/sched.rs +++ b/crates/pipeline/src/sched.rs @@ -377,6 +377,7 @@ mod tests { fn file(session_id: &str) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: format!("{session_id}.jsonl"), stream_id: "transcript".into(), diff --git a/crates/pipeline/src/skim.rs b/crates/pipeline/src/skim.rs index c8fcde6f..61755324 100644 --- a/crates/pipeline/src/skim.rs +++ b/crates/pipeline/src/skim.rs @@ -53,12 +53,13 @@ pub(crate) enum Skimmed { /// taken at capture, so a root-tree build reads them to their capture-time /// end, not to the cut. Of those, only a document that frames records (a /// Claude Code workflow run) can project content newer than the cut; subagent -/// sidecars frame none, and Codex declares no snapshot record queries. A -/// fresh roster holds no transcript without a cut: a transcript the skim -/// could not open is not admitted. +/// sidecars frame none, and Codex selector records emit no canonical ops. A +/// compressed input carries its mtime but no physical-to-logical length +/// guess; capture pins and verifies it once. Composed inputs carry explicit +/// ancestor byte cuts. Missing selected inputs retain an incomplete obligation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct Cut { - pub(crate) len: u64, + pub(crate) len: Option, pub(crate) modified_ms: i64, } @@ -76,6 +77,35 @@ pub(crate) fn skim_transcript( opts: &IngestOptions, instrument: &Instrument, ) -> Result { + if let Some(location) = adapter.transcript_location(&file.relative_path) { + let head = match location + .source + .head(root, crate::MAX_PROJECTED_RECORD_BYTES) + { + Ok(Some(head)) => head, + Ok(None) => return Ok(Skimmed::Vanished), + Err(error) if error.incomplete_input() => return Ok(Skimmed::Vanished), + Err(error) => return Err(error.into()), + }; + instrument.add(Metric::FilesSkimmed, 1); + instrument.add(Metric::BytesRead, head.bytes.len() as u64); + let stat = FileStat { + relative_path: file.relative_path.clone(), + size: head.len.unwrap_or(u64::MAX), + modified_ms: head.modified_ms, + }; + let skim = adapter.skim(&stream.id, &stat, &head.bytes, &[]); + if skim.session_id.is_none() && !head.bytes.ends_with(b"\n") { + return Ok(Skimmed::Unfinished); + } + return Ok(Skimmed::Transcript( + Box::new(skim), + Cut { + len: head.len, + modified_ms: head.modified_ms, + }, + )); + } let path = root.join(&file.relative_path); let Some(windows) = instrument.time(Metric::SkimNanos, || { rosetta_capture::head_tail(&path, opts.skim_head_bytes, opts.skim_tail_bytes) @@ -89,7 +119,7 @@ pub(crate) fn skim_transcript( (windows.head.len() + windows.tail.len()) as u64, ); let cut = Cut { - len: windows.len, + len: Some(windows.len), modified_ms: windows.modified_ms, }; diff --git a/crates/pipeline/src/stream.rs b/crates/pipeline/src/stream.rs index 7ee61d71..1bf87f25 100644 --- a/crates/pipeline/src/stream.rs +++ b/crates/pipeline/src/stream.rs @@ -521,7 +521,7 @@ pub(crate) fn ingest_file( let cursor = cursor_row(adapter, store, runtime, file, instrument)?; let mut pass = IngestPass::begin(adapter, runtime, root, file, opts, cursor, mode)?; if mode == PassMode::Audit { - pass.audit(runtime, file, tailer, store, instrument, report)?; + pass.audit(runtime, file, tailer, opts, store, instrument, report)?; } loop { let started = Instant::now(); diff --git a/crates/pipeline/src/stream/pass.rs b/crates/pipeline/src/stream/pass.rs index 2ec283e5..55f33a27 100644 --- a/crates/pipeline/src/stream/pass.rs +++ b/crates/pipeline/src/stream/pass.rs @@ -103,6 +103,8 @@ struct Staged { pub(crate) struct IngestPass { path: PathBuf, + input_root: PathBuf, + input: Option, cursor_id: i64, /// The planner's exact stored checkpoint: the next batch's CAS expectation. prev_blob: String, @@ -259,6 +261,8 @@ impl IngestPass { && source_generation == active_generation; Ok(Self { path: root.join(&file.relative_path), + input_root: root.to_path_buf(), + input: None, cursor_id, prev_blob, active_generation, @@ -289,11 +293,13 @@ impl IngestPass { /// a mismatch is real content change hiding under a quiet stat, and THAT /// escalates to the full replacement replay. A pre-chain checkpoint has /// its chain computed once and persisted, so the next audit verifies. + #[allow(clippy::too_many_arguments)] pub(crate) fn audit( &mut self, runtime: &RuntimeRow, file: &SessionFile, tailer: &Tailer, + opts: &IngestOptions, store: &mut Store, instrument: &Instrument, report: &mut StreamReport, @@ -304,7 +310,29 @@ impl IngestPass { let Some(tail_checkpoint) = self.tail.as_mut() else { return Ok(()); }; - match tailer.verify_committed(&self.path, tail_checkpoint)? { + let verification = if let Some(source) = &file.input { + if self.input.is_none() { + self.input = match source.open(&self.input_root, || opts.check_cancelled().is_err()) + { + Ok(input) => input, + Err(error) if error.incomplete_input() => return Ok(()), + Err(_) if opts.check_cancelled().is_err() => { + return Err(PipelineError::Cancelled); + } + Err(error) => return Err(error.into()), + }; + } + if let Some(input) = &mut self.input { + input.cancel = opts.cancel.clone(); + } + match &mut self.input { + Some(input) => tailer.verify_input(input, tail_checkpoint)?, + None => CommittedVerification::Missing, + } + } else { + tailer.verify_committed(&self.path, tail_checkpoint)? + }; + match verification { CommittedVerification::Verified => { instrument.add(Metric::BytesRead, tail_checkpoint.committed_offset); } @@ -378,13 +406,81 @@ impl IngestPass { _ => tailer, }; let read = instrument.time(Metric::ReadNanos, || { - read_tailer.read_cut( - &self.path, - self.tail.as_ref(), - self.force_replay, - self.ceiling, - ) - })?; + if let Some(source) = &file.input { + if self.input.is_none() { + self.input = + source.open(&self.input_root, || opts.check_cancelled().is_err())?; + if let Some(input) = &mut self.input { + input.cancel = opts.cancel.clone(); + } + if let (Some(input), Some(previous)) = (&mut self.input, &self.tail) + && PipeCheckpoint::decode(&self.prev_blob) + .and_then(|c| c.snapshot_stamp) + .as_deref() + != Some(input.stamp.as_str()) + && input + .file + .metadata() + .is_ok_and(|m| m.len() <= previous.observed_len) + && matches!( + read_tailer.verify_input(input, previous)?, + CommittedVerification::Mismatch + ) + { + self.force_replay = true; + } + } + if self.input.as_ref().is_some_and(|input| { + input.boundaries.iter().any(|(expected, record)| { + adapter + .transcript_record_index(record) + .and_then(|i| i.checked_add(1)) + != Some(*expected) + }) + }) { + return Err(rosetta_capture::CaptureError::Io { + operation: "validating transcript range", + path: file.relative_path.clone(), + source: std::io::Error::new( + std::io::ErrorKind::InvalidData, + "native ordinal and decoded byte boundary disagree", + ), + }); + } + match &mut self.input { + Some(input) => read_tailer.read_input( + input, + self.tail.as_ref(), + self.force_replay, + self.ceiling, + ), + None => Ok(TailRead::Missing), + } + } else { + read_tailer.read_cut( + &self.path, + self.tail.as_ref(), + self.force_replay, + self.ceiling, + ) + } + }); + let read = match read { + Ok(read) => read, + Err(_) if opts.check_cancelled().is_err() => return Err(PipelineError::Cancelled), + Err(rosetta_capture::CaptureError::Io { source, .. }) + if file.input.is_some() + && matches!( + source.kind(), + std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof + ) => + { + deltas.retries_exhausted += 1; + instrument.add(Metric::RetriesExhausted, 1); + return Ok(self.stop_short(runtime, file, deltas)); + } + Err(error) => return Err(error.into()), + }; match read { TailRead::Missing => { deltas.files_missing += 1; @@ -598,7 +694,7 @@ impl IngestPass { tail: Some(batch.checkpoint.encode_hex()), cumulative: (!last_reading.is_empty()).then_some(last_reading), adapter_state, - snapshot_stamp: None, + snapshot_stamp: self.input.as_ref().map(|input| input.stamp.clone()), snapshot_revision: None, } .encode(); diff --git a/crates/pipeline/src/streamed/ordinals.rs b/crates/pipeline/src/streamed/ordinals.rs index 118051bf..43ccf578 100644 --- a/crates/pipeline/src/streamed/ordinals.rs +++ b/crates/pipeline/src/streamed/ordinals.rs @@ -310,6 +310,7 @@ mod tests { sources .iter() .map(|(source, _)| crate::meta::SessionFile { + input: None, ledger_only: false, relative_path: source.relative_path.clone(), stream_id: source.stream_id.clone(), diff --git a/crates/pipeline/src/streamed/slices_tests.rs b/crates/pipeline/src/streamed/slices_tests.rs index 04996928..da4dea18 100644 --- a/crates/pipeline/src/streamed/slices_tests.rs +++ b/crates/pipeline/src/streamed/slices_tests.rs @@ -70,6 +70,7 @@ fn session_meta(id: &str, parent: Option<&str>) -> IngestOp { fn file(path: &str, session_id: &str) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.into(), stream_id: "transcript".into(), diff --git a/crates/pipeline/tests/codex_storage.rs b/crates/pipeline/tests/codex_storage.rs new file mode 100644 index 00000000..7797d080 --- /dev/null +++ b/crates/pipeline/tests/codex_storage.rs @@ -0,0 +1,674 @@ +//! Storage protocol regressions use synthetic native files only. +use rosetta_adapter_codex::CodexAdapter; +use rosetta_capture::{Ledger, scan_ledger}; +use rosetta_core::{ids, rows::RuntimeRow}; +use rosetta_pipeline::{ + IngestOptions, Instrument, meta_ingest_into, rebuild_runtime, stream_backfill_into, +}; +use rosetta_store::{Store, queries}; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; + +struct World { + _dir: TempDir, + root: PathBuf, + archive: PathBuf, + runtime: RuntimeRow, + store: Store, +} +impl World { + fn new() -> Self { + let dir = TempDir::new().unwrap(); + let root = dir.path().join("native"); + let archive = dir.path().join("ledger"); + std::fs::create_dir_all(&root).unwrap(); + let runtime = RuntimeRow { + id: ids::runtime_id("codex", "~/.codex"), + kind: "codex".into(), + data_root: "~/.codex".into(), + enabled: true, + account: None, + config: None, + skills: None, + versions_seen: None, + discovered_at: 0, + label: None, + provenance: Default::default(), + }; + Self { + _dir: dir, + root, + archive, + runtime, + store: Store::open_in_memory().unwrap(), + } + } + fn write(&self, id: &str, bytes: &[u8], compressed: bool, archived: bool) -> PathBuf { + let filename = format!("rollout-2026-11-01T12-00-00-{id}.jsonl"); + let path = self + .root + .join(if archived { + "archived_sessions" + } else { + "sessions/2026/11/01" + }) + .join(if compressed { + format!("{filename}.zst") + } else { + filename + }); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write( + &path, + if compressed { + zstd::encode_all(bytes, 3).unwrap() + } else { + bytes.to_vec() + }, + ) + .unwrap(); + path + } + fn select(&self, id: &str, path: &Path) { + let db = rusqlite::Connection::open(self.root.join("state_5.sqlite")).unwrap(); + db.execute_batch("CREATE TABLE IF NOT EXISTS threads (id TEXT PRIMARY KEY, rollout_path TEXT, cwd TEXT, cli_version TEXT, created_at INTEGER, updated_at INTEGER, history_mode TEXT);").unwrap(); + let path = path.to_string_lossy(); + let path = path.strip_suffix(".zst").unwrap_or(&path); + db.execute("INSERT OR REPLACE INTO threads VALUES (?1,?2,'/test/project','0.167.2',1,2,'paginated')", rusqlite::params![id, path]).unwrap(); + } + fn ingest(&mut self) -> rosetta_pipeline::StreamReport { + let opts = IngestOptions { + audit_files_per_run: 0, + ..Default::default() + }; + let instrument = Instrument::new(); + let mut ledger = Ledger::open(&self.archive).unwrap(); + let (_, files) = meta_ingest_into( + &CodexAdapter, + &self.runtime, + &self.root, + &mut self.store, + &opts, + &instrument, + Some(&mut ledger), + ) + .unwrap(); + let report = stream_backfill_into( + &CodexAdapter, + &self.runtime, + &self.root, + &mut self.store, + &files, + &opts, + &instrument, + Some(&mut ledger), + ) + .unwrap(); + ledger.flush().unwrap(); + report + } + fn texts(&self, native: &str) -> Vec { + self.store + .conn() + .prepare("SELECT content FROM message WHERE session_id=?1 ORDER BY ordinal") + .unwrap() + .query_map([ids::session_id(&self.runtime.id, native)], |r| r.get(0)) + .unwrap() + .collect::>() + .unwrap() + } + fn indexed(&self, native: &str) -> bool { + queries::session( + self.store.conn(), + &ids::session_id(&self.runtime.id, native), + ) + .unwrap() + .is_some_and(|s| s.stream_state == rosetta_core::rows::StreamState::Indexed) + } + fn rebuild(&mut self) { + std::fs::remove_dir_all(&self.root).unwrap(); + self.store = Store::open_in_memory().unwrap(); + let report = rebuild_runtime( + &CodexAdapter, + &self.runtime, + &self.archive, + &mut self.store, + &Instrument::new(), + ) + .unwrap(); + assert_eq!(report.hard_errors, 0); + } + fn raw_count(&self) -> usize { + scan_ledger(&self.archive) + .unwrap() + .entries + .iter() + .filter(|e| matches!(e, rosetta_capture::LedgerEntry::Raw { .. })) + .count() + } +} +fn line(v: Value) -> Vec { + let mut b = serde_json::to_vec(&v).unwrap(); + b.push(b'\n'); + b +} +fn header(id: &str, base: Option<(&str, u64, u64)>) -> Vec { + let mut payload = json!({"id":id,"timestamp":"2026-11-01T12:00:00Z","cwd":"/test/project","originator":"codex_cli","cli_version":"0.167.2","source":"cli","model_provider":"openai","history_mode":"paginated"}); + if let Some((parent, end, offset)) = base { + payload["history_base"] = + json!({"thread_id":parent,"end_ordinal_exclusive":end,"end_byte_offset":offset}); + } + line( + json!({"timestamp":"2026-11-01T12:00:00Z","ordinal":base.map_or(0, |b| b.1),"type":"session_meta","payload":payload}), + ) +} +fn message(ordinal: u64, text: &str) -> Vec { + line( + json!({"timestamp":"2026-11-01T12:00:01Z","ordinal":ordinal,"type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":text}]}}), + ) +} +fn transcript(id: &str) -> Vec { + [header(id, None), message(1, "first"), message(2, "second")].concat() +} + +#[test] +fn compressed_archived_restore_append_and_offline_rebuild() { + let mut w = World::new(); + let bytes = transcript("one"); + let cold = w.write("one", &bytes, true, true); + w.select("one", &cold); + assert_eq!(w.ingest().hard_errors, 0); + assert_eq!(w.texts("one"), ["first", "second"]); + assert!(w.indexed("one")); + let count = w.raw_count(); + assert_eq!(w.ingest().records, 0); + assert_eq!(w.raw_count(), count); + let hot = w.write("one", &bytes, false, false); + std::fs::remove_file(cold).unwrap(); + w.select("one", &hot); + let restored = w.ingest(); + assert_eq!( + restored.generations_restarted, 1, + "only the changed DB selector is a replacement" + ); + assert_eq!( + restored.records, 1, + "only changed native DB selection is captured" + ); + let appended = [bytes, message(3, "third")].concat(); + std::fs::write(hot, appended).unwrap(); + assert_eq!(w.ingest().generations_restarted, 0); + assert_eq!(w.texts("one"), ["first", "second", "third"]); + w.rebuild(); + assert_eq!(w.texts("one"), ["first", "second", "third"]); +} + +#[test] +fn both_representations_prefer_plain_and_ignore_temporary_files() { + let mut w = World::new(); + w.write("one", &transcript("one"), true, false); + let hot = w.write( + "one", + &[transcript("one"), message(3, "plain wins")].concat(), + false, + false, + ); + std::fs::write(hot.with_extension("jsonl.zst.tmp"), b"not a transcript").unwrap(); + w.ingest(); + assert_eq!(w.texts("one"), ["first", "second", "plain wins"]); +} + +#[test] +fn nested_history_cuts_do_not_grow_with_parents() { + let mut w = World::new(); + let parent = transcript("parent"); + let first_cut = (header("parent", None).len() + message(1, "first").len()) as u64; + w.write("parent", &parent, true, false); + let child = [ + header("child", Some(("parent", 2, first_cut))), + message(3, "child own"), + ] + .concat(); + w.write("child", &child, false, true); + let grandchild = [ + header("grandchild", Some(("child", 4, child.len() as u64))), + message(5, "grandchild own"), + ] + .concat(); + w.write("grandchild", &grandchild, true, false); + assert_eq!(w.ingest().hard_errors, 0); + assert_eq!(w.texts("child"), ["first", "child own"]); + assert_eq!( + w.texts("grandchild"), + ["first", "child own", "grandchild own"] + ); + w.write( + "parent", + &[parent, message(3, "later parent")].concat(), + true, + false, + ); + w.ingest(); + assert_eq!(w.texts("child"), ["first", "child own"]); + w.rebuild(); + assert_eq!( + w.texts("grandchild"), + ["first", "child own", "grandchild own"] + ); +} + +#[test] +fn revert_selects_one_branch_and_replaces_the_old_projection() { + let mut w = World::new(); + let original = transcript("one"); + let old = w.write("one", &original, false, false); + w.select("one", &old); + w.ingest(); + let cut = (header("one", None).len() + message(1, "first").len()) as u64; + let reverted = [ + header("one", Some(("one", 2, cut))), + message(3, "new branch"), + ] + .concat(); + let new = w.write("one_reverted", &reverted, true, false); + w.select("one", &new); + w.ingest(); + assert_eq!(w.texts("one"), ["first", "new branch"]); + w.rebuild(); + assert_eq!(w.texts("one"), ["first", "new branch"]); +} + +#[test] +fn missing_selected_rollout_does_not_resurrect_an_old_branch() { + let mut w = World::new(); + w.write("one", &transcript("one"), false, false); + let missing = w + .root + .join("sessions/2026/11/01/rollout-2026-11-01T12-00-00-one_missing.jsonl"); + w.select("one", &missing); + w.ingest(); + assert!(w.texts("one").is_empty()); + assert!(!w.indexed("one")); +} + +#[test] +fn missing_base_and_cycles_remain_incomplete() { + let mut w = World::new(); + w.write( + "one", + &[ + header("one", Some(("missing", 1, 999))), + message(2, "must wait"), + ] + .concat(), + false, + false, + ); + w.ingest(); + assert!(w.texts("one").is_empty()); + assert!(!w.indexed("one")); + w.write("one", &header("one", Some(("one", 1, 999))), false, false); + w.ingest(); + assert!(w.texts("one").is_empty()); + assert!(!w.indexed("one")); +} + +#[test] +fn corrupted_replacement_keeps_the_last_complete_projection() { + let mut w = World::new(); + let cold = w.write("one", &transcript("one"), true, false); + w.ingest(); + let mut bytes = std::fs::read(&cold).unwrap(); + bytes.truncate(bytes.len() - 4); + std::fs::write(cold, bytes).unwrap(); + let report = w.ingest(); + assert!(report.retries_exhausted > 0 || report.files_missing > 0); + assert_eq!(w.texts("one"), ["first", "second"]); + assert!(!w.indexed("one")); +} + +fn runner(w: &mut World) -> rosetta_pipeline::LiveRunner<'static> { + rosetta_pipeline::LiveRunner::start_with_ledger( + &CodexAdapter, + w.runtime.clone(), + w.root.clone(), + Box::new(rosetta_capture::ScriptedEvents::new([])), + &mut w.store, + IngestOptions { + audit_files_per_run: 0, + ..Default::default() + }, + rosetta_pipeline::LiveOptions { + tick: std::time::Duration::ZERO, + poll_interval: std::time::Duration::ZERO, + max_coalesced: 16, + }, + Some(Ledger::open(&w.archive).unwrap()), + &Instrument::new(), + ) + .unwrap() +} + +#[test] +fn quiet_compressed_polls_do_not_redecode_or_append_ledger_records() { + let mut w = World::new(); + w.write("one", &transcript("one"), true, true); + w.ingest(); + let count = w.raw_count(); + let mut live = runner(&mut w); + let instrument = Instrument::new(); + for _ in 0..3 { + live.catch_up(&mut w.store, &instrument).unwrap(); + } + let report = live.into_report(); + assert_eq!(report.stream.files, 0); + assert_eq!(report.admissions, 0); + assert_eq!(w.raw_count(), count); +} + +#[test] +fn dropped_restore_events_recover_appended_records_on_poll() { + let mut w = World::new(); + let cold = w.write("one", &transcript("one"), true, true); + w.ingest(); + let mut live = runner(&mut w); + w.write( + "one", + &[transcript("one"), message(3, "restored while offline")].concat(), + false, + false, + ); + std::fs::remove_file(cold).unwrap(); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!( + w.texts("one"), + ["first", "second", "restored while offline"] + ); + let report = live.into_report(); + assert_eq!(report.stream.generations_restarted, 0); +} + +#[test] +fn changed_database_selection_streams_the_reverted_view_live() { + let mut w = World::new(); + let old = w.write("one", &transcript("one"), true, false); + w.select("one", &old); + w.ingest(); + let mut live = runner(&mut w); + // Seed the DB stat gate before changing selection. + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + let cut = (header("one", None).len() + message(1, "first").len()) as u64; + let new = w.write( + "one_revert", + &[ + header("one", Some(("one", 2, cut))), + message(3, "live branch"), + ] + .concat(), + true, + false, + ); + w.select("one", &new); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("one"), ["first", "live branch"]); +} + +#[test] +fn same_size_same_mtime_rewrite_and_legacy_to_shared_migration() { + let mut w = World::new(); + let path = w.write("one", &transcript("one"), false, false); + w.ingest(); + let stamp = std::fs::metadata(&path).unwrap().modified().unwrap(); + let changed = [ + header("one", None), + message(1, "FIRST"), + message(2, "second"), + ] + .concat(); + std::fs::write(&path, changed).unwrap(); + std::fs::File::options() + .write(true) + .open(&path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(stamp)) + .unwrap(); + let mut live = runner(&mut w); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("one"), ["FIRST", "second"]); + drop(live); + let parent = transcript("parent"); + w.write("parent", &parent, true, true); + w.write( + "one", + &[ + header("one", Some(("parent", 3, parent.len() as u64))), + message(4, "migrated"), + ] + .concat(), + false, + false, + ); + let mut live = runner(&mut w); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("one"), ["first", "second", "migrated"]); +} + +#[test] +fn reference_requires_matching_ordinal_and_newline_byte_boundaries() { + for bad in 0..3 { + let mut w = World::new(); + let parent = transcript("parent"); + w.write("parent", &parent, true, false); + let (end, offset) = match bad { + 0 => (3, parent.len() as u64 - 1), + 1 => (4, parent.len() as u64), + _ => (3, parent.len() as u64 + 100), + }; + w.write( + "child", + &[ + header("child", Some(("parent", end, offset))), + message(end + 1, "withheld"), + ] + .concat(), + true, + false, + ); + w.ingest(); + assert!(w.texts("child").is_empty()); + assert!(!w.indexed("child")); + } +} + +#[test] +fn missing_base_appearing_later_is_admitted_by_poll() { + let mut w = World::new(); + let parent = transcript("parent"); + w.write( + "child", + &[ + header("child", Some(("parent", 3, parent.len() as u64))), + message(4, "child"), + ] + .concat(), + true, + false, + ); + w.ingest(); + assert!(!w.indexed("child")); + let mut live = runner(&mut w); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + w.write("parent", &parent, true, true); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("child"), ["first", "second", "child"]); + assert!(w.indexed("child")); +} + +#[test] +fn relocated_native_state_paths_bind_only_inside_the_enrolled_root() { + let mut w = World::new(); + let file = w.write("one", &transcript("one"), true, false); + let relative = file.strip_prefix(&w.root).unwrap(); + w.select("one", &Path::new("/old-codex-home").join(relative)); + w.ingest(); + assert_eq!(w.texts("one"), ["first", "second"]); +} + +#[test] +fn offline_rebuild_keeps_a_missing_selected_transcript_incomplete() { + let mut w = World::new(); + let missing = w + .root + .join("sessions/2026/11/01/rollout-2026-11-01T12-00-00-one.jsonl"); + w.select("one", &missing); + w.ingest(); + assert!(!w.indexed("one")); + w.rebuild(); + assert!(!w.indexed("one")); +} + +#[test] +fn shared_history_does_not_duplicate_usage_or_artifact_publications() { + let mut w = World::new(); + let usage = json!({"input_tokens":10,"cached_input_tokens":0,"output_tokens":2,"reasoning_output_tokens":0,"total_tokens":12}); + let parent = [header("parent",None), + message(1,"Review [the design](/tmp/rosetta-missing-review.html)"), + line(json!({"timestamp":"2026-11-01T12:00:01Z","ordinal":2,"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":usage,"last_token_usage":usage}}}))].concat(); + w.write("parent", &parent, true, true); + w.write( + "child", + &[ + header("child", Some(("parent", 3, parent.len() as u64))), + message(4, "own work"), + ] + .concat(), + true, + false, + ); + w.ingest(); + let one = |sql| { + w.store + .conn() + .query_row(sql, [], |r| r.get::<_, i64>(0)) + .unwrap() + }; + assert_eq!(one("SELECT COUNT(*) FROM html_artifact"), 1); + assert_eq!( + one("SELECT SUM(in_tokens + out_tokens) FROM usage_bucket"), + 12 + ); + assert_eq!(w.texts("child").len(), 2); + w.rebuild(); + assert_eq!( + w.store + .conn() + .query_row( + "SELECT SUM(in_tokens + out_tokens) FROM usage_bucket", + [], + |r| r.get::<_, i64>(0) + ) + .unwrap(), + 12 + ); +} + +#[test] +fn attached_child_shows_its_own_history_without_parent_context() { + let mut w = World::new(); + let parent = transcript("parent"); + w.write("parent", &parent, true, false); + let mut meta: Value = + serde_json::from_slice(&header("child", Some(("parent", 3, parent.len() as u64)))).unwrap(); + meta["payload"]["attached_thread"] = json!(true); + w.write( + "child", + &[line(meta), message(4, "attached work")].concat(), + true, + false, + ); + w.ingest(); + assert_eq!(w.texts("child"), ["attached work"]); + w.rebuild(); + assert_eq!(w.texts("child"), ["attached work"]); +} + +/// Same frozen decoded bytes, normalizer, ledger codec and logical source +/// identities in both arms. Run explicitly in release mode; fixture creation +/// and compression are outside the measured ingestion interval. +#[test] +#[ignore = "release-only paired ingestion measurement"] +fn frozen_representation_benchmark() { + let fixtures: Vec<_> = (0..16) + .map(|session| { + let id = format!("benchmark-{session}"); + let mut bytes = header(&id, None); + for ordinal in 1..=500 { + bytes.extend(message( + ordinal, + &format!("synthetic {ordinal} {}", "review ".repeat(48)), + )); + } + (id, bytes) + }) + .collect(); + let decoded_bytes: usize = fixtures.iter().map(|(_, bytes)| bytes.len()).sum(); + let mut digests = Vec::new(); + for compressed in [false, true] { + let mut w = World::new(); + for (id, bytes) in &fixtures { + w.write(id, bytes, compressed, false); + } + let start = std::time::Instant::now(); + let report = w.ingest(); + let cold_ms = start.elapsed().as_secs_f64() * 1000.0; + assert_eq!(report.hard_errors, 0); + assert_eq!(report.files_incomplete, 0); + let mut hash = blake3::Hasher::new(); + let mut rows = 0; + for (id, _) in &fixtures { + for text in w.texts(id) { + hash.update(text.as_bytes()); + rows += 1; + } + } + digests.push(hash.finalize().to_hex().to_string()); + let mut live = runner(&mut w); + let start = std::time::Instant::now(); + for _ in 0..20 { + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + } + let poll_ms = start.elapsed().as_secs_f64() * 1000.0; + let live = live.into_report(); + assert_eq!(live.stream.files, 0); + println!( + "representation={} decoded_bytes={decoded_bytes} rows={rows} cold_ms={cold_ms:.2} twenty_polls_ms={poll_ms:.2} quiet_capture_passes={} digest={}", + if compressed { "zstd" } else { "plain" }, + live.stream.files, + digests.last().unwrap() + ); + } + assert_eq!(digests[0], digests[1]); +} + +#[test] +fn live_corruption_keeps_the_previous_view_and_can_recover() { + let mut w = World::new(); + let cold = w.write("one", &transcript("one"), true, false); + w.ingest(); + let mut live = runner(&mut w); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + std::fs::write(&cold, b"invalid zstd").unwrap(); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("one"), ["first", "second"]); + assert!(!w.indexed("one")); + w.write( + "one", + &[transcript("one"), message(3, "recovered")].concat(), + true, + false, + ); + live.catch_up(&mut w.store, &Instrument::new()).unwrap(); + assert_eq!(w.texts("one"), ["first", "second", "recovered"]); + assert!(w.indexed("one")); +} diff --git a/crates/pipeline/tests/exclusions.rs b/crates/pipeline/tests/exclusions.rs index 5d983cb0..51be2e1c 100644 --- a/crates/pipeline/tests/exclusions.rs +++ b/crates/pipeline/tests/exclusions.rs @@ -208,7 +208,10 @@ fn exclusion_removes_every_sibling_child_and_orphan_placeholder_before_journalin assert!(!raw_payloads(&h.ledger_dir).contains("excludedpayload")); for entry in scan_ledger(&h.ledger_dir).unwrap().entries { if let LedgerEntry::Meta { payload, .. } = entry { - for op in serde_json::from_str::>(&payload).unwrap() { + for op in rosetta_pipeline::LedgerMetadata::decode(&payload) + .unwrap() + .ops + { if let IngestOp::SessionMeta(row) = op { assert_eq!(row.native_session_id, "allowed"); } diff --git a/crates/pipeline/tests/ledger_only.rs b/crates/pipeline/tests/ledger_only.rs index af04d3b4..e7a2089c 100644 --- a/crates/pipeline/tests/ledger_only.rs +++ b/crates/pipeline/tests/ledger_only.rs @@ -343,7 +343,9 @@ fn fresh_ingest_resume_and_ledger_replay_keep_only_user_facing_sessions() { assert!(scanned.entries.iter().any(|entry| matches!(entry, LedgerEntry::Raw { source, payload, .. } if source.ledger_only && String::from_utf8_lossy(payload).contains("private-review-sentinel resumed")))); for entry in &scanned.entries { if let LedgerEntry::Meta { payload, .. } = entry { - let ops: Vec = serde_json::from_str(payload).unwrap(); + let ops: Vec = rosetta_pipeline::LedgerMetadata::decode(payload) + .unwrap() + .ops; assert!(ops.iter().all(|op| !matches!(op, IngestOp::SessionMeta(s) if s.native_session_id.ends_with("guardian")))); } } @@ -512,7 +514,9 @@ fn old_archives_rebuild_without_empty_review_headers_and_resume_capture() { schema_version, payload, } => { - let mut ops: Vec = serde_json::from_str(&payload).unwrap(); + let mut ops: Vec = rosetta_pipeline::LedgerMetadata::decode(&payload) + .unwrap() + .ops; ops.extend(internal_meta.clone()); old.append_meta(schema_version, &serde_json::to_string(&ops).unwrap()) .unwrap(); diff --git a/crates/rosettad/src/ingest_metrics/tests.rs b/crates/rosettad/src/ingest_metrics/tests.rs index 9e7d29a7..89ae5632 100644 --- a/crates/rosettad/src/ingest_metrics/tests.rs +++ b/crates/rosettad/src/ingest_metrics/tests.rs @@ -45,6 +45,7 @@ fn roster_file( age_ms: i64, ) -> SessionFile { SessionFile { + input: None, ledger_only: false, relative_path: path.to_owned(), stream_id: "stream".to_owned(), diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6750725c..913723fa 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -168,7 +168,7 @@ timing. > census size). Each lane holds whole trees by recency key `k` (the newest skim mtime over the > tree's own sources) descending, then tree id, each tree's sources in path order; the serial > driver walks `P`, then every lane's trees merged by the same key. A plan that cannot hoist a -> shared member (one with per-session record queries) or cannot place a source in a tree walks +> shared member (one whose records can emit canonical operations) or cannot place a source in a tree walks > paths instead, and a truncated discovery or metadata walk falls back to runtime admission > before the producer starts. Only a transcript whose session still has an > uncaptured transcript, or a pending sibling heal after a restart, is held; the heal runs as soon @@ -3895,3 +3895,49 @@ start until the rebuild is run by hand. The release notes must say so. record ruled that confinement is the serving host's policy and a stored page is never rewritten (its DR-D1 (b) and DR-D11 (a)), so the door places the page by digest and the shell became the option above. The draft never shipped. + + +### Decoded transcript inputs and fixed history ranges (2026-11-12) + +Adapters can declare a stable logical transcript path and an ordered set of +physical representations through `transcript_location`. Capture owns plain/zstd +IO, root-relative path validation, bounded headers, pinned file handles, anonymous +seekable spools, cancellation and decoded-size limits. The adapter owns native +filename rules, current-source selection and immutable history references. Metadata +resolves those references into transient ordered byte ranges; the canonical SQLite +schema contains neither the graph nor native selection rows. + +All cursor offsets and prefix checks apply to decoded bytes. A changed physical +identity can continue only after verifying the complete committed prefix; legacy +checkpoints without a content chain replay safely. Ordinary quiet polls compare +physical stamps without decoding. A compressed source is decoded and checked to +EOF once per pass, not once per batch. A changed header forces metadata admission; +a changed body stages a replacement while the previous valid projection remains +readable. Native files are read without restoration or writer locks. Limits are +32 MiB for bounded headers and reference-boundary records, 64 GiB per decoded +source, and 128 lineage segments. Invalid, missing or truncated inputs are not EOF. + +The selected header precedes the ancestral ranges and each range retains its +native header. The raw ledger therefore contains a closed, deterministic selected +view, even if native files disappear before rebuilding. Original record ownership +prevents duplicated inherited usage and artifact publications. Codex's SQLite +selector rows are also captured as raw records and emit no canonical operations. +This preserves its root-tree closure contract. Existing ordinary rollout cursor +paths remain stable across compression, archive moves and later reverts. + +Metadata journal schema 3 adds bounded required-source identifiers and availability +beside the ordinary metadata operations. Rebuild uses these obligations to avoid +marking a metadata-only session Indexed when its selected transcript was never +captured. Legacy metadata arrays (schemas 0 and 2) remain readable; the disposable +election cache advances to schema 4. This is an additive ledger change, not a +canonical database migration. + + +A release-mode frozen comparison on 2026-11-12 used 16 synthetic sessions, +4,136,966 decoded bytes and 8,000 canonical messages in both representations. +One paired run took 485.63 ms (plain) and 493.43 ms (zstd), with identical +projection-content digest `6dd1498dc98c2d5216ddca147574c7d2eb9d95ae7d0d59ef6539ce62b5f34d47`. +Twenty quiet polls took 9.86/10.32 ms and started zero capture passes. Corpus +creation/compression was excluded. This small warm-filesystem measurement is +not an estimate for a full user archive or long shared prefixes. Reproduce with +`cargo test --release -p rosetta-pipeline --test codex_storage frozen_representation_benchmark -- --ignored --nocapture`. From 59bbd8d03e6c849f3bef02e7cb1ec8d2b0f47963 Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 16:02:01 -0800 Subject: [PATCH 4/6] fix(codex): reconcile legacy branch cursors with selected history --- crates/pipeline/src/meta/transcripts.rs | 23 ++++- crates/pipeline/tests/codex_storage.rs | 112 ++++++++++++++++++++++++ crates/pipeline/tests/regressions.rs | 10 +-- docs/ARCHITECTURE.md | 5 ++ 4 files changed, 142 insertions(+), 8 deletions(-) diff --git a/crates/pipeline/src/meta/transcripts.rs b/crates/pipeline/src/meta/transcripts.rs index 7f33d41d..f1ba98f2 100644 --- a/crates/pipeline/src/meta/transcripts.rs +++ b/crates/pipeline/src/meta/transcripts.rs @@ -190,10 +190,14 @@ pub(super) fn resolve( file.relative_path = format!("missing-transcript/{native}"); } } - let existing = cursors.iter().find(|(path, _, session)| { - session == &file.session_id && adapter.transcript_location(path).is_some() - }); + let existing: Vec<_> = cursors + .iter() + .filter(|(path, _, session)| { + session == &file.session_id && adapter.transcript_location(path).is_some() + }) + .collect(); file.relative_path = existing + .first() .map(|(path, _, _)| path.clone()) .unwrap_or_else(|| { ids.iter() @@ -217,6 +221,19 @@ pub(super) fn resolve( // explicit and its own file is pinned when capture starts. file.cut_len = None; } + // Pre-selection importers could have projected several branches as + // independent sources. Keep those bounded compatibility cursors on + // the SAME selected view, so each replaces its abandoned contribution + // and offline replay cannot resurrect it. No new alias is created; + // fresh imports still have exactly one logical transcript per thread. + // Mirroring later changes is essential: merely fixing the aliases + // once would leave their old selected view alive after another revert. + for (path, stream, _) in existing.into_iter().skip(1) { + let mut alias = file.clone(); + alias.relative_path = path.clone(); + alias.stream_id = stream.clone(); + retained.push(alias); + } retained.push(file); } output diff --git a/crates/pipeline/tests/codex_storage.rs b/crates/pipeline/tests/codex_storage.rs index 7797d080..3923c4a3 100644 --- a/crates/pipeline/tests/codex_storage.rs +++ b/crates/pipeline/tests/codex_storage.rs @@ -279,6 +279,118 @@ fn revert_selects_one_branch_and_replaces_the_old_projection() { assert_eq!(w.texts("one"), ["first", "new branch"]); } +#[test] +fn legacy_branch_cursors_follow_the_selected_view_through_later_reverts() { + let mut w = World::new(); + let original = transcript("one"); + let old = w.write("one", &original, false, false); + w.select("one", &old); + w.ingest(); + + // Reproduce the old importer's independent plain-file cursor, bypassing + // representation/selection planning just as that importer did. + let legacy = w.write( + "one_legacy", + &[header("one", None), message(10, "discarded legacy branch")].concat(), + false, + false, + ); + let opts = IngestOptions::default(); + let instrument = Instrument::new(); + let mut ledger = Ledger::open(&w.archive).unwrap(); + let (_, files) = meta_ingest_into( + &CodexAdapter, + &w.runtime, + &w.root, + &mut w.store, + &opts, + &instrument, + Some(&mut ledger), + ) + .unwrap(); + let mut file = files.into_iter().find(|f| f.input.is_some()).unwrap(); + file.relative_path = legacy + .strip_prefix(&w.root) + .unwrap() + .to_str() + .unwrap() + .into(); + file.input = None; + file.cut_len = None; + stream_backfill_into( + &CodexAdapter, + &w.runtime, + &w.root, + &mut w.store, + &[file], + &opts, + &instrument, + Some(&mut ledger), + ) + .unwrap(); + ledger.flush().unwrap(); + drop(ledger); + assert!( + w.texts("one") + .iter() + .any(|t| t == "discarded legacy branch") + ); + + let cut = (header("one", None).len() + message(1, "first").len()) as u64; + let review = "Review [the design](/tmp/rosetta-missing-review.html)"; + let totals = |w: &World| { + let one = |sql| { + w.store + .conn() + .query_row(sql, [], |r| r.get::<_, i64>(0)) + .unwrap() + }; + ( + one("SELECT COUNT(*) FROM html_artifact"), + one("SELECT SUM(in_tokens + out_tokens) FROM usage_bucket"), + ) + }; + for (index, branch) in ["replacement", "later replacement"].into_iter().enumerate() { + let mut live = (index > 0).then(|| runner(&mut w)); + let usage = json!({"input_tokens":10,"cached_input_tokens":0,"output_tokens":2,"reasoning_output_tokens":0,"total_tokens":12}); + let current = w.write( + "one_reverted", + &[ + header("one", Some(("one", 2, cut))), + message(3, branch), + message(4, review), + line(json!({ + "timestamp":"2026-11-01T12:00:01Z", "ordinal":5, "type":"token_usage_record", + "payload":{ + "thread_id":"one", + "turn_id":"turn", "session_id":"execution", "root_turn_id":"turn", + "response_id":"native-response", "usage":usage, + "turn_token_usage":usage, "thread_token_usage":usage + } + })), + ] + .concat(), + true, + false, + ); + w.select("one", ¤t); + if let Some(live) = live.as_mut() { + live.catch_up(&mut w.store, &instrument).unwrap(); + } else { + let report = w.ingest(); + assert_eq!((report.hard_errors, report.unknown), (0, 0)); + } + assert_eq!(w.texts("one"), ["first", branch, review]); + assert!(w.indexed("one")); + assert_eq!(totals(&w), (1, 12)); + drop(live); + assert_eq!(w.ingest().records, 0); + } + w.rebuild(); + assert_eq!(w.texts("one"), ["first", "later replacement", review]); + assert_eq!(totals(&w), (1, 12)); +} + #[test] fn missing_selected_rollout_does_not_resurrect_an_old_branch() { let mut w = World::new(); diff --git a/crates/pipeline/tests/regressions.rs b/crates/pipeline/tests/regressions.rs index b13ebed2..32380eaa 100644 --- a/crates/pipeline/tests/regressions.rs +++ b/crates/pipeline/tests/regressions.rs @@ -442,10 +442,9 @@ fn codex_large_opening_record_gets_a_bounded_identity_retry() { fn codex_state_titles_merge_with_rollouts_and_update_incrementally() { let root = TempDir::new().unwrap(); let native = "01a036c5-6e18-73f3-8c07-d4430763840b"; + let rollout = format!("sessions/2026/08/25/rollout-2026-08-25T02-35-03-{native}.jsonl"); write_lines( - &root - .path() - .join("sessions/2026/08/25/rollout-session.jsonl"), + &root.path().join(&rollout), &[json!({ "timestamp": "2026-08-25T02:35:03.577Z", "type": "session_meta", @@ -473,11 +472,12 @@ fn codex_state_titles_merge_with_rollouts_and_update_incrementally() { db.execute( "INSERT INTO threads (id, rollout_path, created_at, updated_at, source, cwd, title, cli_version, name) - VALUES (?1, 'rollout', 10, 11, 'exec', '/w/project', ?2, '0.150.1', ?3)", + VALUES (?1, ?4, 10, 11, 'exec', '/w/project', ?2, '0.150.1', ?3)", rusqlite::params![ native, "a very long first prompt that is only a fallback", - "Review Rosetta codebase" + "Review Rosetta codebase", + rollout ], ) .unwrap(); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 913723fa..fdd940f0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -3924,6 +3924,11 @@ prevents duplicated inherited usage and artifact publications. Codex's SQLite selector rows are also captured as raw records and emit no canonical operations. This preserves its root-tree closure contract. Existing ordinary rollout cursor paths remain stable across compression, archive moves and later reverts. +An older database may already have separate cursors for several branches of one +thread. Those existing cursors follow the same selected view, replacing each old +contribution and keeping later reverts and offline replay consistent. This bounded +compatibility case captures the selected view once per existing cursor when it +changes; fresh imports create only one logical transcript per thread. Metadata journal schema 3 adds bounded required-source identifiers and availability beside the ordinary metadata operations. Rebuild uses these obligations to avoid From 0aaca97fcc063a7ec400c5f556b4d38c84714631 Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 16:06:39 -0800 Subject: [PATCH 5/6] fix(capture): keep oversized transcript headers out of metadata admission --- crates/capture/src/lib.rs | 3 +- crates/pipeline/src/meta/inventory.rs | 5 +++ crates/pipeline/src/skim.rs | 19 +++++++-- crates/pipeline/src/stream/pass.rs | 8 +--- .../pipeline/tests/unfinished_first_record.rs | 39 ++++++++++++------- 5 files changed, 47 insertions(+), 27 deletions(-) diff --git a/crates/capture/src/lib.rs b/crates/capture/src/lib.rs index 02e2f8fb..c6ab7eca 100644 --- a/crates/capture/src/lib.rs +++ b/crates/capture/src/lib.rs @@ -101,6 +101,7 @@ impl CaptureError { /// A native source exists but cannot yet supply a complete decoded view. /// This is not EOF and must not activate an empty replacement. pub fn incomplete_input(&self) -> bool { - matches!(self, Self::Io { source, .. } if matches!(source.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof)) + matches!(self, Self::LimitExceeded(_)) + || matches!(self, Self::Io { source, .. } if matches!(source.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof)) } } diff --git a/crates/pipeline/src/meta/inventory.rs b/crates/pipeline/src/meta/inventory.rs index 8e6e8bb4..dcfd3e90 100644 --- a/crates/pipeline/src/meta/inventory.rs +++ b/crates/pipeline/src/meta/inventory.rs @@ -160,6 +160,11 @@ pub(super) fn read_inventory( report.files_unfinished += 1; instrument.add(Metric::FilesUnfinished, 1); } + crate::skim::Skimmed::Unidentified => { + report.files_skimmed += 1; + report.files_unidentified += 1; + instrument.add(Metric::FilesUnidentified, 1); + } crate::skim::Skimmed::Transcript(skim, cut) => { report.files_skimmed += 1; admit_skim( diff --git a/crates/pipeline/src/skim.rs b/crates/pipeline/src/skim.rs index 61755324..78463a89 100644 --- a/crates/pipeline/src/skim.rs +++ b/crates/pipeline/src/skim.rs @@ -34,6 +34,8 @@ pub(crate) enum Skimmed { /// A transcript no skim identifies whose first record no delimiter has /// ended yet: its writer is still writing it, so it is not yet a session. Unfinished, + /// No bounded opening record can identify this source. + Unidentified, /// A transcript's skim, after any bounded identity or title fallback, /// and the file's length and mtime the skim read it at. Transcript(Box, Cut), @@ -78,12 +80,21 @@ pub(crate) fn skim_transcript( instrument: &Instrument, ) -> Result { if let Some(location) = adapter.transcript_location(&file.relative_path) { - let head = match location - .source - .head(root, crate::MAX_PROJECTED_RECORD_BYTES) - { + let head = match instrument.time(Metric::SkimNanos, || { + location + .source + .head(root, crate::MAX_PROJECTED_RECORD_BYTES) + }) { Ok(Some(head)) => head, Ok(None) => return Ok(Skimmed::Vanished), + Err(rosetta_capture::CaptureError::LimitExceeded(_)) => { + instrument.add(Metric::FilesSkimmed, 1); + instrument.add( + Metric::BytesRead, + crate::MAX_PROJECTED_RECORD_BYTES as u64 + 1, + ); + return Ok(Skimmed::Unidentified); + } Err(error) if error.incomplete_input() => return Ok(Skimmed::Vanished), Err(error) => return Err(error.into()), }; diff --git a/crates/pipeline/src/stream/pass.rs b/crates/pipeline/src/stream/pass.rs index 55f33a27..3cc7438c 100644 --- a/crates/pipeline/src/stream/pass.rs +++ b/crates/pipeline/src/stream/pass.rs @@ -468,13 +468,7 @@ impl IngestPass { let read = match read { Ok(read) => read, Err(_) if opts.check_cancelled().is_err() => return Err(PipelineError::Cancelled), - Err(rosetta_capture::CaptureError::Io { source, .. }) - if file.input.is_some() - && matches!( - source.kind(), - std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof - ) => - { + Err(error) if file.input.is_some() && error.incomplete_input() => { deltas.retries_exhausted += 1; instrument.add(Metric::RetriesExhausted, 1); return Ok(self.stop_short(runtime, file, deltas)); diff --git a/crates/pipeline/tests/unfinished_first_record.rs b/crates/pipeline/tests/unfinished_first_record.rs index 7325fc8c..f274900f 100644 --- a/crates/pipeline/tests/unfinished_first_record.rs +++ b/crates/pipeline/tests/unfinished_first_record.rs @@ -120,24 +120,33 @@ fn a_rollout_still_writing_its_first_record_is_not_yet_a_session() { /// fresh gate tolerates both; the strict gate refuses both.) #[test] fn an_unterminated_opening_record_over_the_projection_limit_is_unidentified() { - let home = TempDir::new().unwrap(); - let root = home.path().join(".codex"); let mut record = String::from( "{\"timestamp\":\"2026-08-25T02:35:03.601Z\",\"type\":\"session_meta\",\ \"payload\":{\"id\":\"03c258e7-8a3a-7515-ae29-f6652985a62d\",\"base_instructions\":\"", ); record.push_str(&"q".repeat(rosetta_pipeline::MAX_PROJECTED_RECORD_BYTES)); - std::fs::write( - rollout(&root, "rollout-2026-08-24T21-00-00-huge.jsonl"), - &record, - ) - .unwrap(); - let mut store = Store::open_in_memory().unwrap(); - let (report, files) = meta(&root, &mut store); - assert_eq!( - (report.files_unfinished, report.files_unidentified), - (0, 1), - "{report:?}" - ); - assert!(files.is_empty()); + for compressed in [false, true] { + let home = TempDir::new().unwrap(); + let root = home.path().join(".codex"); + let (name, bytes) = if compressed { + ( + "rollout-2026-08-24T21-00-00-huge.jsonl.zst", + zstd::encode_all(record.as_bytes(), 3).unwrap(), + ) + } else { + ( + "rollout-2026-08-24T21-00-00-huge.jsonl", + record.as_bytes().to_vec(), + ) + }; + std::fs::write(rollout(&root, name), bytes).unwrap(); + let mut store = Store::open_in_memory().unwrap(); + let (report, files) = meta(&root, &mut store); + assert_eq!( + (report.files_unfinished, report.files_unidentified), + (0, 1), + "{report:?}" + ); + assert!(files.is_empty()); + } } From 0b6566657fc3cb59712a717e6576271a79a073a8 Mon Sep 17 00:00:00 2001 From: James Yong Date: Thu, 12 Nov 2026 16:15:21 -0800 Subject: [PATCH 6/6] test(rosettad): include all Codex transcript storage streams --- crates/rosettad/tests/adapter_optimization_matrix.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/rosettad/tests/adapter_optimization_matrix.rs b/crates/rosettad/tests/adapter_optimization_matrix.rs index 53c7752d..643268b5 100644 --- a/crates/rosettad/tests/adapter_optimization_matrix.rs +++ b/crates/rosettad/tests/adapter_optimization_matrix.rs @@ -185,7 +185,8 @@ fn every_driver_matches_its_declared_storage_contract() { // The summary database and the per-conversation database. ("antigravity", 1, 0, 2), ("claude-code", 3, 2, 0), - ("codex", 1, 0, 1), + // Plain/zstd representations in active and archived storage. + ("codex", 4, 0, 1), // The session-store database carries the per-request usage. ("copilot", 1, 1, 1), ("grok", 1, 2, 0), @@ -272,9 +273,9 @@ fn every_driver_matches_its_declared_storage_contract() { panic!("SQLite stream {kind}/{} has no snapshot plan", stream.id) }); assert!(!plan.sessions_sql.trim().is_empty(), "{kind}/{}", stream.id); - // A snapshot may be metadata-only (Antigravity's summary - // database or Codex's state database) and therefore have - // no timeline queries. + // A metadata-only snapshot may have no record queries, + // or capture native selectors without emitting canonical + // operations (Codex's state database). assert!( plan.record_queries.iter().all(|query| { !query.name.is_empty() && !query.sql.trim().is_empty()