Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
154 commits
Select commit Hold shift + click to select a range
e5feef2
perf(code-index): pin staged segment reads per preparation
ScriptedAlchemy Nov 8, 2026
cbe966b
fix(code-index): scope staged reader test imports to Unix
ScriptedAlchemy Nov 8, 2026
0d6c177
perf(code-index): expose staged open counts in tracing
ScriptedAlchemy Nov 8, 2026
b1cfe79
fix(code-index): refuse replaced pinned staging files
ScriptedAlchemy Nov 8, 2026
ce6566c
fix(code-index): poison replaced staging reads and count open attempts
ScriptedAlchemy Nov 8, 2026
4dfd243
fix(code-index): refuse publication after staged identity failure
ScriptedAlchemy Nov 8, 2026
38137c8
fix(code-index): recheck path identity without another open
ScriptedAlchemy Nov 8, 2026
21827ea
style(code-index): format staged segment reader regressions
ScriptedAlchemy Nov 8, 2026
262e340
fix(code-index): preserve staged read identity across mutations
ScriptedAlchemy Nov 8, 2026
b850dcb
fix(code-index): retain sealed identities after handle eviction
ScriptedAlchemy Nov 8, 2026
18269c6
test(code-index): verify staged reads through durable reload
ScriptedAlchemy Nov 8, 2026
103101b
style(code-index): format pinned segment read regression
ScriptedAlchemy Nov 8, 2026
c76238b
fix(code-index): resolve Bazel Clippy failures in staged reads
ScriptedAlchemy Nov 8, 2026
3286852
perf(storage): flush staged files through bounded writer handles
ScriptedAlchemy Nov 8, 2026
4d0ad18
fix(code-index): preserve cached handles on rejected reads
ScriptedAlchemy Nov 8, 2026
688f3a6
style(storage): satisfy Bazel formatting gate
ScriptedAlchemy Nov 8, 2026
70188c5
fix(code-index): never replace raced sealed segment names
ScriptedAlchemy Nov 8, 2026
7184d12
perf(code-index): avoid directory opens during publication checks
ScriptedAlchemy Nov 8, 2026
df3986a
fix(code-index): preserve immutable manifest publication and reuse fd
ScriptedAlchemy Nov 8, 2026
b3bc9ec
fix(code-index): use atomic immutable publish for manifest names
ScriptedAlchemy Nov 8, 2026
bcb039e
style(code-index): align staged segment reads with rustfmt
ScriptedAlchemy Nov 8, 2026
1d021a5
fix(code-index): refuse retained reads after I/O interruption
ScriptedAlchemy Nov 8, 2026
ae10ea5
style(code-index): format retained read interruption regression
ScriptedAlchemy Nov 8, 2026
0f9e7f2
fix(code-index): authenticate cold whole-segment reads
ScriptedAlchemy Nov 8, 2026
5a9edde
test(code-index): use valid fixture and canonical regression layout
ScriptedAlchemy Nov 8, 2026
1b53e6d
perf(storage): reject invalid batch members before window fsync
ScriptedAlchemy Nov 8, 2026
5f9c013
perf(code-index): reuse authenticated segment reads at commit
ScriptedAlchemy Nov 8, 2026
0d749b0
fix(code-index): preserve pending segment identity on dedup
ScriptedAlchemy Nov 8, 2026
108d864
style(code-index): apply pinned Rust format to staged regression
ScriptedAlchemy Nov 8, 2026
8a2ffc0
perf(code-index): reuse canonical segment witness on repeated staging
ScriptedAlchemy Nov 8, 2026
045bb99
fix(code-index): reject unflushed segment after staging failure
ScriptedAlchemy Nov 8, 2026
97814f3
style(code-index): format failed-admission regression
ScriptedAlchemy Nov 8, 2026
9b34ee7
style(code-index): format staged segment regressions
ScriptedAlchemy Nov 8, 2026
a65bea9
perf(code-index): authenticate staged bytes without commit opens
ScriptedAlchemy Nov 8, 2026
2200bd0
style(code-index): match hosted Bazel Rust formatting
ScriptedAlchemy Nov 8, 2026
9c08e11
fix(code-index): bind batch admission to authenticated writer identity
ScriptedAlchemy Nov 8, 2026
badb863
perf(code-index): validate active pointer through one open
ScriptedAlchemy Nov 8, 2026
e7ab03f
style(code-index): normalize pointer open regression formatting
ScriptedAlchemy Nov 8, 2026
da385bc
perf(code-index): reuse published pointer writer for memo validation
ScriptedAlchemy Nov 8, 2026
ed1786b
style(code-index): align staged publication with rustfmt
ScriptedAlchemy Nov 8, 2026
c851742
fix(code-index): retain digest for generation collision regression
ScriptedAlchemy Nov 8, 2026
1bcaf00
fix(code-index): authenticate evidence packs through writer handles
ScriptedAlchemy Nov 8, 2026
402923f
test(code-index): witness evidence pack writer identity
ScriptedAlchemy Nov 8, 2026
8dda2ad
fix(code-index): validate evidence pack file kind before digest
ScriptedAlchemy Nov 8, 2026
96daef5
test(code-index): exercise publication retries under a shared lease
ScriptedAlchemy Nov 8, 2026
650e1cc
fix(code-index): reject substituted staged packs before naming
ScriptedAlchemy Nov 8, 2026
8a683a3
test(automation): bind scheduler fixtures to backend identities
ScriptedAlchemy Nov 8, 2026
9e80313
perf(code-index): reuse parsed pointer bytes for publication CAS
ScriptedAlchemy Nov 8, 2026
08a2177
style(code-index): format publication and staged read changes
ScriptedAlchemy Nov 8, 2026
8ea3d31
fix(code-index): bind memoized pointer to named inode
ScriptedAlchemy Nov 8, 2026
c42535c
fix(code-index): scope Linux watcher trait and own test identity
ScriptedAlchemy Nov 8, 2026
46c5d97
style(code-index): satisfy Rustfmt for native pointer regressions
ScriptedAlchemy Nov 8, 2026
5f2e890
fix(code-index): authenticate cold pinned whole reads
ScriptedAlchemy Nov 9, 2026
95a93a8
style(code-index): satisfy staged corruption test rustfmt
ScriptedAlchemy Nov 9, 2026
262cf1f
test(code-index): compare the canonical pointer generation ID
ScriptedAlchemy Nov 9, 2026
5ffbb82
test(code-index): compare the captured pointer bytes explicitly
ScriptedAlchemy Nov 9, 2026
b42f7ca
chore(ci): stack staged segment reads on launcher distribution
ScriptedAlchemy Nov 9, 2026
4ea334e
Merge master into fix/4533-staged-segment-read-opens
ScriptedAlchemy Nov 9, 2026
e9b6e8e
fix(storage): recheck flushed windows after final barriers
ScriptedAlchemy Nov 9, 2026
835bd99
Merge current master into fix/4533-staged-segment-read-opens
ScriptedAlchemy Nov 9, 2026
76b7556
fix(code-index): poison staged reads after digest failures
ScriptedAlchemy Nov 9, 2026
d3b089d
Merge master into fix/4533-staged-segment-read-opens
ScriptedAlchemy Nov 9, 2026
ec0c903
style(storage): apply hosted Bazel rustfmt to durability regression
ScriptedAlchemy Nov 9, 2026
2e1b241
fix(code-index): preserve cached handles on failed cold opens
ScriptedAlchemy Nov 9, 2026
e18d295
fix(code-index): reject deduplication across swapped inodes
ScriptedAlchemy Nov 9, 2026
8e17075
fix(code-index): retain staged inode authority through deduplication
ScriptedAlchemy Nov 9, 2026
52b335d
fix(code-index): honor tracing capture policy in storage regressions
ScriptedAlchemy Nov 9, 2026
3d12658
style(code-index): format durability retry assertions
ScriptedAlchemy Nov 9, 2026
33d1b0c
style(code-index): apply hosted Bazel rustfmt corrections
ScriptedAlchemy Nov 9, 2026
0384da6
test(private-fs): exercise final fsync identity at owning target
ScriptedAlchemy Nov 9, 2026
6c0b52e
fix(storage): keep fsync pause crate-test-only
ScriptedAlchemy Nov 9, 2026
ff7015e
style(private-fs): apply hosted Bazel rustfmt corrections
ScriptedAlchemy Nov 9, 2026
dacc5fc
perf(code-index): reuse retained segment descriptors
ScriptedAlchemy Nov 9, 2026
b91f221
fix(code-index): format retained-read regression for CI
ScriptedAlchemy Nov 9, 2026
3bbff2b
perf(retention): reuse manifest descriptor across mark probe
ScriptedAlchemy Nov 9, 2026
b46aeeb
fix(code-index): preserve healthy handles on failed cold reads
ScriptedAlchemy Nov 9, 2026
8d260d5
fix(retention): preserve cancellation at manifest open boundary
ScriptedAlchemy Nov 9, 2026
598c1ff
perf(code-index): reuse retained segment handles across requests
ScriptedAlchemy Nov 9, 2026
9ea8a39
fix(code-index): apply hosted Bazel formatting corrections
ScriptedAlchemy Nov 9, 2026
61394f7
fix(code-index): retire cached descriptors before generation GC
ScriptedAlchemy Nov 9, 2026
43ca411
fix(code-index): format retention cache release for Bazel
ScriptedAlchemy Nov 9, 2026
fba00d6
perf(code-index): reuse authenticated publication pointer descriptor
ScriptedAlchemy Nov 9, 2026
0accee8
fix(code-index): close pointer handles before scope reset
ScriptedAlchemy Nov 9, 2026
63a01e7
fix(code-index): repair Bazel retention and reset regressions
ScriptedAlchemy Nov 10, 2026
ec15c30
fix(code-index): align cached handle type visibility
ScriptedAlchemy Nov 10, 2026
46c0141
fix(code-index): keep native Bazel retention tests compiling
ScriptedAlchemy Nov 10, 2026
a840b5c
perf(code-index): reuse historical generation lock descriptor
ScriptedAlchemy Nov 10, 2026
30747aa
test(code-index): prove cached reader lock exclusion and inode refusal
ScriptedAlchemy Nov 10, 2026
53b765c
fix(code-index): reject symlink aliases of cached read lock
ScriptedAlchemy Nov 10, 2026
20c5a35
fix(code-index): refuse symlinked pointer publication slots
ScriptedAlchemy Nov 10, 2026
ebb9f1d
fix(code-index): refuse changed cold pointer authorities
ScriptedAlchemy Nov 10, 2026
fcbe281
test(code-index): reauthenticate pointer ctime after rename
ScriptedAlchemy Nov 10, 2026
e86a4e5
fix(code-index): distinguish pointer replacement from corruption
ScriptedAlchemy Nov 10, 2026
33df990
style(code-index): apply hosted Bazel rustfmt corrections
ScriptedAlchemy Nov 10, 2026
b852ca0
fix(private-fs): revalidate completed durability windows
ScriptedAlchemy Nov 10, 2026
108b39f
style(private-fs): apply hosted Bazel formatting corrections
ScriptedAlchemy Nov 10, 2026
60f5bcd
fix(code-index): authenticate retention generation metadata reads
ScriptedAlchemy Nov 10, 2026
9176416
fix(code-index): restore census error typing and pass CI gates
ScriptedAlchemy Nov 10, 2026
333fa75
fix(code-index): model atomic pointer publication in runtime tests
ScriptedAlchemy Nov 10, 2026
035f4fe
fix(code-index): retry atomic pointer rotation during warm reads
ScriptedAlchemy Nov 10, 2026
d235689
fix(code-index): correct pointer witness method boundary
ScriptedAlchemy Nov 10, 2026
f97c7ac
style(code-index): apply hosted Bazel formatting repairs
ScriptedAlchemy Nov 10, 2026
c929b03
chore(code-index): integrate current master without file overlap
ScriptedAlchemy Nov 10, 2026
3029dc0
fix(private-fs): authenticate macOS durability writer paths
ScriptedAlchemy Nov 10, 2026
9cd370f
fix(private-fs): recheck macOS staged paths before publication
ScriptedAlchemy Nov 10, 2026
662a258
style(private-fs): format staged durability assertions
ScriptedAlchemy Nov 10, 2026
29f3a3e
fix(code-index): refuse stale Windows pointer readback
ScriptedAlchemy Nov 10, 2026
6c1f74f
style(code-index): format Windows pointer witnesses
ScriptedAlchemy Nov 10, 2026
6ac133d
perf(code-index): reuse Windows publication pointer handles
ScriptedAlchemy Nov 11, 2026
1a08e5b
fix(code-index): reload stable Windows pointer rotations
ScriptedAlchemy Nov 11, 2026
1d003ff
style(code-index): format pointer rotation regression
ScriptedAlchemy Nov 11, 2026
05eb005
test(global-db): remove redundant runtime reopens
ScriptedAlchemy Nov 11, 2026
e4cb82b
test(global-db): retire cancellation fixture before reopening
ScriptedAlchemy Nov 11, 2026
dd8e3f5
chore: merge master into staged segment reads
ScriptedAlchemy Nov 11, 2026
5c881a5
fix(storage): authenticate staged publication snapshots
ScriptedAlchemy Nov 11, 2026
5a5e94e
test(global-db): batch overlap reconciliation fixture writes
ScriptedAlchemy Nov 12, 2026
93845d1
test(storage): retire pinned handles before moving sealed fixtures
ScriptedAlchemy Nov 12, 2026
7e83286
test(code-index): settle owner before holding publication decode
ScriptedAlchemy Nov 12, 2026
8331e1c
chore(merge): reconcile staged segment reads with master
ScriptedAlchemy Nov 12, 2026
86bcab5
chore(merge): sync staged segment reads with master
ScriptedAlchemy Nov 12, 2026
a9f46b9
perf(private-fs): batch durability windows on every platform
ScriptedAlchemy Nov 13, 2026
e976b60
fix(index): keep staged admission checks inside durability batch
ScriptedAlchemy Nov 13, 2026
332c42b
test(daemon): retain shutdown logs in WAL failures
ScriptedAlchemy Nov 13, 2026
623a8c2
test(db): reuse published schema for cancellation sweep
ScriptedAlchemy Nov 13, 2026
0179b22
fix(code-index): publish over retained Windows pointers
ScriptedAlchemy Nov 13, 2026
ddb38e2
ci(windows): retain native test panic output
ScriptedAlchemy Nov 13, 2026
7197db5
style(code-index): format Windows pointer regression
ScriptedAlchemy Nov 13, 2026
936395f
ci: restore native test output settings
ScriptedAlchemy Nov 13, 2026
0510346
test(daemon): drive terminal retention continuations
ScriptedAlchemy Nov 13, 2026
fab9422
chore(merge): sync staged segment reads with master
ScriptedAlchemy Nov 13, 2026
0dd62a4
perf(test): avoid duplicate dashboard relation projections
ScriptedAlchemy Nov 14, 2026
91d9617
fix(code-index): fence escaped readers during segment retirement
ScriptedAlchemy Nov 14, 2026
00d1df6
chore(merge): sync staged segment reads with master
ScriptedAlchemy Nov 14, 2026
a0d6ba5
feat(code-index): store sealed generations in fixed payload packs
ScriptedAlchemy Nov 14, 2026
8ffab7f
fix(code-index): validate packed readers across platforms
ScriptedAlchemy Nov 14, 2026
3492dfa
fix(code-index): complete packed reader caller types
ScriptedAlchemy Nov 14, 2026
a82b15d
fix(code-index): retain bounded Windows pack read leases
ScriptedAlchemy Nov 14, 2026
3707de3
feat(private-fs): retain immutable Windows read leases
ScriptedAlchemy Nov 14, 2026
e021160
fix(code-index): complete packed validation callers
ScriptedAlchemy Nov 14, 2026
030cab6
Merge master into fix/4533-staged-segment-read-opens
ScriptedAlchemy Nov 14, 2026
30b556c
fix(code-index): anchor Windows stores before deriving pack paths
ScriptedAlchemy Nov 14, 2026
540286b
fix(code-index): align packed fixtures with current master
ScriptedAlchemy Nov 14, 2026
27d75a4
test(code-index): sweep released source entries at the pass boundary
ScriptedAlchemy Nov 14, 2026
3184dc6
chore(code-index): integrate the beta.103 release baseline
ScriptedAlchemy Nov 14, 2026
043eab1
test(code-index): corrupt the consumed packed file extent
ScriptedAlchemy Nov 14, 2026
9652377
chore(code-index): integrate the packed corruption fixture
ScriptedAlchemy Nov 14, 2026
85c6fbd
test(cli): retain external backend refusal context
ScriptedAlchemy Nov 14, 2026
92434e7
test(cli): retain cold daemon logs on configuration failures
ScriptedAlchemy Nov 14, 2026
393f484
chore(code-index): integrate release runner recovery
ScriptedAlchemy Nov 14, 2026
92b2a78
test(global-db): avoid unused raw fixture connections
ScriptedAlchemy Nov 14, 2026
74f4a86
test(global-db): assert initialization refusal directly
ScriptedAlchemy Nov 14, 2026
e712da6
chore(code-index): integrate merged submodule support
ScriptedAlchemy Nov 14, 2026
d848053
chore(code-index): integrate database fixture cleanup
ScriptedAlchemy Nov 14, 2026
1168133
chore(merge): sync staged segment reads with master
ScriptedAlchemy Nov 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1894,14 +1894,18 @@ fn automation_config_enable_writes_canonical_project_setting_noninteractively()
std::fs::write(project.path().join("src/lib.rs"), "pub fn marker() {}\n").unwrap();

init_project_fixture(home.path(), project.path());
let mut enable = tracedecay_command(home.path(), project.path());
let daemon_log = home.path().join("automation-config-daemon.log");
let _daemon = crate::common::spawn_tracedecay_daemon_logged(home.path(), &daemon_log, |_| {});
let mut enable = tracedecay_command_without_daemon(home.path(), project.path());
enable.args(["automation", "config", "enable"]);
let enable_output = run_with_timeout(enable, cli_timeout());
assert!(
enable_output.status.success(),
"automation config enable should succeed\nstdout:\n{}\nstderr:\n{}",
"automation config enable should succeed\nstdout:\n{}\nstderr:\n{}\ndaemon:\n{}",
String::from_utf8_lossy(&enable_output.stdout),
String::from_utf8_lossy(&enable_output.stderr)
String::from_utf8_lossy(&enable_output.stderr),
std::fs::read_to_string(&daemon_log)
.unwrap_or_else(|error| format!("daemon log unavailable: {error}"))
);
let payload: serde_json::Value = serde_json::from_slice(&enable_output.stdout)
.expect("automation config enable should print JSON");
Expand All @@ -1912,14 +1916,16 @@ fn automation_config_enable_writes_canonical_project_setting_noninteractively()
assert_eq!(payload["explanation"]["automatic_memory_apply"], true);
assert_eq!(payload["explanation"]["automatic_skill_activation"], true);

let mut explain = tracedecay_command(home.path(), project.path());
let mut explain = tracedecay_command_without_daemon(home.path(), project.path());
explain.args(["automation", "config", "explain", "--json"]);
let explain_output = run_with_timeout(explain, cli_timeout());
assert!(
explain_output.status.success(),
"automation config explain should succeed\nstdout:\n{}\nstderr:\n{}",
"automation config explain should succeed\nstdout:\n{}\nstderr:\n{}\ndaemon:\n{}",
String::from_utf8_lossy(&explain_output.stdout),
String::from_utf8_lossy(&explain_output.stderr)
String::from_utf8_lossy(&explain_output.stderr),
std::fs::read_to_string(&daemon_log)
.unwrap_or_else(|error| format!("daemon log unavailable: {error}"))
);
let explain_payload: serde_json::Value = serde_json::from_slice(&explain_output.stdout)
.expect("automation config explain should print JSON");
Expand All @@ -1943,7 +1949,9 @@ fn automation_config_set_rejects_unimplemented_external_backend() {
std::fs::write(project.path().join("src/lib.rs"), "pub fn marker() {}\n").unwrap();
init_project_fixture(home.path(), project.path());

let mut set = tracedecay_command(home.path(), project.path());
let daemon_log = home.path().join("automation-config-daemon.log");
let _daemon = crate::common::spawn_tracedecay_daemon_logged(home.path(), &daemon_log, |_| {});
let mut set = tracedecay_command_without_daemon(home.path(), project.path());
set.args([
"automation",
"config",
Expand All @@ -1954,13 +1962,29 @@ fn automation_config_set_rejects_unimplemented_external_backend() {
let output = run_with_timeout(set, cli_timeout());
assert!(
!output.status.success(),
"external backend should be rejected\nstdout:\n{}\nstderr:\n{}",
"external backend should be rejected\nstdout:\n{}\nstderr:\n{}\ndaemon:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
String::from_utf8_lossy(&output.stderr),
std::fs::read_to_string(&daemon_log)
.unwrap_or_else(|error| format!("daemon log unavailable: {error}"))
);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("unknown automation backend"));
assert!(stderr.contains("disabled, codex-app-server"));
assert!(
stderr.contains("unknown automation backend"),
"external backend should report its refusal\nstatus: {}\nstdout:\n{}\nstderr:\n{stderr}\ndaemon:\n{}",
output.status,
String::from_utf8_lossy(&output.stdout),
std::fs::read_to_string(&daemon_log)
.unwrap_or_else(|error| format!("daemon log unavailable: {error}"))
);
assert!(
stderr.contains("disabled, codex-app-server"),
"external backend refusal should list implemented choices\nstatus: {}\nstdout:\n{}\nstderr:\n{stderr}\ndaemon:\n{}",
output.status,
String::from_utf8_lossy(&output.stdout),
std::fs::read_to_string(&daemon_log)
.unwrap_or_else(|error| format!("daemon log unavailable: {error}"))
);
}

#[test]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::{BufReader, Read};
use std::io::{BufReader, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
Expand Down Expand Up @@ -112,7 +112,7 @@ use text_artifacts::{
plan_collectable_text_artifacts_cancellable, stage_collectable_text_artifacts,
};

use generation_scan::{read_generation_format_revision, read_generation_metadata};
use generation_scan::{read_generation_format_revision_from_file, read_generation_metadata};
#[cfg(test)]
use scope_quarantine::ScopeQuarantineAuthority;

Expand Down Expand Up @@ -1414,20 +1414,37 @@ fn sweep_unreferenced_generation_segments(
})?
.to_owned()
};
if read_generation_format_revision(&path, is_cancelled)?
// The revision probe and full descriptor mark must observe the
// same opened inode. Reopening here paid a second endpoint-security
// authorization for every current-revision manifest and could read
// different bytes if the pathname changed between the two opens.
// Cancellation must win immediately before endpoint-authorized
// open(), including when a listed generation disappeared meanwhile.
if is_cancelled() {
return Err(CodeGenerationRetentionErrorV1::Cancelled);
}
let mut file = File::open(&path).map_err(|error| {
if is_cancelled() {
CodeGenerationRetentionErrorV1::Cancelled
} else {
deferred_if_absent(error)
}
})?;
if read_generation_format_revision_from_file(&mut file, &path, is_cancelled)?
!= SEALED_GENERATION_FORMAT_REVISION_V1
{
continue;
}
file.seek(SeekFrom::Start(0)).map_err(storage)?;
let mut reader = CancellableGenerationManifestReaderV1 {
file: File::open(&path).map_err(deferred_if_absent)?,
file,
hasher: Sha256::new(),
is_cancelled,
cancelled: false,
};
let identities = {
let buffered = BufReader::with_capacity(1024 * 1024, &mut reader);
tracedecay_code_index::production::CodeIndexPublishedGenerationV1::partitioned_segment_identities_from_reader(
tracedecay_code_index::production::CodeIndexPublishedGenerationV1::partitioned_pack_identities_from_reader(
buffered,
)
};
Expand Down Expand Up @@ -1819,6 +1836,9 @@ pub fn execute_code_generation_retention_cancellable(
"active generation changed after the retention mark phase".to_owned(),
));
}
// Every active payload pass keeps its handles inside the shared store lease.
// Acquiring this exclusive lease therefore also proves that no reader can
// retain a payload inode while the following sweep retires its pathname.
let mut generation_segment_batch_exhausted = if plan.collectable_generations.is_empty()
&& plan.collectable_generation_segments == GenerationSegmentCensusV1::Present
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use std::fs::File;
use std::io::Read;
#[cfg(unix)]
use std::os::unix::fs::MetadataExt;
use std::path::Path;

use sha2::{Digest, Sha256};
Expand All @@ -14,14 +16,14 @@ use super::{
const MAX_FORMAT_REVISION_PREFIX_BYTES: usize = 4 * 1024;
const GENERATION_METADATA_READ_CHUNK_BYTES: usize = 4 * 1024;

pub(super) fn read_generation_format_revision(
pub(super) fn read_generation_format_revision_from_file(
file: &mut File,
path: &Path,
is_cancelled: &dyn Fn() -> bool,
) -> Result<u32, CodeGenerationRetentionErrorV1> {
if is_cancelled() {
return Err(CodeGenerationRetentionErrorV1::Cancelled);
}
let file = File::open(path).map_err(deferred_if_absent)?;
let mut prefix = Vec::with_capacity(MAX_FORMAT_REVISION_PREFIX_BYTES);
file.take(MAX_FORMAT_REVISION_PREFIX_BYTES as u64)
.read_to_end(&mut prefix)
Expand All @@ -47,8 +49,29 @@ pub(super) fn read_generation_metadata(
if is_cancelled() {
return Err(CodeGenerationRetentionErrorV1::Cancelled);
}
// Enumerating a content-addressed name does not authorize following an
// alias to another generation. This metadata witness adds no pathname open.
let named_before = path.symlink_metadata().map_err(deferred_if_absent)?;
if !named_before.file_type().is_file() {
let message = format!("generation file '{}' is not a regular file", path.display());
// A substituted link is an authority violation. Other non-files
// remain storage-shape failures, not missing publisher entries.
return Err(if named_before.file_type().is_symlink() {
CodeGenerationRetentionErrorV1::UnsafeState(message)
} else {
storage(message)
});
}
let mut file = File::open(path).map_err(deferred_if_absent)?;
let size_bytes = file.metadata().map_err(storage)?.len();
let opened_before = file.metadata().map_err(storage)?;
#[cfg(unix)]
if !same_generation_file_identity(&named_before, &opened_before) {
return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!(
"generation file '{}' changed while opening",
path.display()
)));
}
let size_bytes = opened_before.len();
let mut hasher = Sha256::new();
let (format_revision, manifest) =
read_generation_metadata_prefix(&mut file, path, verification, &mut hasher, is_cancelled)?;
Expand All @@ -75,9 +98,44 @@ pub(super) fn read_generation_metadata(
}
GenerationDigestVerificationV1::MetadataOnly => named_state_digest(path)?,
};
// The manifest, digest and reported size must belong to the pathname
// enumerated by retention, not a replaced inode or modified old handle.
// Metadata-only scans cannot fall back to trusting a substituted name.
let named_after = path.symlink_metadata().map_err(deferred_if_absent)?;
let opened_after = file.metadata().map_err(storage)?;
#[cfg(unix)]
let unchanged = same_generation_file_identity(&named_before, &named_after)
&& same_generation_file_identity(&named_before, &opened_after);
#[cfg(not(unix))]
let unchanged = named_after.file_type().is_file()
&& opened_after.is_file()
&& named_after.len() == size_bytes
&& opened_after.len() == size_bytes;
if !unchanged {
return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!(
"generation file '{}' changed during metadata read",
path.display()
)));
}
Ok((format_revision, manifest, state_digest, size_bytes))
}

#[cfg(unix)]
fn same_generation_file_identity(
original: &std::fs::Metadata,
current: &std::fs::Metadata,
) -> bool {
original.file_type().is_file()
&& current.file_type().is_file()
&& original.dev() == current.dev()
&& original.ino() == current.ino()
&& original.len() == current.len()
&& original.mtime() == current.mtime()
&& original.mtime_nsec() == current.mtime_nsec()
&& original.ctime() == current.ctime()
&& original.ctime_nsec() == current.ctime_nsec()
}

/// Read only as far as complete metadata requires, up to the existing bound.
/// Geometrically spaced probes keep repeated prefix parsing linear in bytes
/// even when a manifest is large. Reads stay small and cancellable between
Expand Down Expand Up @@ -233,6 +291,8 @@ mod tests {
use std::cell::Cell;
use std::fs::{self, File, OpenOptions};
use std::io::{Seek, SeekFrom, Write};
#[cfg(unix)]
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};

use sha2::{Digest, Sha256};
Expand All @@ -242,7 +302,8 @@ mod tests {
use super::{
CodeGenerationRetentionErrorV1, GENERATION_METADATA_READ_CHUNK_BYTES,
GenerationDigestVerificationV1, MAX_GENERATION_METADATA_PREFIX_BYTES, parse_json_u32_field,
read_generation_format_revision, read_generation_metadata, read_generation_metadata_prefix,
read_generation_format_revision_from_file, read_generation_metadata,
read_generation_metadata_prefix,
};

fn generation_bytes(manifest_padding: usize, payload_bytes: usize) -> Vec<u8> {
Expand Down Expand Up @@ -270,10 +331,13 @@ mod tests {
let root = tempfile::tempdir().unwrap();
let bytes = generation_bytes(0, 256 * 1024);
let (path, expected_digest) = write_generation(root.path(), &bytes);
let mut revision_file = File::open(&path).unwrap();
assert_eq!(
read_generation_format_revision(&path, &|| false).unwrap(),
read_generation_format_revision_from_file(&mut revision_file, &path, &|| false)
.unwrap(),
SEALED_GENERATION_FORMAT_REVISION_V1
);
drop(revision_file);
let mut file = File::open(&path).unwrap();
let (revision, manifest) = read_generation_metadata_prefix(
&mut file,
Expand Down Expand Up @@ -428,6 +492,80 @@ mod tests {
);
}

#[cfg(unix)]
#[test]
fn generation_metadata_refuses_symlink_aliases_in_both_modes() {
let root = tempfile::tempdir().unwrap();
let bytes = generation_bytes(0, 32 * 1024);
let (target, _) = write_generation(root.path(), &bytes);
let alias = root
.path()
.join(format!("generation-{}.json", "a".repeat(64)));
std::os::unix::fs::symlink(&target, &alias).unwrap();
assert_eq!(
fs::read(&alias).unwrap(),
bytes,
"alias resolves to valid bytes"
);

for mode in [
GenerationDigestVerificationV1::MetadataOnly,
GenerationDigestVerificationV1::Full,
] {
assert!(matches!(
read_generation_metadata(&alias, mode, &|| false),
Err(CodeGenerationRetentionErrorV1::UnsafeState(_))
));
}
let (_, _, digest, size) =
read_generation_metadata(&target, GenerationDigestVerificationV1::Full, &|| false)
.unwrap();
assert_eq!(
digest,
encode_tagged_lowercase_hex("sha256:", &Sha256::digest(&bytes))
);
assert_eq!(size, bytes.len() as u64);
}

#[cfg(unix)]
#[test]
fn generation_metadata_refuses_same_byte_inode_rotation_during_read() {
for mode in [
GenerationDigestVerificationV1::MetadataOnly,
GenerationDigestVerificationV1::Full,
] {
let root = tempfile::tempdir().unwrap();
let bytes = generation_bytes(0, 64 * 1024);
let (path, _) = write_generation(root.path(), &bytes);
let substitute = root.path().join("substitute.tmp");
fs::write(&substitute, &bytes).unwrap();
let original = fs::metadata(&path).unwrap();
let foreign = fs::metadata(&substitute).unwrap();
assert_ne!(
(original.dev(), original.ino()),
(foreign.dev(), foreign.ino()),
"the replacement must use a different real inode"
);
let probes = Cell::new(0_usize);
let result = read_generation_metadata(&path, mode, &|| {
let previous = probes.get();
probes.set(previous + 1);
// The third cancellation checkpoint is after reading the
// first nonempty prefix from the already-open descriptor.
if previous == 2 {
fs::rename(&substitute, &path).unwrap();
}
false
});
assert!(probes.get() >= 3, "the real reader must cross the swap");
assert!(matches!(
result,
Err(CodeGenerationRetentionErrorV1::UnsafeState(_))
));
assert_eq!(fs::read(&path).unwrap(), bytes);
}
}

#[test]
fn format_revision_requires_a_complete_unsigned_json_integer() {
for input in [
Expand Down
Loading
Loading