diff --git a/scripts/flake-check.mjs b/scripts/flake-check.mjs index 13a0bbd53b8..4274ca4065b 100644 --- a/scripts/flake-check.mjs +++ b/scripts/flake-check.mjs @@ -950,6 +950,92 @@ export function rustGroups(entries) { return { groups, runner: [...workspaces][0] }; } +// Only explicit per-test platform guards can excuse an unlisted test. Read +// the platforms nextest actually built, including cross-compilation targets; +// the Node host is not evidence of the test binary's target. Missing metadata, +// custom targets, unsupported cfg expressions and absent declarations stay +// unlisted failures. Feature predicates are deliberately never evaluated here. +function rustPlatformCfgs(listing, cache) { + const platforms = listing["rust-build-meta"]?.platforms; + if (!platforms?.host?.platform || !Array.isArray(platforms.targets)) + return undefined; + const built = [platforms.host, ...platforms.targets].map((p) => p.platform); + const cfgs = []; + for (const platform of built) { + if ( + !platform || + platform["custom-json"] || + platform["custom-cfg"] || + !/^[a-zA-Z0-9_-]+$/.test(platform.triple ?? "") + ) + return undefined; + if (!cache.has(platform.triple)) { + const result = run("rustc", [ + "--print", + "cfg", + "--target", + platform.triple, + ]); + cache.set( + platform.triple, + result.status === 0 + ? new Set(result.stdout.trim().split("\n")) + : undefined, + ); + } + const cfg = cache.get(platform.triple); + // Confirm real platform output before treating a missing predicate as false. + if (!cfg || ![...cfg].some((line) => line.startsWith('target_os="'))) + return undefined; + cfgs.push(cfg); + } + return cfgs; +} + +function rustPlatformExcluded(entry, cfgs) { + if (!cfgs?.length) return false; + const [file, ...parts] = entry.split("::"); + if (!existsSync(`${root}${file}`)) return false; + const source = readFileSync(`${root}${file}`, "utf8"); + const tests = rustTests(source).filter( + (test) => test.name === parts.join("::"), + ); + if (tests.length !== 1) return false; + const test = tests[0]; + const span = source + .split("\n") + .slice(test.start - 1, test.end) + .join("\n"); + const masked = blankRust(span); + const guards = []; + let at = /^\s*/.exec(masked)[0].length; + while (masked.startsWith("#[", at)) { + const close = matching(masked, "[", "]", at + 1); + const attr = span.slice(at + 2, close).trim(); + // Deliberately support only atomic built-in platform predicates and their + // negation. Complex/feature/custom predicates retain the existing failure. + const guard = + /^cfg\(\s*(windows|unix|target_(?:os|arch|family|env|vendor|endian|pointer_width)\s*=\s*"[\w-]+")\s*\)$/.exec( + attr, + ); + const negated = + /^cfg\(\s*not\(\s*(windows|unix|target_(?:os|arch|family|env|vendor|endian|pointer_width)\s*=\s*"[\w-]+")\s*\)\s*\)$/.exec( + attr, + ); + const value = guard?.[1] ?? negated?.[1]; + if (value) + guards.push({ + value: value.replace(/\s/g, ""), + negated: Boolean(negated), + }); + at = close + 1; + while (/\s/.test(masked[at] ?? "")) at += 1; + } + return cfgs.every((cfg) => + guards.some(({ value, negated }) => cfg.has(value) === negated), + ); +} + // nextest has no repeat flag, so run the selection once per iteration under // the `flake` profile (the workspace's .config/nextest.toml: no retries, JUnit // per run), one cargo invocation per package and feature graph so each builds @@ -963,7 +1049,8 @@ export function rustGroups(entries) { // keyed per graph, so one graph's pass cannot stand for another's failure. // An entry standing only for ignored or infrastructure-backed tests is // excluded; one no graph lists (a test behind a cfg the regular lanes do not -// build here) fails the lane rather than passing unrepeated. The Tauri +// build here) fails the lane unless an explicit platform guard proves it cannot +// run on any build platform. The Tauri // workspace is addressed by its manifest; the root one is cargo's default. function repeatRust(entries, iterations) { const { groups, runner } = rustGroups(entries); @@ -973,8 +1060,11 @@ function repeatRust(entries, iterations) { const cargoArgs = runner === "rust" ? [] : ["--manifest-path", manifest]; const invocations = []; const unlisted = []; + const platformExcluded = []; + const cfgCache = new Map(); for (const [crate, group] of groups) { const missing = new Set(group.tests); + const excluded = new Set(group.tests); for (const { features, args } of group.graphs) { const listed = run("cargo", [ "nextest", @@ -986,18 +1076,21 @@ function repeatRust(entries, iterations) { ]); if (listed.status !== 0) throw new Error(`cargo nextest list: ${listed.stderr}`); - const candidates = Object.entries( - JSON.parse(listed.stdout)["rust-suites"], - ).flatMap(([binary, suite]) => - Object.entries(suite.testcases).map(([name, test]) => ({ - binary, - name, - runnable: !test.ignored && !infraTest(suite["binary-name"], name), - })), + const listing = JSON.parse(listed.stdout); + const cfgs = rustPlatformCfgs(listing, cfgCache); + const candidates = Object.entries(listing["rust-suites"]).flatMap( + ([binary, suite]) => + Object.entries(suite.testcases).map(([name, test]) => ({ + binary, + name, + runnable: !test.ignored && !infraTest(suite["binary-name"], name), + })), ); const choice = chooseRust(group.tests, candidates); - for (const test of group.tests) + for (const test of group.tests) { if (!choice.unlisted.includes(test)) missing.delete(test); + else if (!rustPlatformExcluded(test.entry, cfgs)) excluded.delete(test); + } const chosen = choice.chosen.filter((test) => test.runnable); if (chosen.length) invocations.push({ @@ -1012,8 +1105,16 @@ function repeatRust(entries, iterations) { .join(" | "), }); } - unlisted.push(...[...missing].map((test) => test.entry)); + for (const test of missing) + (excluded.has(test) ? platformExcluded : unlisted).push(test.entry); } + if (platformExcluded.length) + summarize( + `## ${title}: ${platformExcluded.length} platform-excluded test(s)\n\n` + + "Explicit platform guards exclude these tests on every platform nextest built; they were not repeated:\n\n" + + platformExcluded.map((entry) => `- \`${entry}\``).join("\n") + + "\n\n", + ); if (unlisted.length) { summarize( [ @@ -1036,7 +1137,7 @@ function repeatRust(entries, iterations) { if (expected.length === 0) { if (!unlisted.length) summarize( - `## ${title}: nothing to repeat\n\nEvery selected test is ignored or infrastructure-backed.\n\n`, + `## ${title}: nothing to repeat\n\nEvery selected test is ignored, infrastructure-backed or explicitly excluded on the build platforms.\n\n`, ); return; } @@ -1135,8 +1236,9 @@ export function parseDartReport(text, mobileRoot = `${root}mobile/`) { const test = started.get(event.testID); if (!test) continue; const url = test.root_url ?? test.url; - const source = suites.get(test.suiteID) ?? - (url?.startsWith("file:") ? fileURLToPath(url) : url ?? "unknown"); + const source = + suites.get(test.suiteID) ?? + (url?.startsWith("file:") ? fileURLToPath(url) : (url ?? "unknown")); const file = source.replace(mobileRoot, ""); statuses.set( `${file} › ${test.name}`, diff --git a/scripts/flake-check.test.mjs b/scripts/flake-check.test.mjs index 304668d77a3..5c73f7930c4 100644 --- a/scripts/flake-check.test.mjs +++ b/scripts/flake-check.test.mjs @@ -1133,7 +1133,7 @@ test("repeat invocations mirror the lanes they stand in for", () => { // `tests` ({pkg, binary, binaryName, name, feature?, ignored?, failsWith?}; a // feature test only under that feature) and passing every test a run selects // but one failing in the graph enabling its `failsWith` feature. -function repeatWithFakeCargo(runner, entries, tests) { +function repeatWithFakeCargo(runner, entries, tests, options = {}) { const dir = realpathSync(mkdtempSync(join(tmpdir(), "flake-repeat-"))); try { mkdirSync(join(dir, "scripts")); @@ -1153,7 +1153,20 @@ function repeatWithFakeCargo(runner, entries, tests) { ); writeFileSync(join(dir, crateDir, "src/lib.rs"), ""); } + for (const [file, source] of Object.entries(options.sources ?? {})) { + mkdirSync(join(dir, file, ".."), { recursive: true }); + writeFileSync(join(dir, file), source); + } mkdirSync(join(dir, "bin")); + writeFileSync( + join(dir, "bin/rustc"), + `#!/usr/bin/env node +const target = process.argv.at(-1); +if (process.env.FAKE_CFG_FAILURE === "true") process.exit(1); +process.stdout.write(target.includes("windows") ? 'windows\\ntarget_os="windows"\\n' : 'unix\\ntarget_os="linux"\\n'); +`, + { mode: 0o755 }, + ); writeFileSync( join(dir, "fake-cargo.mjs"), `import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; @@ -1174,7 +1187,7 @@ if (args[1] === "list") { "filter-match": test.ignored ? { status: "mismatch", reason: "ignored" } : { status: "matches" }, }; } - process.stdout.write(JSON.stringify({ "rust-suites": suites })); + process.stdout.write(JSON.stringify({ "rust-suites": suites, "rust-build-meta": JSON.parse(process.env.FAKE_BUILD_META ?? "null") })); } else { const junit = join( args.includes("--manifest-path") ? dirname(value("--manifest-path")) : ".", @@ -1215,6 +1228,8 @@ if (args[1] === "list") { GITHUB_STEP_SUMMARY: summary, FAKE_CARGO_LOG: log, FAKE_CARGO_TESTS: JSON.stringify(tests), + FAKE_BUILD_META: JSON.stringify(options.metadata ?? null), + FAKE_CFG_FAILURE: String(Boolean(options.cfgFailure)), }, }, ); @@ -1445,3 +1460,128 @@ test("feature-gated Rust tests repeat in every graph the regular lanes test them ); assert.doesNotMatch(unlisted.summary, /nothing to repeat/); }); + +test("Rust repeats excuse only source-proven platform exclusions on the listed build platforms", () => { + const file = "crates/buzz-feature-flags/src/platform.rs"; + const entry = (name) => `${file}::tests::${name}`; + const suite = { + pkg: "buzz-feature-flags", + binary: "buzz-feature-flags", + binaryName: "buzz_feature_flags", + name: "platform::tests::plain", + }; + const platform = (triple) => ({ platform: { triple } }); + const linux = { + platforms: { host: platform("x86_64-unknown-linux-gnu"), targets: [] }, + }; + const sources = { + [file]: ` +#[cfg(test)] +mod tests { + #[cfg(windows)] + #[test] + fn windows_only() {} + #[test] + #[cfg(not(unix))] + fn non_unix() {} + #[cfg(target_os = "windows")] + #[test] + fn windows_os() {} + #[cfg(feature = "unbuilt")] + #[test] + fn feature_only() {} + #[cfg(custom_platform)] + #[test] + fn custom_guard() {} + #[cfg(any(windows, feature = "unbuilt"))] + #[test] + fn ambiguous_guard() {} + // #[cfg(windows)] + #[test] + fn commented_guard() { let text = "#[cfg(windows)]"; } + #[test] + fn plain() {} +} +`, + }; + const excluded = ["windows_only", "non_unix", "windows_os"].map(entry); + const mixed = repeatWithFakeCargo( + "rust", + [...excluded, entry("plain")], + [suite], + { sources, metadata: linux }, + ); + assert.equal(mixed.status, 0, mixed.summary); + assert.match(mixed.summary, /3 platform-excluded test\(s\)/); + assert.match(mixed.summary, /0 of 2 iterations failed/); + assert.equal(mixed.calls.filter((call) => call[1] === "run").length, 2); + assert.ok( + mixed.calls + .filter((call) => call[1] === "run") + .every( + (call) => + call.at(-1).includes("platform::tests::plain") && + !call.at(-1).includes("windows_only"), + ), + ); + + const only = repeatWithFakeCargo("rust", excluded, [], { + sources, + metadata: linux, + }); + assert.equal(only.status, 0, only.summary); + assert.match(only.summary, /3 platform-excluded test\(s\)/); + assert.match(only.summary, /nothing to repeat/); + assert.ok(only.calls.every((call) => call[1] === "list")); + + const unsupported = [ + "feature_only", + "custom_guard", + "ambiguous_guard", + "commented_guard", + "windows_but_missing", + ].map(entry); + const unknown = repeatWithFakeCargo("rust", unsupported, [], { + sources, + metadata: linux, + }); + assert.equal(unknown.status, 1, unknown.summary); + assert.match(unknown.summary, /5 selected test\(s\) not listed/); + assert.doesNotMatch(unknown.summary, /platform-excluded/); + for (const selected of unsupported) + assert.ok(unknown.summary.includes(selected)); + + // A Node process on Linux cannot excuse a missing test built for Windows. + // Include the host too, conservatively: binaries may also be built for it. + for (const metadata of [ + { platforms: { host: platform("x86_64-pc-windows-msvc"), targets: [] } }, + { + platforms: { + ...linux.platforms, + targets: [platform("x86_64-pc-windows-msvc")], + }, + }, + { + platforms: { + ...linux.platforms, + host: { platform: { triple: "custom", "custom-json": "{}" } }, + }, + }, + undefined, + ]) { + const result = repeatWithFakeCargo("rust", [entry("windows_only")], [], { + sources, + metadata, + }); + assert.equal(result.status, 1, result.summary); + assert.match(result.summary, /1 selected test\(s\) not listed/); + assert.doesNotMatch(result.summary, /platform-excluded/); + } + const failedCfg = repeatWithFakeCargo("rust", excluded, [], { + sources, + metadata: linux, + cfgFailure: true, + }); + assert.equal(failedCfg.status, 1, failedCfg.summary); + assert.match(failedCfg.summary, /3 selected test\(s\) not listed/); +});