From 80c91c1b4c51cf308a8eb86a22fa86c510e0d02f Mon Sep 17 00:00:00 2001 From: Colin Neilens Date: Wed, 14 Oct 2026 10:59:45 -0700 Subject: [PATCH 1/3] Validate offline UI scenarios against external claim authority Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 017c733a-252b-471a-9c43-26a6a67554df Signed-off-by: Colin Neilens --- Tools/ci/tests/classify-changes.test.sh | 6 + Tools/windows/Test-UiScenario.ps1 | 334 +++++++++++ Tools/windows/Tests/UiScenario.Tests.ps1 | 519 ++++++++++++++++++ .../windows/Tests/ValidationRunner.Tests.ps1 | 12 + windows-tests/ui-scenarios/README.md | 194 +++++++ .../ui-scenarios/claim-authority.schema.json | 52 ++ .../ui-scenarios/fixture.schema.json | 23 + .../ui-scenarios/scenario.schema.json | 173 ++++++ 8 files changed, 1313 insertions(+) create mode 100644 Tools/windows/Test-UiScenario.ps1 create mode 100644 Tools/windows/Tests/UiScenario.Tests.ps1 create mode 100644 windows-tests/ui-scenarios/README.md create mode 100644 windows-tests/ui-scenarios/claim-authority.schema.json create mode 100644 windows-tests/ui-scenarios/fixture.schema.json create mode 100644 windows-tests/ui-scenarios/scenario.schema.json diff --git a/Tools/ci/tests/classify-changes.test.sh b/Tools/ci/tests/classify-changes.test.sh index 4ffd12f9..84a30962 100755 --- a/Tools/ci/tests/classify-changes.test.sh +++ b/Tools/ci/tests/classify-changes.test.sh @@ -53,6 +53,12 @@ expect "Windows validation tooling" true false false \ Tools/windows/validate.ps1 expect "Windows production tests" true false false \ windows-tests/WindowsDaemonTests.swift +expect "UI scenario schema" true false false \ + windows-tests/ui-scenarios/scenario.schema.json +expect "UI claim scenario path" true false false \ + windows-tests/ui-scenarios/818-C1.json +expect "offline UI scenario validator" true false false \ + Tools/windows/Test-UiScenario.ps1 expect "Windows spike package" true false false \ investigation/spikes/swift-contracts/Package.swift expect "TDD evidence tooling validated by Windows suite" true false false \ diff --git a/Tools/windows/Test-UiScenario.ps1 b/Tools/windows/Test-UiScenario.ps1 new file mode 100644 index 00000000..5e8fa4f0 --- /dev/null +++ b/Tools/windows/Test-UiScenario.ps1 @@ -0,0 +1,334 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $ScenarioRoot, + [Parameter(Mandatory)][string] $AuthorityPath, + [Parameter(Mandatory)][string] $AuthoritySha256, + [string[]] $ClaimId +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$utf8 = [Text.UTF8Encoding]::new($false, $true) +$schemaRoot = Join-Path $PSScriptRoot "..\..\windows-tests\ui-scenarios" +$states = @("success", "empty", "error", "disabled", "persistence-after-relaunch", "keyboard") +$hasClaimSelection = $PSBoundParameters.ContainsKey("ClaimId") + +function Stop-UiScenario([string] $code) { + $error = [InvalidOperationException]::new($code) + $error.Data["UiScenarioCode"] = $code + throw $error +} + +function Assert-UiScenarioPath([string] $path) { + try { + if ($path.StartsWith("\\") -or $path.Contains(":") -and $path.Substring(2).Contains(":")) { + Stop-UiScenario "UnsafePath" + } + $full = [IO.Path]::GetFullPath($path) + $current = $full + while ($current) { + $item = Get-Item -LiteralPath $current -Force + if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + Stop-UiScenario "UnsafePath" + } + $current = [IO.Path]::GetDirectoryName($current) + } + return $full + } catch { + Stop-UiScenario "UnsafePath" + } +} + +function Get-UiScenarioHash([byte[]] $bytes) { + return [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() +} + +function Assert-UiScenarioKeys([Text.Json.JsonElement] $element) { + switch ($element.ValueKind) { + Object { + $names = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($property in $element.EnumerateObject()) { + if (-not $names.Add($property.Name)) { Stop-UiScenario "DuplicateProperty" } + Assert-UiScenarioKeys $property.Value + } + } + Array { + foreach ($child in $element.EnumerateArray()) { Assert-UiScenarioKeys $child } + } + } +} + +function Read-UiScenarioJson([string] $path) { + $full = Assert-UiScenarioPath $path + $document = $null + try { + $stream = [IO.File]::OpenRead($full) + try { + if ($stream.Length -eq 0 -or $stream.Length -gt 1MB) { Stop-UiScenario "DocumentSize" } + $bytes = [byte[]]::new([int]$stream.Length) + $stream.ReadExactly($bytes, 0, $bytes.Length) + if ($stream.ReadByte() -ne -1) { Stop-UiScenario "DocumentSize" } + } finally { $stream.Dispose() } + $text = $utf8.GetString($bytes) + $options = [Text.Json.JsonDocumentOptions]::new() + $options.MaxDepth = 32 + $document = [Text.Json.JsonDocument]::Parse($text, $options) + if ($document.RootElement.ValueKind -ne [Text.Json.JsonValueKind]::Object) { + Stop-UiScenario "DocumentRoot" + } + Assert-UiScenarioKeys $document.RootElement + return @{ + text = $text + hash = Get-UiScenarioHash $bytes + data = ConvertFrom-Json -InputObject $text -AsHashtable -DateKind String -Depth 32 + } + } catch { + if ($_.Exception.Data.Contains("UiScenarioCode")) { throw } + Stop-UiScenario "InvalidJson" + } finally { + if ($null -ne $document) { $document.Dispose() } + } +} + +function Assert-UiScenarioLocalRefs($node) { + if ($node -is [Collections.IDictionary]) { + foreach ($key in $node.Keys) { + if ($key -ceq '$ref' -and $node[$key] -cnotmatch '^#/definitions/[A-Za-z]+\z') { + Stop-UiScenario "UnsafeSchemaReference" + } + Assert-UiScenarioLocalRefs $node[$key] + } + } elseif ($node -is [array]) { + foreach ($child in $node) { Assert-UiScenarioLocalRefs $child } + } +} + +function Assert-UiScenarioSchema($json, [string] $schemaName) { + $schema = Read-UiScenarioJson (Join-Path $schemaRoot $schemaName) + Assert-UiScenarioLocalRefs $schema.data + try { + $valid = Test-Json -Json $json.text -Schema $schema.text -ErrorAction SilentlyContinue + } catch { Stop-UiScenario "SchemaInvalid" } + if (-not $valid) { Stop-UiScenario "SchemaInvalid" } + return $schema.hash +} + +function Assert-UiScenarioAuthority($authority) { + if ($AuthoritySha256 -cnotmatch '^[a-f0-9]{64}\z' -or $authority.hash -cne $AuthoritySha256) { + Stop-UiScenario "ClaimAuthorityMismatch" + } + $data = $authority.data + $expectedUrl = "https://github.com/scgopi/GraphCode/issues/$($data.issueNumber)#issuecomment-$($data.sourceCommentId)" + if ($data.sourceUrl -cne $expectedUrl) { Stop-UiScenario "ClaimAuthorityMismatch" } + $timestamp = [DateTimeOffset]::MinValue + if (-not [DateTimeOffset]::TryParseExact($data.sourceUpdatedAt, + [string[]]@("yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss.FFFFFFF'Z'"), + [Globalization.CultureInfo]::InvariantCulture, + [Globalization.DateTimeStyles]::AssumeUniversal, [ref]$timestamp)) { + Stop-UiScenario "ClaimAuthorityMismatch" + } + $claims = @{} + foreach ($claim in $data.claims) { + if (-not $claim.id.StartsWith("$($data.issueNumber)-C", [StringComparison]::Ordinal) -or + (Get-UiScenarioHash $utf8.GetBytes($claim.criterion)) -cne $claim.criterionSha256) { + Stop-UiScenario "ClaimAuthorityMismatch" + } + if ($claims.ContainsKey($claim.id)) { Stop-UiScenario "DuplicateClaim" } + $claims[$claim.id] = $claim + } + return $claims +} + +function Resolve-UiScenarioFixture($scenario, [string] $root) { + $fixture = $scenario.fixture + $directory = Join-Path $root ("fixtures\" + $fixture.id) + $manifest = Read-UiScenarioJson (Join-Path $directory "manifest.json") + $schemaHash = Assert-UiScenarioSchema $manifest "fixture.schema.json" + if ($manifest.hash -cne $fixture.manifestSha256 -or + $manifest.data.fixtureId -cne $fixture.id -or $manifest.data.fixtureVersion -ne $fixture.version) { + Stop-UiScenario "FixtureMismatch" + } + $paths = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($file in $manifest.data.files) { + $segments = $file.path -split '[\\/]' + foreach ($segment in $segments) { + if ($segment -in @("", ".", "..") -or $segment -match '[<>:"|?*\x00-\x1f]' -or + $segment.EndsWith(".") -or $segment.EndsWith(" ") -or + $segment -match '^(?i:CON|PRN|AUX|NUL|COM[0-9\u00b9\u00b2\u00b3]|LPT[0-9\u00b9\u00b2\u00b3])(?:[ .]|$)' -or + $segment -match '^(?i:CONIN\$|CONOUT\$|CLOCK\$)(?:\.|$)') { + Stop-UiScenario "UnsafeFixturePath" + } + } + $relative = $segments -join "\" + if (-not $paths.Add($relative)) { Stop-UiScenario "DuplicateFixtureFile" } + $full = Assert-UiScenarioPath (Join-Path $directory $relative) + if ((Get-Item -LiteralPath $full).PSIsContainer) { Stop-UiScenario "UnsafeFixturePath" } + if ((Get-FileHash -LiteralPath $full -Algorithm SHA256).Hash.ToLowerInvariant() -cne $file.sha256) { + Stop-UiScenario "FixtureFileMismatch" + } + } + return [ordered]@{ id = $fixture.id; version = $fixture.version; manifestSha256 = $manifest.hash + fileCount = $paths.Count; schemaSha256 = $schemaHash } +} + +function Assert-UiScenarioRootAnchor($target) { + if ($target.scope -cne "owned-root" -or $target.controlType -cne "Window") { + Stop-UiScenario "HealthyRootRequired" + } +} + +function Assert-UiScenarioAssertion($assertion) { + if ($assertion.kind -ceq "absent") { + Assert-UiScenarioRootAnchor $assertion.healthyRoot + if ($assertion.target.automationId -ceq $assertion.healthyRoot.automationId -and + $assertion.target.scope -ceq $assertion.healthyRoot.scope) { + Stop-UiScenario "HealthyRootRequired" + } + } +} + +function Get-UiScenarioCoverage($scenario, $claim) { + if ($claim.proofClass -ceq "visual" -or + ($scenario.coverage -ceq "complete-semantic" -and $claim.proofClass -cne "semantic-uia")) { + Stop-UiScenario "ProofClassMismatch" + } + $caseIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $covered = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $caseCount = 0 + $assertionCount = 0 + $budget = 0L + foreach ($case in $scenario.cases) { + if (-not $caseIds.Add($case.id)) { Stop-UiScenario "DuplicateCase" } + if (-not $claim.states[$case.state].required) { Stop-UiScenario "StateNotApplicable" } + $null = $covered.Add($case.state) + Assert-UiScenarioRootAnchor $case.preconditions.readyRoot + $allocated = [long]$case.preconditions.timeoutMs + $case.cleanup.timeoutMs + foreach ($assertion in @($case.preconditions.initial) + @($case.expected)) { + Assert-UiScenarioAssertion $assertion + $assertionCount++ + } + foreach ($step in $case.steps) { + $allocated += $step.timeoutMs + if ($step.operation -ceq "send-keys" -and $case.tier -cne "interactive") { + Stop-UiScenario "InputTierRequired" + } + if ($step.operation -ceq "wait-for") { + Assert-UiScenarioAssertion $step.expected + $assertionCount++ + } + if ($step.operation -ceq "relaunch") { Assert-UiScenarioRootAnchor $step.readyRoot } + } + if ($case.state -ceq "keyboard" -and + @($case.steps | Where-Object operation -CEQ "send-keys").Count -eq 0) { + Stop-UiScenario "KeyboardStepRequired" + } + if ($case.state -ceq "persistence-after-relaunch" -and + @($case.steps | Where-Object operation -CEQ "relaunch").Count -eq 0) { + Stop-UiScenario "RelaunchStepRequired" + } + if ($allocated -gt $case.timeoutMs) { Stop-UiScenario "CaseBudgetExceeded" } + $budget += $case.timeoutMs + $caseCount++ + } + if ($budget -gt 180000) { Stop-UiScenario "ScenarioBudgetExceeded" } + $uncovered = @($states | Where-Object { $claim.states[$_].required -and -not $covered.Contains($_) }) + if ($scenario.coverage -ceq "complete-semantic" -and $uncovered.Count -gt 0) { + Stop-UiScenario "IncompleteSemanticCoverage" + } + $components = @(switch ($claim.proofClass) { + "mixed" { @("non-uia-evidence-required-by-criterion") } + "input" { @("input-transport-and-end-state", "physical-input-if-required-by-criterion") } + default { @() } + }) + return [ordered]@{ claimId = $scenario.claimId; coverage = $scenario.coverage; proofClass = $claim.proofClass + coveredStates = @($states | Where-Object { $covered.Contains($_) }) + uncoveredRequiredStates = $uncovered; uncoveredProofComponents = @($components) + caseCount = $caseCount; assertionCount = $assertionCount; budgetMs = $budget } +} + +function Test-UiScenarioSet { + $root = Assert-UiScenarioPath $ScenarioRoot + if (-not (Get-Item -LiteralPath $root).PSIsContainer) { Stop-UiScenario "ScenarioRoot" } + $authority = Read-UiScenarioJson $AuthorityPath + $authoritySchemaHash = Assert-UiScenarioSchema $authority "claim-authority.schema.json" + $claims = Assert-UiScenarioAuthority $authority + $requested = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + if ($hasClaimSelection) { + if ($ClaimId.Count -eq 0) { Stop-UiScenario "EmptySelection" } + foreach ($id in $ClaimId) { + if ($id -cnotmatch '^[1-9][0-9]*-C[1-9][0-9]*\z') { Stop-UiScenario "UnknownClaim" } + if (-not $requested.Add($id)) { Stop-UiScenario "DuplicateRequest" } + if (-not $claims.ContainsKey($id)) { Stop-UiScenario "UnknownClaim" } + } + } + $scenarios = @{} + $queue = [Collections.Generic.Queue[string]]::new() + $queue.Enqueue($root) + $scenarioSchemaHash = $null + while ($queue.Count -gt 0) { + $directory = $queue.Dequeue() + foreach ($item in Get-ChildItem -LiteralPath $directory -Force) { + if ($directory -ceq $root -and $item.PSIsContainer -and $item.Name -ceq "fixtures") { continue } + $null = Assert-UiScenarioPath $item.FullName + if ($item.PSIsContainer) { $queue.Enqueue($item.FullName); continue } + if ($item.Extension -ine ".json") { continue } + if ($directory -ceq $root -and + $item.Name -cin @("scenario.schema.json", "claim-authority.schema.json", "fixture.schema.json")) { continue } + $json = Read-UiScenarioJson $item.FullName + $scenarioSchemaHash = Assert-UiScenarioSchema $json "scenario.schema.json" + $id = $json.data.claimId + if ($scenarios.ContainsKey($id)) { Stop-UiScenario "DuplicateClaimFile" } + if ($directory -cne $root -or $item.Name -cne "$id.json") { Stop-UiScenario "ScenarioFileIdentity" } + $scenarios[$id] = $json + } + } + if ($scenarios.Count -eq 0) { Stop-UiScenario "EmptySelection" } + foreach ($id in $requested) { + if (-not $scenarios.ContainsKey($id)) { Stop-UiScenario "MissingScenario" } + } + $selectedIds = @(if ($requested.Count -gt 0) { @($requested | Sort-Object -CaseSensitive) } else { + @($scenarios.Keys | Sort-Object -CaseSensitive) + }) + $coverage = @() + $fixtures = @() + $scenarioHashes = [ordered]@{} + $caseCount = 0 + $assertionCount = 0 + foreach ($id in $selectedIds) { + $json = $scenarios[$id] + if (-not $claims.ContainsKey($id) -or $json.data.authoritySha256 -cne $authority.hash) { + Stop-UiScenario "ClaimAuthorityMismatch" + } + $fixtures += Resolve-UiScenarioFixture $json.data $root + $report = Get-UiScenarioCoverage $json.data $claims[$id] + $coverage += $report + $scenarioHashes[$id] = $json.hash + $caseCount += $report.caseCount + $assertionCount += $report.assertionCount + } + if ($selectedIds.Count -eq 0 -or $caseCount -eq 0 -or $assertionCount -eq 0) { + Stop-UiScenario "EmptySelection" + } + return [ordered]@{ + status = "valid-data" + schemaVersion = 1 + authoritySha256 = $authority.hash + schemaHashes = @{ authority = $authoritySchemaHash; scenario = $scenarioSchemaHash } + scenarioHashes = $scenarioHashes + documentCount = $selectedIds.Count + caseCount = $caseCount + assertionCount = $assertionCount + selectedClaimIds = $selectedIds + fixtures = $fixtures + coverage = $coverage + } +} + +try { + Test-UiScenarioSet | ConvertTo-Json -Depth 32 -Compress + exit 0 +} catch { + $code = if ($_.Exception.Data.Contains("UiScenarioCode")) { $_.Exception.Data["UiScenarioCode"] } else { "ValidationError" } + [Console]::Error.WriteLine("UiScenario: $code (offline data validation refused)") + exit 1 +} diff --git a/Tools/windows/Tests/UiScenario.Tests.ps1 b/Tools/windows/Tests/UiScenario.Tests.ps1 new file mode 100644 index 00000000..43a1da43 --- /dev/null +++ b/Tools/windows/Tests/UiScenario.Tests.ps1 @@ -0,0 +1,519 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest +$repoRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\..\..")) +$validator = Join-Path $repoRoot "Tools\windows\Test-UiScenario.ps1" +$pwsh = (Get-Command pwsh -ErrorAction Stop).Source +$scratch = Join-Path ([IO.Path]::GetTempPath()) ("gc-ui-contract-" + [guid]::NewGuid().ToString("N")) +$utf8 = [Text.UTF8Encoding]::new($false) +$results = [Collections.Generic.List[object]]::new() + +function Assert-Contract([bool] $condition, [string] $message) { + if (-not $condition) { throw "RED: $message" } +} +function Write-Json([string] $path, $data) { + [IO.File]::WriteAllText($path, (ConvertTo-Json -InputObject $data -Depth 32 -Compress), $utf8) +} +function Hash-File([string] $path) { return (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant() } +function Hash-Text([string] $text) { + return [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($utf8.GetBytes($text))).ToLowerInvariant() +} +function Invoke-Validator([hashtable] $arguments = @{}) { + $parameters = @{ ScenarioRoot = $root; AuthorityPath = $authorityPath; AuthoritySha256 = $authorityHash } + foreach ($key in $arguments.Keys) { $parameters[$key] = $arguments[$key] } + $serialized = ConvertTo-Json -InputObject $parameters -Compress + $escaped = $serialized.Replace("'", "''") + $command = "`$p = ConvertFrom-Json -AsHashtable '$escaped'; & '$($validator.Replace("'", "''"))' @p" + $start = [Diagnostics.ProcessStartInfo]::new($pwsh) + $start.UseShellExecute = $false + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($arg in @("-NoProfile", "-NonInteractive", "-EncodedCommand", + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($command)))) { $start.ArgumentList.Add($arg) } + $process = [Diagnostics.Process]::Start($start) + try { + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(20000)) { + $process.Kill($true) + $process.WaitForExit() + throw "Validator child exceeded 20 seconds" + } + return @{ exitCode = $process.ExitCode; output = $stdout.GetAwaiter().GetResult().Trim() + error = $stderr.GetAwaiter().GetResult() } + } finally { $process.Dispose() } +} +function Reset-Inputs { + if (Test-Path -LiteralPath $root) { Remove-Item -LiteralPath $root -Recurse -Force } + $null = New-Item -ItemType Directory -Path $fixtureRoot + [IO.File]::WriteAllText($dataPath, $baseData, $utf8) + $script:authority = ConvertFrom-Json $baseAuthority -AsHashtable -DateKind String + $script:manifest = ConvertFrom-Json $baseManifest -AsHashtable + $script:scenario = ConvertFrom-Json $baseScenario -AsHashtable + Save-Inputs +} +function Save-Inputs { + Write-Json $manifestPath $manifest + Write-Json $authorityPath $authority + $script:authorityHash = Hash-File $authorityPath + $scenario.authoritySha256 = $authorityHash + $scenario.fixture.manifestSha256 = Hash-File $manifestPath + Write-Json $scenarioPath $scenario +} +function Refuse([string] $code, [hashtable] $arguments = @{}) { + $result = Invoke-Validator $arguments + Assert-Contract ($result.exitCode -ne 0 -and $result.error.Contains("UiScenario: $code ")) ` + "expected $code refusal, exit=$($result.exitCode) error=$($result.error)" + Assert-Contract (-not ($result.output + $result.error).Contains("SECRET_CANARY")) "private input leaked" +} +function Accept([hashtable] $arguments = @{}) { + $result = Invoke-Validator $arguments + Assert-Contract ($result.exitCode -eq 0 -and $result.error.Length -eq 0) "positive data refused: $($result.error)" + $summary = ConvertFrom-Json $result.output + Assert-Contract ($summary.status -ceq "valid-data" -and $summary.documentCount -gt 0 -and + $summary.caseCount -gt 0 -and $summary.assertionCount -gt 0 -and + $summary.authoritySha256 -ceq $authorityHash) "positive structured counts/binding absent" + Assert-Contract ($result.output -notmatch 'verified|verdict|SECRET_CANARY') "data report implies UI verification" + return $summary +} +function Test-Contract([string] $name, [scriptblock] $body) { + Reset-Inputs + & $body + $results.Add(@{ name = $name; passed = $true }) + Write-Host "PASS: $name" +} + +try { + $root = Join-Path $scratch "scenarios" + $fixtureRoot = Join-Path $root "fixtures\synthetic" + $null = New-Item -ItemType Directory -Path $fixtureRoot + $authorityPath = Join-Path $scratch "external-authority.json" + $scenarioPath = Join-Path $root "818-C1.json" + $dataPath = Join-Path $fixtureRoot "data.txt" + [IO.File]::WriteAllText($dataPath, "Synthetic offline fixture, not product evidence.", $utf8) + $manifest = @{ schemaVersion = 1; fixtureId = "synthetic"; fixtureVersion = 1 + files = @(@{ path = "data.txt"; sha256 = Hash-File $dataPath }) } + $manifestPath = Join-Path $fixtureRoot "manifest.json" + Write-Json $manifestPath $manifest + $states = @{} + foreach ($state in @("success", "empty", "error", "disabled", "persistence-after-relaunch", "keyboard")) { + $states[$state] = @{ required = ($state -ceq "success"); reason = "Synthetic contract applicability only." } + } + $criterion = "Synthetic semantic claim; no actual product acceptance statement." + $authority = @{ schemaVersion = 1; repository = "scgopi/GraphCode"; issueNumber = 818 + sourceUrl = "https://github.com/scgopi/GraphCode/issues/818#issuecomment-1"; sourceCommentId = 1 + sourceUpdatedAt = "2026-10-14T00:00:00Z"; sourceBodySha256 = Hash-Text "Synthetic source body." + claims = @(@{ id = "818-C1"; criterion = $criterion; criterionSha256 = Hash-Text $criterion + proofClass = "semantic-uia"; states = $states }) } + Write-Json $authorityPath $authority + $authorityHash = Hash-File $authorityPath + $target = @{ automationId = "synthetic-button"; controlType = "Button"; scope = "owned-root"; matchCount = 1 } + $anchor = @{ automationId = "synthetic-root"; controlType = "Window"; scope = "owned-root"; matchCount = 1 } + $assertion = @{ kind = "property"; target = $target; property = "IsEnabled"; value = $true } + $scenario = @{ schemaVersion = 1; claimId = "818-C1"; authoritySha256 = $authorityHash; coverage = "complete-semantic" + fixture = @{ id = "synthetic"; version = 1; manifestSha256 = Hash-File $manifestPath } + cases = @(@{ id = "synthetic-success"; state = "success"; tier = "uia" + preconditions = @{ profile = "owned-isolated"; process = "owned-production"; readyRoot = $anchor + initial = @($assertion); timeoutMs = 1000 } + steps = @(@{ operation = "invoke"; target = $target; timeoutMs = 1000 }) + expected = @($assertion); timeoutMs = 5000 + cleanup = @{ intent = "owned-processes-and-fixture"; timeoutMs = 1000 } }) } + Write-Json $scenarioPath $scenario + $baseData = [IO.File]::ReadAllText($dataPath) + $baseAuthority = [IO.File]::ReadAllText($authorityPath) + $baseManifest = [IO.File]::ReadAllText($manifestPath) + $baseScenario = [IO.File]::ReadAllText($scenarioPath) + Test-Contract "positive synthetic data has nonzero document/case/assertion counts" { + $summary = Accept + Assert-Contract ($summary.documentCount -eq 1 -and $summary.caseCount -eq 1 -and + $summary.assertionCount -eq 2 -and $summary.fixtures[0].fileCount -eq 1 -and + $summary.coverage[0].uncoveredRequiredStates.Count -eq 0) "exact positive counts wrong" + } + Test-Contract "wrong authority revision is refused with ClaimAuthorityMismatch" { + $scenario.authoritySha256 = "a" * 64 + Write-Json $scenarioPath $scenario + Refuse "ClaimAuthorityMismatch" + } + Test-Contract "wrong caller hash" { Refuse "ClaimAuthorityMismatch" @{ AuthoritySha256 = "a" * 64 } } + Test-Contract "missing authority" { Refuse "UnsafePath" @{ AuthorityPath = (Join-Path $scratch "missing.json") } } + foreach ($mutation in @("issue", "repo", "comment", "source", "criterion", "claim-prefix", "date", "duplicate")) { + Test-Contract "authority refuses $mutation" { + $code = "ClaimAuthorityMismatch" + switch ($mutation) { + "issue" { $authority.issueNumber = 819 } + "repo" { $authority.repository = "other/SECRET_CANARY"; $code = "SchemaInvalid" } + "comment" { $authority.sourceCommentId = 2 } + "source" { $authority.sourceUrl = "https://github.com/scgopi/GraphCode/issues/819#issuecomment-1" } + "criterion" { $authority.claims[0].criterion += " changed" } + "claim-prefix" { $authority.claims[0].id = "819-C1" } + "date" { $authority.sourceUpdatedAt = "2026-02-30T00:00:00Z" } + "duplicate" { $authority.claims += $authority.claims[0]; $code = "DuplicateClaim" } + } + Save-Inputs + Refuse $code + } + } + foreach ($mutation in @("source-revision", "applicability")) { + Test-Contract "external trusted hash rejects edited $mutation" { + if ($mutation -ceq "source-revision") { $authority.sourceUpdatedAt = "2026-10-15T00:00:00Z" } else { + $authority.claims[0].states.success.required = $false + } + Write-Json $authorityPath $authority + Refuse "ClaimAuthorityMismatch" + } + } + foreach ($mutation in @("malformed", "truncated", "null", "scalar", "array", "empty", "oversize", "depth", "utf8")) { + Test-Contract "parser refuses $mutation" { + $code = "InvalidJson" + switch ($mutation) { + "malformed" { $text = '{"SECRET_CANARY":}' } + "truncated" { $text = $baseScenario.Substring(0, $baseScenario.Length - 1) } + "null" { $text = "null"; $code = "DocumentRoot" } + "scalar" { $text = '"SECRET_CANARY"'; $code = "DocumentRoot" } + "array" { $text = "[]"; $code = "DocumentRoot" } + "empty" { $text = ""; $code = "DocumentSize" } + "oversize" { $text = " " * (1MB + 1); $code = "DocumentSize" } + "depth" { $text = '{"nested":' + ("[" * 33) + "0" + ("]" * 33) + "}" } + "utf8" { + [IO.File]::WriteAllBytes($scenarioPath, [byte[]]@(123, 34, 255, 34, 58, 49, 125)) + Refuse $code + return + } + } + [IO.File]::WriteAllText($scenarioPath, $text, $utf8) + Refuse $code + } + } + foreach ($document in @("scenario", "authority", "manifest")) { + foreach ($duplicate in @('{"schemaVersion":1,', '{"SchemaVersion":1,')) { + Test-Contract "$document recursive duplicate root property $duplicate" { + $path = switch ($document) { "scenario" { $scenarioPath }; "authority" { $authorityPath }; "manifest" { $manifestPath } } + $text = [IO.File]::ReadAllText($path) + [IO.File]::WriteAllText($path, ($duplicate + $text.Substring(1)), $utf8) + Refuse "DuplicateProperty" + } + } + } + foreach ($level in @("fixture", "case", "preconditions", "target", "assertion", "step", "cleanup", "state", "file")) { + Test-Contract "duplicate property in nested $level" { + $path = $scenarioPath + $needle = switch ($level) { + "fixture" { '"manifestSha256":' }; "case" { '"tier":' }; "preconditions" { '"profile":' } + "target" { '"automationId":' }; "assertion" { '"kind":' }; "step" { '"operation":' } + "cleanup" { '"intent":' }; "state" { $path = $authorityPath; '"reason":' } + "file" { $path = $manifestPath; '"path":' } + } + $text = [IO.File]::ReadAllText($path) + $text = $text.Replace($needle, ($needle + '"SECRET_CANARY",' + $needle)) + [IO.File]::WriteAllText($path, $text, $utf8) + Refuse "DuplicateProperty" + } + } + foreach ($mutation in @("unknown", "version", "claim-null", "claim-empty", "case-empty", "steps-empty", "expected-empty", + "cases-missing", "action", "property", "proof", "approval", "coordinates", "visual-tier", "command", "cleanup-missing", "cleanup-disabled")) { + Test-Contract "scenario schema refuses $mutation" { + switch ($mutation) { + "unknown" { $scenario.SECRET_CANARY = "ignored?" } + "version" { $scenario.schemaVersion = 2 } + "claim-null" { $scenario.claimId = $null } + "claim-empty" { $scenario.claimId = "" } + "case-empty" { $scenario.cases = @() } + "steps-empty" { $scenario.cases[0].steps = @() } + "expected-empty" { $scenario.cases[0].expected = @() } + "cases-missing" { $scenario.Remove("cases") } + "action" { $scenario.cases[0].steps[0].operation = "SECRET_CANARY" } + "property" { $scenario.cases[0].expected[0].property = "SECRET_CANARY" } + "proof" { $scenario.proofClass = "physical-input" } + "approval" { $scenario.reviewed = $true } + "coordinates" { $scenario.cases[0].steps[0].x = 1 } + "visual-tier" { $scenario.cases[0].tier = "visual" } + "command" { $scenario.cases[0].steps[0].command = "Set-Content '$scratch\executed' SECRET_CANARY" } + "cleanup-missing" { $scenario.cases[0].Remove("cleanup") } + "cleanup-disabled" { $scenario.cases[0].cleanup.intent = "none" } + } + Save-Inputs + Refuse "SchemaInvalid" + Assert-Contract (-not (Test-Path -LiteralPath (Join-Path $scratch "executed"))) "command canary executed" + } + } + Test-Contract "unknown authority claim" { + $scenario.claimId = "818-C99" + Save-Inputs + Move-Item -LiteralPath $scenarioPath -Destination (Join-Path $root "818-C99.json") + Refuse "ClaimAuthorityMismatch" + } + Test-Contract "duplicate case identity" { + $scenario.cases += $scenario.cases[0]; Save-Inputs; Refuse "DuplicateCase" + } + Test-Contract "mislabeled file identity" { + Move-Item -LiteralPath $scenarioPath -Destination (Join-Path $root "818-C2.json"); Refuse "ScenarioFileIdentity" + } + Test-Contract "duplicate claim files" { + $directory = New-Item -ItemType Directory -Path (Join-Path $root "nested") + Copy-Item $scenarioPath (Join-Path $directory.FullName "818-C1.json"); Refuse "DuplicateClaimFile" + } + Test-Contract "unexpected JSON is not ignored" { + [IO.File]::WriteAllText((Join-Path $root "unexpected.json"), "{}", $utf8); Refuse "SchemaInvalid" + } + Test-Contract "schema-only production discovery fails" { + Remove-Item -LiteralPath $scenarioPath + Copy-Item (Join-Path $repoRoot "windows-tests\ui-scenarios\scenario.schema.json") $root + Refuse "EmptySelection" + } + Test-Contract "empty scenario directory" { + Remove-Item -LiteralPath $scenarioPath; Refuse "EmptySelection" + } + Test-Contract "exact requested claim" { $null = Accept @{ ClaimId = @("818-C1") } } + Test-Contract "unknown requested claim" { Refuse "UnknownClaim" @{ ClaimId = @("818-C99") } } + Test-Contract "duplicate requested claim" { Refuse "DuplicateRequest" @{ ClaimId = @("818-C1", "818-C1") } } + Test-Contract "empty requested selection" { Refuse "EmptySelection" @{ ClaimId = @() } } + Test-Contract "known claim missing scenario" { + $claim = ConvertFrom-Json (ConvertTo-Json $authority.claims[0] -Depth 32) -AsHashtable + $claim.id = "818-C2"; $authority.claims += $claim; Save-Inputs + Refuse "MissingScenario" @{ ClaimId = @("818-C2") } + } + Test-Contract "multiple states and exact hashes reported" { + $authority.claims[0].states.empty.required = $true + $case = ConvertFrom-Json (ConvertTo-Json $scenario.cases[0] -Depth 32) -AsHashtable + $case.id = "synthetic-empty"; $case.state = "empty" + $case.expected[0].property = "Value"; $case.expected[0].value = "" + $scenario.cases += $case; Save-Inputs + $summary = Accept + Assert-Contract ($summary.caseCount -eq 2 -and $summary.assertionCount -eq 4 -and + $summary.scenarioHashes.'818-C1' -ceq (Hash-File $scenarioPath) -and + $summary.fixtures[0].manifestSha256 -ceq (Hash-File $manifestPath)) "multiple-state hashes/counts wrong" + } + Test-Contract "mixed component reports uncovered states and proof" { + $authority.claims[0].proofClass = "mixed"; $authority.claims[0].states.error.required = $true + $scenario.coverage = "component"; Save-Inputs + $summary = Accept + Assert-Contract ($summary.coverage[0].uncoveredRequiredStates -contains "error" -and + $summary.coverage[0].uncoveredProofComponents.Count -gt 0) "mixed claim silently closed" + } + Test-Contract "input component remains incomplete evidence" { + $authority.claims[0].proofClass = "input"; $scenario.coverage = "component"; Save-Inputs + $summary = Accept + Assert-Contract ($summary.coverage[0].uncoveredProofComponents.Count -gt 0) "input proof silently closed" + } + foreach ($proof in @("visual", "mixed", "input")) { + Test-Contract "$proof cannot declare complete semantic coverage" { + $authority.claims[0].proofClass = $proof; Save-Inputs; Refuse "ProofClassMismatch" + } + } + Test-Contract "complete semantic requires every applicable state" { + $authority.claims[0].states.error.required = $true; Save-Inputs; Refuse "IncompleteSemanticCoverage" + } + Test-Contract "not-applicable state cannot be manufactured" { + $scenario.cases[0].state = "error"; Save-Inputs; Refuse "StateNotApplicable" + } + foreach ($transport in @("post-message", "send-input")) { + Test-Contract "interactive $transport is valid data not physical proof" { + $scenario.cases[0].tier = "interactive" + $scenario.cases[0].steps = @(@{ operation = "send-keys"; target = $target + keys = "ENTER"; transport = $transport; timeoutMs = 1000 }) + Save-Inputs; $null = Accept + } + } + Test-Contract "send-keys requires interactive tier" { + $scenario.cases[0].steps = @(@{ operation = "send-keys"; target = $target + keys = "ENTER"; transport = "post-message"; timeoutMs = 1000 }) + Save-Inputs; Refuse "InputTierRequired" + } + Test-Contract "keyboard state requires explicit key transport" { + $authority.claims[0].states.keyboard.required = $true; $scenario.cases[0].state = "keyboard" + $scenario.coverage = "component"; Save-Inputs; Refuse "KeyboardStepRequired" + } + Test-Contract "persistence state requires relaunch" { + $authority.claims[0].states.'persistence-after-relaunch'.required = $true + $scenario.cases[0].state = "persistence-after-relaunch"; $scenario.coverage = "component" + Save-Inputs; Refuse "RelaunchStepRequired" + } + foreach ($mutation in @("empty", "wildcard", "newline", "name", "index", "first-match", "scope", "count-zero", "count-two", "count-bool", "type")) { + Test-Contract "selector refuses $mutation" { + $selector = $scenario.cases[0].steps[0].target + switch ($mutation) { + "empty" { $selector.automationId = "" } + "wildcard" { $selector.automationId = "synthetic-*" } + "newline" { $selector.automationId = "synthetic`n" } + "name" { $selector.Name = "SECRET_CANARY" } + "index" { $selector.index = 0 } + "first-match" { $selector.firstMatch = $true } + "scope" { $selector.scope = "desktop" } + "count-zero" { $selector.matchCount = 0 } + "count-two" { $selector.matchCount = 2 } + "count-bool" { $selector.matchCount = $true } + "type" { $selector.controlType = "SECRET_CANARY" } + } + Save-Inputs; Refuse "SchemaInvalid" + } + } + Test-Contract "absence requires positive healthy root" { + $scenario.cases[0].expected = @(@{ kind = "absent"; target = $target }) + Save-Inputs; Refuse "SchemaInvalid" + } + Test-Contract "absence anchor must be owned top-level Window" { + $scenario.cases[0].expected = @(@{ kind = "absent"; target = $target; healthyRoot = $target }) + Save-Inputs; Refuse "HealthyRootRequired" + } + Test-Contract "absence with explicit healthy root accepted" { + $scenario.cases[0].expected = @(@{ kind = "absent"; target = $target; healthyRoot = $anchor }) + Save-Inputs; $null = Accept + } + Test-Contract "explicit empty text accepted but missing value refused" { + $scenario.cases[0].expected[0].property = "Value"; $scenario.cases[0].expected[0].value = "" + Save-Inputs; $null = Accept + $scenario.cases[0].expected[0].Remove("value"); Save-Inputs; Refuse "SchemaInvalid" + } + foreach ($operation in @("toggle", "select", "focus", "set-value", "wait-for", "relaunch")) { + Test-Contract "closed operation $operation accepts its typed fields" { + $step = @{ operation = $operation; target = $target; timeoutMs = 1000 } + switch ($operation) { + "set-value" { $step.value = "" } + "wait-for" { $step.Remove("target"); $step.expected = $assertion } + "relaunch" { $step.Remove("target"); $step.readyRoot = $anchor } + } + $scenario.cases[0].steps = @($step); Save-Inputs; $null = Accept + } + } + foreach ($mutation in @("manifest-missing", "data-missing", "manifest-hash", "file-hash", "id", "version", "schema", + "duplicate", "empty")) { + Test-Contract "fixture refuses $mutation" { + $code = "FixtureMismatch" + switch ($mutation) { + "manifest-missing" { Remove-Item -LiteralPath $manifestPath; $code = "UnsafePath" } + "data-missing" { Remove-Item -LiteralPath $dataPath; $code = "UnsafePath" } + "manifest-hash" { $scenario.fixture.manifestSha256 = "a" * 64; Write-Json $scenarioPath $scenario } + "file-hash" { [IO.File]::WriteAllText($dataPath, "changed", $utf8); $code = "FixtureFileMismatch" } + "id" { $manifest.fixtureId = "different"; Save-Inputs } + "version" { $manifest.fixtureVersion = 2; Save-Inputs } + "schema" { $manifest.schemaVersion = 2; Save-Inputs; $code = "SchemaInvalid" } + "duplicate" { $manifest.files += $manifest.files[0]; Save-Inputs; $code = "DuplicateFixtureFile" } + "empty" { $manifest.files = @(); Save-Inputs; $code = "SchemaInvalid" } + } + Refuse $code + } + } + foreach ($path in @("..\data.txt", "/data.txt", "C:\data.txt", "\\server\data.txt", "\\?\C:\data.txt", + "data.txt:secret", "NUL", "con.txt", "COM1.txt", "CONIN$", "CONOUT$", "CLOCK$", "CON .txt", + ("COM" + [char]0x00b9 + ".txt"), "folder\..\data.txt", "data.txt.", "data.txt ", "a\\data.txt")) { + Test-Contract "fixture path refuses $path" { + $manifest.files[0].path = $path; Save-Inputs; Refuse "UnsafeFixturePath" + } + } + Test-Contract "fixture reparse escape refused without reading foreign data" { + $outside = Join-Path $scratch "outside" + $null = New-Item -ItemType Directory -Path $outside -Force + [IO.File]::WriteAllText((Join-Path $outside "SECRET_CANARY.txt"), "SECRET_CANARY", $utf8) + $link = Join-Path $fixtureRoot "escape" + $null = New-Item -ItemType Junction -Path $link -Target $outside + try { + $manifest.files[0].path = "escape\SECRET_CANARY.txt"; Save-Inputs; Refuse "UnsafePath" + } finally { Remove-Item -LiteralPath $link -Force } + } + Test-Contract "reparse scenario root refused" { + $link = Join-Path $scratch "scenario-link" + $null = New-Item -ItemType Junction -Path $link -Target $root + try { Refuse "UnsafePath" @{ ScenarioRoot = $link } } finally { Remove-Item -LiteralPath $link -Force } + } + foreach ($value in @("100", $true, -1, 0, 0.5, 10001)) { + Test-Contract "ordinary deadline refuses $value" { + $scenario.cases[0].steps[0].timeoutMs = $value; Save-Inputs; Refuse "SchemaInvalid" + } + } + foreach ($mutation in @("relaunch", "cleanup", "case", "aggregate-case", "aggregate-scenario")) { + Test-Contract "budget refuses $mutation" { + $code = "SchemaInvalid" + switch ($mutation) { + "relaunch" { + $scenario.cases[0].steps = @(@{ operation = "relaunch"; readyRoot = $anchor; timeoutMs = 30001 }) + } + "cleanup" { $scenario.cases[0].cleanup.timeoutMs = 15001 } + "case" { $scenario.cases[0].timeoutMs = 60001 } + "aggregate-case" { $scenario.cases[0].timeoutMs = 2999; $code = "CaseBudgetExceeded" } + "aggregate-scenario" { + $scenario.cases[0].timeoutMs = 60000 + for ($index = 2; $index -le 4; $index++) { + $case = ConvertFrom-Json (ConvertTo-Json $scenario.cases[0] -Depth 32) -AsHashtable + $case.id = "synthetic-$index"; $scenario.cases += $case + } + $code = "ScenarioBudgetExceeded" + } + } + Save-Inputs; Refuse $code + } + } + foreach ($document in @("scenario", "authority", "fixture")) { + Test-Contract "$document unknown version is refused" { + switch ($document) { + "scenario" { $scenario.schemaVersion = 99 } + "authority" { $authority.schemaVersion = 99 } + "fixture" { $manifest.schemaVersion = 99 } + } + Save-Inputs; Refuse "SchemaInvalid" + } + } + foreach ($collection in @("cases", "steps", "expected")) { + Test-Contract "$collection upper count bound" { + if ($collection -ceq "cases") { + $scenario.cases = @($scenario.cases[0]) * 17 + } else { $scenario.cases[0][$collection] = @($scenario.cases[0][$collection][0]) * 129 } + Save-Inputs; Refuse "SchemaInvalid" + } + } + Test-Contract "exact relaunch cleanup case and scenario budget upper boundaries" { + $scenario.cases[0].preconditions.timeoutMs = 10000 + $scenario.cases[0].steps = @(@{ operation = "relaunch"; readyRoot = $anchor; timeoutMs = 30000 }) + $scenario.cases[0].cleanup.timeoutMs = 15000 + $scenario.cases[0].timeoutMs = 60000 + for ($index = 2; $index -le 3; $index++) { + $case = ConvertFrom-Json (ConvertTo-Json $scenario.cases[0] -Depth 32) -AsHashtable + $case.id = "boundary-$index"; $scenario.cases += $case + } + Save-Inputs + $summary = Accept + Assert-Contract ($summary.coverage[0].budgetMs -eq 180000) "exact maximum budget not reported" + } + Test-Contract "minimum deadlines and exact aggregate case budget" { + $scenario.cases[0].preconditions.timeoutMs = 1; $scenario.cases[0].steps[0].timeoutMs = 1 + $scenario.cases[0].cleanup.timeoutMs = 1; $scenario.cases[0].timeoutMs = 3 + Save-Inputs; $null = Accept + } + Test-Contract "fixture directory is not a data file" { + $null = New-Item -ItemType Directory -Path (Join-Path $fixtureRoot "directory") + $manifest.files[0].path = "directory"; Save-Inputs; Refuse "UnsafeFixturePath" + } + Test-Contract "case-insensitive normalized fixture paths are duplicates" { + $file = @{ path = "DATA.TXT"; sha256 = $manifest.files[0].sha256 } + $manifest.files += $file; Save-Inputs; Refuse "DuplicateFixtureFile" + } + Test-Contract "strict case-sensitive scenario properties" { + $scenario.SchemaVERSION = $scenario.schemaVersion; $scenario.Remove("schemaVersion") + Save-Inputs; Refuse "SchemaInvalid" + } + Test-Contract "typed Toggle pattern assertion" { + $scenario.cases[0].expected = @(@{ kind = "pattern"; target = $target; pattern = "Toggle"; value = "On" }) + Save-Inputs; $null = Accept + } + Test-Contract "positive keyboard and persistence cases retain component limits" { + $authority.claims[0].proofClass = "mixed" + $authority.claims[0].states.keyboard.required = $true + $authority.claims[0].states.'persistence-after-relaunch'.required = $true + $scenario.coverage = "component" + $keyboard = ConvertFrom-Json (ConvertTo-Json $scenario.cases[0] -Depth 32) -AsHashtable + $keyboard.id = "keyboard"; $keyboard.state = "keyboard"; $keyboard.tier = "interactive" + $keyboard.steps = @(@{ operation = "send-keys"; target = $target + keys = "ENTER"; transport = "send-input"; timeoutMs = 1000 }) + $persist = ConvertFrom-Json (ConvertTo-Json $scenario.cases[0] -Depth 32) -AsHashtable + $persist.id = "persistence"; $persist.state = "persistence-after-relaunch" + $persist.steps = @(@{ operation = "relaunch"; readyRoot = $anchor; timeoutMs = 1000 }) + $scenario.cases += @($keyboard, $persist); Save-Inputs + $summary = Accept + Assert-Contract ($summary.caseCount -eq 3 -and $summary.coverage[0].uncoveredRequiredStates.Count -eq 0 -and + $summary.coverage[0].uncoveredProofComponents.Count -gt 0) "component became a whole-claim result" + } + Assert-Contract ($results.Count -gt 0) "zero invoked contract cases" + Write-Host "All $($results.Count) offline contract tests passed" + [ordered]@{ suite = "UiScenario"; executed = $results.Count; passed = $results.Count; failed = 0 } | + ConvertTo-Json -Compress +} finally { + if (Test-Path -LiteralPath $scratch) { Remove-Item -LiteralPath $scratch -Recurse -Force } +} diff --git a/Tools/windows/Tests/ValidationRunner.Tests.ps1 b/Tools/windows/Tests/ValidationRunner.Tests.ps1 index db1f090b..48bbf9ed 100644 --- a/Tools/windows/Tests/ValidationRunner.Tests.ps1 +++ b/Tools/windows/Tests/ValidationRunner.Tests.ps1 @@ -4213,5 +4213,17 @@ try { } } +$uiScenarioOutput = @(& $pwsh -NoProfile -File (Join-Path $PSScriptRoot "UiScenario.Tests.ps1")) +if ($LASTEXITCODE -ne 0) { throw "RED: offline UI scenario contract child failed" } +$uiScenarioSummaries = @($uiScenarioOutput | Where-Object { $_ -is [string] -and $_.StartsWith('{"suite":"UiScenario",') }) +if ($uiScenarioSummaries.Count -ne 1) { throw "RED: offline UI scenario structured count missing or duplicated" } +$uiScenarioSummary = ConvertFrom-Json $uiScenarioSummaries[0] +if ($uiScenarioSummary.executed -isnot [long] -or $uiScenarioSummary.executed -le 0 -or + $uiScenarioSummary.passed -isnot [long] -or $uiScenarioSummary.passed -ne $uiScenarioSummary.executed -or + $uiScenarioSummary.failed -isnot [long] -or $uiScenarioSummary.failed -ne 0) { + throw "RED: offline UI scenario child has no positive executed/passed contract count" +} +$uiScenarioOutput | ForEach-Object { Write-Host $_ } + Write-Host "ValidationRunner.Tests.ps1: PASS" exit 0 diff --git a/windows-tests/ui-scenarios/README.md b/windows-tests/ui-scenarios/README.md new file mode 100644 index 00000000..92364db0 --- /dev/null +++ b/windows-tests/ui-scenarios/README.md @@ -0,0 +1,194 @@ +# Offline UI scenario contracts (G1) + +These schemas and `Tools\windows\Test-UiScenario.ps1` validate **data**, not UI. +There are deliberately no real product scenarios or claim declarations here. +The functional tests create explicitly synthetic claims in disposable scratch. +A schema-only or empty scenario directory fails with `EmptySelection`. + +## Issue-owned authority + +Before new UI work starts, the issue owner/coordinator assigns stable IDs such +as `768-C3`, immutable acceptance statements, allowed proof classes, and state +applicability. Every claim records `success`, `empty`, `error`, `disabled`, +`persistence-after-relaunch`, and `keyboard` as required or not applicable, +with a reason. Scenario authors cannot rename, weaken, split away, or close +those claims. A changed meaning requires explicit supersession and a new ID. +This convention does not retroactively add states or obligations to previously +approved wave-2 work. + +The caller supplies a frozen **external** authority snapshot and an independently +trusted SHA-256. There is no default PR-head registry. The coordinator must +compare the export against the issue-owned declaration and provide the expected +hash through a trusted channel; a future CI caller must not obtain both inputs +from the PR head. `claim-authority.schema.json` is an input format, not authority +created by adding a file to this directory. + +The snapshot binds `repository`, `issueNumber`, `sourceUrl`, `sourceCommentId`, +`sourceUpdatedAt`, `sourceBodySha256`, and nonempty claims containing `id`, +`criterion`, `criterionSha256`, `proofClass`, and all six `states`. URLs identify +a comment on the same `scgopi/GraphCode` issue, and IDs have that issue's numeric +prefix. The validator checks criterion hashes and snapshot byte identity. +It does not fetch the comment, verify its body hash, authenticate GitHub roles, +or establish human approval. Changing applicability/revision invalidates the +old snapshot hash, but a caller who also changes the trusted hash changes the +trust input. There is no `approvedBy`, `reviewed`, or verdict field. + +Digests are lowercase SHA-256 over exact UTF-8 bytes. Criterion digests use +the decoded string's UTF-8 bytes; document digests use the actual file bytes. +No newline normalization, BOM removal, or canonical JSON reserialization is +performed. Documents must be UTF-8 without a BOM. + +| Proof class | Evidence boundary | +| --- | --- | +| `semantic-uia` | Future passing production UIA observations/actions can support the exact semantic component, not rendering or physical input. | +| `visual` | Local screenshot review against macOS is required; v1 rejects visual scenarios. | +| `input` | Requires explicit transport and end-state evidence; posted/injected keys are not physical input. V1 accepts component data only. | +| `mixed` | UIA/input components cannot close the whole claim without its visual, physical-input, or other required evidence. V1 accepts component data only. | + +`complete-semantic` is only a declaration of complete **state coverage in data** +for a `semantic-uia` claim; every required state must have a case. `component` +may leave required states uncovered. Reports list uncovered states and non-UIA +proof obligations for mixed/input claims. The exact criterion, not a generic +class label, determines which external evidence is actually required. + +## Scenario v1 + +Use one `issueNumber-Cn.json` file per claim directly under `ScenarioRoot`. +Only the three exact schema filenames at that root and the `fixtures` subtree +are excluded from scenario JSON discovery. Unexpected JSON is not ignored. +Optional `-ClaimId` values select exact IDs; unknown, duplicate, empty, missing, +or mislabeled selections fail. All discovered scenario documents are parsed +and schema/file-identity checked even when not selected; cross-document +authority, fixture, and coverage checks apply to selected claims. + +A scenario contains only `schemaVersion`, `claimId`, `authoritySha256`, +`coverage`, `fixture` (`id`, `version`, `manifestSha256`), and nonempty `cases`. +It cannot repeat editable acceptance text, proof class, approval, or a verdict. +Each case contains a unique `id`, applicable `state`, `tier` (`uia` or +`interactive`), `preconditions`, ordered `steps`, `expected` end states, +`timeoutMs`, and `cleanup`. + +- Preconditions declare `profile: owned-isolated`, `process: owned-production`, + a named `readyRoot`, nonempty typed `initial` assertions, and `timeoutMs`. + Readiness/healthy-root anchors must be an `owned-root` `Window`. +- Targets contain literal case-sensitive `automationId`, exact UIA + `controlType`, `scope` (`owned-root` or `owned-modal`), and `matchCount: 1`. + There is no wildcard, regex, index, Name fallback, or ancestor substitution. + V1 accepts ASCII literal IDs (letters, digits, `_`, `.`, `:`, `-`). +- Operations are `invoke`, `toggle`, `select`, `focus` (target/deadline), + `set-value` (also typed `value`), `wait-for` (typed `expected`/deadline), + `relaunch` (`readyRoot`/deadline), or `send-keys` + (target, `keys`, `transport`, deadline). Send-keys requires `interactive`, + including `post-message`; `send-input` is also injected, not physical input. + Keyboard cases require send-keys; persistence cases require relaunch. +- Assertions are exact `property` (`IsEnabled`, `IsSelected`, + `HasKeyboardFocus` as booleans; `Name`, `Value` as strings), `pattern` + (`Toggle` with `On`, `Off`, `Indeterminate`), or `absent` with a distinct + positive `healthyRoot` anchor. Explicit empty Name/Value is valid; a missing + value is not an empty-text observation. +- Cleanup requires `intent: owned-processes-and-fixture` and a deadline. + No cleanup-disable field exists. This validates intent, not execution. + +Every object rejects unknown fields. Types, enums, required arrays, duplicate +and case-confusable properties (recursively), and versions are strict. The +draft-07 schemas use only local definitions and are the structural source of +truth; PowerShell implements cross-document semantics, not a second schema. +No command/script, private gate message, coordinate, screenshot, image, +golden/pixel baseline, visual tier, or arbitrary setup path is accepted. + +| Bound | V1 limit | +| --- | --- | +| UTF-8 JSON document / nesting | 1 MiB / depth 32 | +| Cases per scenario | 1-16 | +| Steps per case / assertions per assertion array | 1-128 / 1-128 | +| Ordinary step, wait, precondition deadline | 1-10000 ms | +| Relaunch readiness / cleanup deadline | 1-30000 ms / 1-15000 ms | +| Case budget / sum of scenario case budgets | 1-60000 ms / at most 180000 ms | + +The case budget must cover precondition, all step, and cleanup deadlines. +Cleanup is included in case and scenario budgets, not omitted from the sum. +These are data bounds, not cancelable UIA/COM or wall-clock execution guarantees. + +## Fixture v1 + +The only manifest location is `fixtures\id\manifest.json` under `ScenarioRoot`. +`fixture.schema.json` requires `schemaVersion: 1`, matching `fixtureId` and +positive `fixtureVersion`, and 1-128 `files` with relative `path` and `sha256`. +The scenario's manifest digest and every referenced data-file digest must match. +Fixture versions are identities, not a declaration that a future adapter +supports their execution. + +File paths may use either slash separator; they are normalized for +case-insensitive Windows duplicate detection. Absolute, empty-segment, +traversal, device, alternate-stream, trailing-dot/space, invalid Windows +character, missing, directory-as-file, and reparse-point paths fail. No +reparse-point ancestor is accepted, including for authority or scenario paths. +The caller must supply owned, privacy-safe packaged data and keep it stable +during validation. Static path checks and hashes are not protection against a +concurrent hostile filesystem mutation, filesystem attestation, or proof that +data was ever installed in a production profile. The validator only reads; +it performs no fixture setup or mutation. + +## Commands and results + +Use PowerShell 7.5 or newer (built-in `Test-Json`, `System.Text.Json`, and +string-preserving `ConvertFrom-Json -DateKind String`; no packages installed). +For a future real, human-reviewed scenario set: + +```powershell +pwsh -NoProfile -File Tools\windows\Test-UiScenario.ps1 ` + -ScenarioRoot windows-tests\ui-scenarios ` + -AuthorityPath C:\trusted-authority\818.json ` + -AuthoritySha256 $trustedHash -ClaimId 818-C1 +``` + +This repository currently has **zero** real scenarios, so that production +discovery cannot pass. Do not use synthetic functional fixtures as UI evidence. +Run the offline contracts instead: + +```powershell +pwsh -NoProfile -File Tools\windows\Tests\UiScenario.Tests.ps1 +pwsh -NoProfile -File Tools\windows\Tests\ValidationRunner.Tests.ps1 +bash Tools/ci/tests/classify-changes.test.sh +pwsh -NoProfile -File Tools\tdd\Tests\TddEvidence.Tests.ps1 +``` + +The validator exits 0 only for nonempty valid data, emitting one JSON summary +with `status: valid-data`, schema/authority/scenario/fixture hashes, selected +IDs, positive document/case/assertion counts, fixture file counts, and coverage. +Assertion counts include initial, end-state, and wait-for declarations. It +never emits a UI verdict or dumps arbitrary input. Invalid data exits nonzero +with a bounded refusal code, without echoing input contents. + +Refusal codes identify parsing (`InvalidJson`, `DocumentRoot`, `DocumentSize`, +`DuplicateProperty`), schema (`SchemaInvalid`, `UnsafeSchemaReference`), +binding (`ClaimAuthorityMismatch`, `DuplicateClaim`), discovery/selection +(`EmptySelection`, `UnknownClaim`, `DuplicateRequest`, `MissingScenario`, +`ScenarioFileIdentity`, `DuplicateClaimFile`), fixtures (`UnsafePath`, +`UnsafeFixturePath`, `DuplicateFixtureFile`, `FixtureMismatch`, +`FixtureFileMismatch`), coverage (`ProofClassMismatch`, `StateNotApplicable`, +`IncompleteSemanticCoverage`, `HealthyRootRequired`, `InputTierRequired`, +`KeyboardStepRequired`, `RelaunchStepRequired`), or budgets +(`CaseBudgetExceeded`, `ScenarioBudgetExceeded`). `ValidationError` denotes +an unexpected tooling/I/O failure, never a success-shaped fallback. + +The executable test suite invokes the actual validator in bounded child pwsh +processes, prints named results and one `UiScenario` structured count. The +existing ValidationRunner requires that child to exit successfully with positive +executed/passed counts and zero failures. Path classification tests only confirm +existing ancestor classification; they do not discover or execute scenarios. + +## Not verified and later work + +G1 establishes no human approval, runtime selector uniqueness/ownership, +current PID/HWND/generation, UIA pattern behavior, input, pixels, rendering, +cleanup completion, persistence, cross-platform parity, or issue closure. +Missing required UIA must not later be replaced by coordinates or private +commands. Cleanup failures cannot become successful claim verification. + +G2 verifier/preflight and G3 replay/opt-in/result transport need separately +approved plans and leases. G1 installs no WinApp tool, creates no canonical +project root or lock, launches no product, changes no gate/workflow/TDD rule, +and adds no screenshot pipeline. Literal canvas IDs observed under a future +fixed project root are not portable across roots. Actual scenario authoring +waits for independent production verification and human review. diff --git a/windows-tests/ui-scenarios/claim-authority.schema.json b/windows-tests/ui-scenarios/claim-authority.schema.json new file mode 100644 index 00000000..2540a13f --- /dev/null +++ b/windows-tests/ui-scenarios/claim-authority.schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Externally supplied UI claim authority snapshot v1", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "repository", "issueNumber", "sourceUrl", "sourceCommentId", "sourceUpdatedAt", "sourceBodySha256", "claims"], + "properties": { + "schemaVersion": { "type": "integer", "const": 1 }, + "repository": { "type": "string", "const": "scgopi/GraphCode" }, + "issueNumber": { "type": "integer", "minimum": 1 }, + "sourceUrl": { "type": "string", "pattern": "^https://github\\.com/scgopi/GraphCode/issues/[1-9][0-9]*#issuecomment-[1-9][0-9]*(?![\\s\\S])" }, + "sourceCommentId": { "type": "integer", "minimum": 1 }, + "sourceUpdatedAt": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{1,7})?Z(?![\\s\\S])" }, + "sourceBodySha256": { "$ref": "#/definitions/hash" }, + "claims": { + "type": "array", "minItems": 1, "maxItems": 128, + "items": { + "type": "object", "additionalProperties": false, + "required": ["id", "criterion", "criterionSha256", "proofClass", "states"], + "properties": { + "id": { "type": "string", "pattern": "^[1-9][0-9]*-C[1-9][0-9]*(?![\\s\\S])" }, + "criterion": { "type": "string", "minLength": 1, "maxLength": 8192, "pattern": "\\S" }, + "criterionSha256": { "$ref": "#/definitions/hash" }, + "proofClass": { "enum": ["semantic-uia", "visual", "input", "mixed"] }, + "states": { + "type": "object", "additionalProperties": false, + "required": ["success", "empty", "error", "disabled", "persistence-after-relaunch", "keyboard"], + "properties": { + "success": { "$ref": "#/definitions/applicability" }, + "empty": { "$ref": "#/definitions/applicability" }, + "error": { "$ref": "#/definitions/applicability" }, + "disabled": { "$ref": "#/definitions/applicability" }, + "persistence-after-relaunch": { "$ref": "#/definitions/applicability" }, + "keyboard": { "$ref": "#/definitions/applicability" } + } + } + } + } + } + }, + "definitions": { + "hash": { "type": "string", "pattern": "^[a-f0-9]{64}(?![\\s\\S])" }, + "applicability": { + "type": "object", "additionalProperties": false, + "required": ["required", "reason"], + "properties": { + "required": { "type": "boolean" }, + "reason": { "type": "string", "minLength": 1, "maxLength": 2048, "pattern": "\\S" } + } + } + } +} diff --git a/windows-tests/ui-scenarios/fixture.schema.json b/windows-tests/ui-scenarios/fixture.schema.json new file mode 100644 index 00000000..6f962673 --- /dev/null +++ b/windows-tests/ui-scenarios/fixture.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Packaged UI scenario fixture manifest v1", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "fixtureId", "fixtureVersion", "files"], + "properties": { + "schemaVersion": { "type": "integer", "const": 1 }, + "fixtureId": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,63}(?![\\s\\S])" }, + "fixtureVersion": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "files": { + "type": "array", "minItems": 1, "maxItems": 128, + "items": { + "type": "object", "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { "type": "string", "minLength": 1, "maxLength": 240 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}(?![\\s\\S])" } + } + } + } + } +} diff --git a/windows-tests/ui-scenarios/scenario.schema.json b/windows-tests/ui-scenarios/scenario.schema.json new file mode 100644 index 00000000..c0e412cf --- /dev/null +++ b/windows-tests/ui-scenarios/scenario.schema.json @@ -0,0 +1,173 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Offline UI scenario data v1 (not runtime evidence)", + "$comment": "Identifier/hash patterns use an absolute end lookahead: $ alone also accepts a final newline.", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "claimId", "authoritySha256", "coverage", "fixture", "cases"], + "properties": { + "schemaVersion": { "type": "integer", "const": 1 }, + "claimId": { "type": "string", "pattern": "^[1-9][0-9]*-C[1-9][0-9]*(?![\\s\\S])" }, + "authoritySha256": { "type": "string", "pattern": "^[a-f0-9]{64}(?![\\s\\S])" }, + "coverage": { "enum": ["component", "complete-semantic"] }, + "fixture": { + "type": "object", "additionalProperties": false, + "required": ["id", "version", "manifestSha256"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,63}(?![\\s\\S])" }, + "version": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "manifestSha256": { "type": "string", "pattern": "^[a-f0-9]{64}(?![\\s\\S])" } + } + }, + "cases": { + "type": "array", "minItems": 1, "maxItems": 16, + "items": { "$ref": "#/definitions/case" } + } + }, + "definitions": { + "target": { + "type": "object", "additionalProperties": false, + "required": ["automationId", "controlType", "scope", "matchCount"], + "properties": { + "automationId": { "type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*(?![\\s\\S])" }, + "controlType": { "enum": ["Button", "Calendar", "CheckBox", "ComboBox", "Edit", "Hyperlink", "Image", "ListItem", "List", "Menu", "MenuBar", "MenuItem", "ProgressBar", "RadioButton", "ScrollBar", "Slider", "Spinner", "StatusBar", "Tab", "TabItem", "Text", "ToolBar", "ToolTip", "Tree", "TreeItem", "Custom", "Group", "Thumb", "DataGrid", "DataItem", "Document", "SplitButton", "Window", "Pane", "Header", "HeaderItem", "Table", "TitleBar", "Separator", "SemanticZoom", "AppBar"] }, + "scope": { "enum": ["owned-root", "owned-modal"] }, + "matchCount": { "type": "integer", "const": 1 } + } + }, + "timeout": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "assertion": { + "oneOf": [ + { + "type": "object", "additionalProperties": false, + "required": ["kind", "target", "property", "value"], + "properties": { + "kind": { "const": "property" }, + "target": { "$ref": "#/definitions/target" }, + "property": { "enum": ["IsEnabled", "IsSelected", "HasKeyboardFocus"] }, + "value": { "type": "boolean" } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["kind", "target", "property", "value"], + "properties": { + "kind": { "const": "property" }, + "target": { "$ref": "#/definitions/target" }, + "property": { "enum": ["Name", "Value"] }, + "value": { "type": "string", "maxLength": 8192 } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["kind", "target", "pattern", "value"], + "properties": { + "kind": { "const": "pattern" }, + "target": { "$ref": "#/definitions/target" }, + "pattern": { "const": "Toggle" }, + "value": { "enum": ["On", "Off", "Indeterminate"] } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["kind", "target", "healthyRoot"], + "properties": { + "kind": { "const": "absent" }, + "target": { "$ref": "#/definitions/target" }, + "healthyRoot": { "$ref": "#/definitions/target" } + } + } + ] + }, + "assertions": { + "type": "array", "minItems": 1, "maxItems": 128, + "items": { "$ref": "#/definitions/assertion" } + }, + "step": { + "oneOf": [ + { + "type": "object", "additionalProperties": false, + "required": ["operation", "target", "timeoutMs"], + "properties": { + "operation": { "enum": ["invoke", "toggle", "select", "focus"] }, + "target": { "$ref": "#/definitions/target" }, + "timeoutMs": { "$ref": "#/definitions/timeout" } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["operation", "target", "value", "timeoutMs"], + "properties": { + "operation": { "const": "set-value" }, + "target": { "$ref": "#/definitions/target" }, + "value": { "type": "string", "maxLength": 8192 }, + "timeoutMs": { "$ref": "#/definitions/timeout" } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["operation", "expected", "timeoutMs"], + "properties": { + "operation": { "const": "wait-for" }, + "expected": { "$ref": "#/definitions/assertion" }, + "timeoutMs": { "$ref": "#/definitions/timeout" } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["operation", "readyRoot", "timeoutMs"], + "properties": { + "operation": { "const": "relaunch" }, + "readyRoot": { "$ref": "#/definitions/target" }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 30000 } + } + }, + { + "type": "object", "additionalProperties": false, + "required": ["operation", "target", "keys", "transport", "timeoutMs"], + "properties": { + "operation": { "const": "send-keys" }, + "target": { "$ref": "#/definitions/target" }, + "keys": { "type": "string", "minLength": 1, "maxLength": 8192, "pattern": "\\S" }, + "transport": { "enum": ["post-message", "send-input"] }, + "timeoutMs": { "$ref": "#/definitions/timeout" } + } + } + ] + }, + "case": { + "type": "object", "additionalProperties": false, + "required": ["id", "state", "tier", "preconditions", "steps", "expected", "timeoutMs", "cleanup"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,63}(?![\\s\\S])" }, + "state": { "enum": ["success", "empty", "error", "disabled", "persistence-after-relaunch", "keyboard"] }, + "tier": { "enum": ["uia", "interactive"] }, + "preconditions": { + "type": "object", "additionalProperties": false, + "required": ["profile", "process", "readyRoot", "initial", "timeoutMs"], + "properties": { + "profile": { "const": "owned-isolated" }, + "process": { "const": "owned-production" }, + "readyRoot": { "$ref": "#/definitions/target" }, + "initial": { "$ref": "#/definitions/assertions" }, + "timeoutMs": { "$ref": "#/definitions/timeout" } + } + }, + "steps": { + "type": "array", "minItems": 1, "maxItems": 128, + "items": { "$ref": "#/definitions/step" } + }, + "expected": { "$ref": "#/definitions/assertions" }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 60000 }, + "cleanup": { + "type": "object", "additionalProperties": false, + "required": ["intent", "timeoutMs"], + "properties": { + "intent": { "const": "owned-processes-and-fixture" }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 15000 } + } + } + } + } + } +} From 9de3be1da6d0678fa49083904432ee5926d0af0c Mon Sep 17 00:00:00 2001 From: Colin Neilens Date: Wed, 14 Oct 2026 11:17:58 -0700 Subject: [PATCH 2/3] Refuse normalized network paths before scenario reads Add independent visual-component and healthy-root negatives and isolate each synthetic case. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 017c733a-252b-471a-9c43-26a6a67554df Signed-off-by: Colin Neilens --- Tools/windows/Test-UiScenario.ps1 | 4 +- Tools/windows/Tests/UiScenario.Tests.ps1 | 53 +++++++++++++++++++++++- windows-tests/ui-scenarios/README.md | 13 +++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/Tools/windows/Test-UiScenario.ps1 b/Tools/windows/Test-UiScenario.ps1 index 5e8fa4f0..9784cb4b 100644 --- a/Tools/windows/Test-UiScenario.ps1 +++ b/Tools/windows/Test-UiScenario.ps1 @@ -21,10 +21,10 @@ function Stop-UiScenario([string] $code) { function Assert-UiScenarioPath([string] $path) { try { - if ($path.StartsWith("\\") -or $path.Contains(":") -and $path.Substring(2).Contains(":")) { + $full = [IO.Path]::GetFullPath($path) + if ($full.StartsWith("\\") -or $full.IndexOf(":", 2) -ge 0) { Stop-UiScenario "UnsafePath" } - $full = [IO.Path]::GetFullPath($path) $current = $full while ($current) { $item = Get-Item -LiteralPath $current -Force diff --git a/Tools/windows/Tests/UiScenario.Tests.ps1 b/Tools/windows/Tests/UiScenario.Tests.ps1 index 43a1da43..b021ec78 100644 --- a/Tools/windows/Tests/UiScenario.Tests.ps1 +++ b/Tools/windows/Tests/UiScenario.Tests.ps1 @@ -43,7 +43,12 @@ function Invoke-Validator([hashtable] $arguments = @{}) { } finally { $process.Dispose() } } function Reset-Inputs { - if (Test-Path -LiteralPath $root) { Remove-Item -LiteralPath $root -Recurse -Force } + $script:root = Join-Path $scratch ("case-" + $results.Count) + $script:fixtureRoot = Join-Path $root "fixtures\synthetic" + $script:scenarioPath = Join-Path $root "818-C1.json" + $script:dataPath = Join-Path $fixtureRoot "data.txt" + $script:manifestPath = Join-Path $fixtureRoot "manifest.json" + $script:authorityPath = Join-Path $scratch ("authority-" + $results.Count + ".json") $null = New-Item -ItemType Directory -Path $fixtureRoot [IO.File]::WriteAllText($dataPath, $baseData, $utf8) $script:authority = ConvertFrom-Json $baseAuthority -AsHashtable -DateKind String @@ -81,6 +86,26 @@ function Test-Contract([string] $name, [scriptblock] $body) { $results.Add(@{ name = $name; passed = $true }) Write-Host "PASS: $name" } +function Assert-PathRefusedBeforeRead([string] $path) { + $errors = $null + $ast = [Management.Automation.Language.Parser]::ParseFile($validator, [ref]$null, [ref]$errors) + Assert-Contract ($errors.Count -eq 0) "validator does not parse" + foreach ($definition in $ast.FindAll({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -cin @("Stop-UiScenario", "Assert-UiScenarioPath") + }, $true)) { . ([scriptblock]::Create($definition.Extent.Text)) } + $script:pathGuardReads = 0 + function Get-Item { + param([string] $LiteralPath, [switch] $Force) + $script:pathGuardReads++ + throw "Filesystem access forbidden in path guard probe" + } + $code = $null + try { $null = Assert-UiScenarioPath $path } catch { $code = $_.Exception.Data["UiScenarioCode"] } + Assert-Contract ($code -ceq "UnsafePath" -and $script:pathGuardReads -eq 0) ` + "unsafe caller path reached filesystem before refusal" +} try { $root = Join-Path $scratch "scenarios" @@ -133,6 +158,24 @@ try { Write-Json $scenarioPath $scenario Refuse "ClaimAuthorityMismatch" } + foreach ($parameter in @("ScenarioRoot", "AuthorityPath")) { + foreach ($path in @('\\server\share\root', '//server/share/root', '\\?\C:\root', 'C:\owned\file.json:stream')) { + Test-Contract "$parameter refuses UNC device or ADS before filesystem access: $path" { + Assert-PathRefusedBeforeRead $path + Refuse "UnsafePath" @{ $parameter = $path } + } + } + } + Test-Contract "authority reparse ancestor refused" { + $directory = Join-Path $scratch "authority-copy" + $null = New-Item -ItemType Directory -Path $directory -Force + Copy-Item -LiteralPath $authorityPath -Destination (Join-Path $directory "external-authority.json") + $link = Join-Path $scratch "authority-link" + $null = New-Item -ItemType Junction -Path $link -Target $directory + try { + Refuse "UnsafePath" @{ AuthorityPath = (Join-Path $link "external-authority.json") } + } finally { Remove-Item -LiteralPath $link -Force } + } Test-Contract "wrong caller hash" { Refuse "ClaimAuthorityMismatch" @{ AuthoritySha256 = "a" * 64 } } Test-Contract "missing authority" { Refuse "UnsafePath" @{ AuthorityPath = (Join-Path $scratch "missing.json") } } foreach ($mutation in @("issue", "repo", "comment", "source", "criterion", "claim-prefix", "date", "duplicate")) { @@ -299,6 +342,10 @@ try { $authority.claims[0].proofClass = $proof; Save-Inputs; Refuse "ProofClassMismatch" } } + Test-Contract "pure visual claim rejects component scenarios too" { + $authority.claims[0].proofClass = "visual"; $scenario.coverage = "component" + Save-Inputs; Refuse "ProofClassMismatch" + } Test-Contract "complete semantic requires every applicable state" { $authority.claims[0].states.error.required = $true; Save-Inputs; Refuse "IncompleteSemanticCoverage" } @@ -354,6 +401,10 @@ try { $scenario.cases[0].expected = @(@{ kind = "absent"; target = $target; healthyRoot = $target }) Save-Inputs; Refuse "HealthyRootRequired" } + Test-Contract "absence cannot use its own target as healthy Window anchor" { + $scenario.cases[0].expected = @(@{ kind = "absent"; target = $anchor; healthyRoot = $anchor }) + Save-Inputs; Refuse "HealthyRootRequired" + } Test-Contract "absence with explicit healthy root accepted" { $scenario.cases[0].expected = @(@{ kind = "absent"; target = $target; healthyRoot = $anchor }) Save-Inputs; $null = Accept diff --git a/windows-tests/ui-scenarios/README.md b/windows-tests/ui-scenarios/README.md index 92364db0..fb280e18 100644 --- a/windows-tests/ui-scenarios/README.md +++ b/windows-tests/ui-scenarios/README.md @@ -142,6 +142,17 @@ pwsh -NoProfile -File Tools\windows\Test-UiScenario.ps1 ` -AuthoritySha256 $trustedHash -ClaimId 818-C1 ``` +`pwsh -File` supports a single claim argument here; it does not turn a +comma-separated native argument into an array. To select multiple claims, +invoke the script from PowerShell with an actual array: + +```powershell +& .\Tools\windows\Test-UiScenario.ps1 ` + -ScenarioRoot windows-tests\ui-scenarios ` + -AuthorityPath C:\trusted-authority\818.json ` + -AuthoritySha256 $trustedHash -ClaimId @("818-C1", "818-C2") +``` + This repository currently has **zero** real scenarios, so that production discovery cannot pass. Do not use synthetic functional fixtures as UI evidence. Run the offline contracts instead: @@ -163,7 +174,7 @@ with a bounded refusal code, without echoing input contents. Refusal codes identify parsing (`InvalidJson`, `DocumentRoot`, `DocumentSize`, `DuplicateProperty`), schema (`SchemaInvalid`, `UnsafeSchemaReference`), binding (`ClaimAuthorityMismatch`, `DuplicateClaim`), discovery/selection -(`EmptySelection`, `UnknownClaim`, `DuplicateRequest`, `MissingScenario`, +(`ScenarioRoot`, `EmptySelection`, `UnknownClaim`, `DuplicateRequest`, `MissingScenario`, `ScenarioFileIdentity`, `DuplicateClaimFile`), fixtures (`UnsafePath`, `UnsafeFixturePath`, `DuplicateFixtureFile`, `FixtureMismatch`, `FixtureFileMismatch`), coverage (`ProofClassMismatch`, `StateNotApplicable`, From d1abb6ec0f9ca2e3820915426c367ddbd0188ae0 Mon Sep 17 00:00:00 2001 From: Colin Neilens Date: Wed, 14 Oct 2026 11:36:05 -0700 Subject: [PATCH 3/3] Keep scenario reads off mapped network drives Exercise the no-read refusal with a controlled drive-type probe. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 017c733a-252b-471a-9c43-26a6a67554df Signed-off-by: Colin Neilens --- Tools/windows/Test-UiScenario.ps1 | 7 +++++++ Tools/windows/Tests/UiScenario.Tests.ps1 | 9 ++++++++- windows-tests/ui-scenarios/README.md | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Tools/windows/Test-UiScenario.ps1 b/Tools/windows/Test-UiScenario.ps1 index 9784cb4b..4110c1e3 100644 --- a/Tools/windows/Test-UiScenario.ps1 +++ b/Tools/windows/Test-UiScenario.ps1 @@ -19,12 +19,19 @@ function Stop-UiScenario([string] $code) { throw $error } +function Get-UiScenarioDriveType([string] $root) { + return [IO.DriveInfo]::new($root).DriveType +} + function Assert-UiScenarioPath([string] $path) { try { $full = [IO.Path]::GetFullPath($path) if ($full.StartsWith("\\") -or $full.IndexOf(":", 2) -ge 0) { Stop-UiScenario "UnsafePath" } + if ((Get-UiScenarioDriveType ([IO.Path]::GetPathRoot($full))) -eq [IO.DriveType]::Network) { + Stop-UiScenario "UnsafePath" + } $current = $full while ($current) { $item = Get-Item -LiteralPath $current -Force diff --git a/Tools/windows/Tests/UiScenario.Tests.ps1 b/Tools/windows/Tests/UiScenario.Tests.ps1 index b021ec78..a6c3c202 100644 --- a/Tools/windows/Tests/UiScenario.Tests.ps1 +++ b/Tools/windows/Tests/UiScenario.Tests.ps1 @@ -86,7 +86,7 @@ function Test-Contract([string] $name, [scriptblock] $body) { $results.Add(@{ name = $name; passed = $true }) Write-Host "PASS: $name" } -function Assert-PathRefusedBeforeRead([string] $path) { +function Assert-PathRefusedBeforeRead([string] $path, [IO.DriveType] $driveType = [IO.DriveType]::Fixed) { $errors = $null $ast = [Management.Automation.Language.Parser]::ParseFile($validator, [ref]$null, [ref]$errors) Assert-Contract ($errors.Count -eq 0) "validator does not parse" @@ -96,6 +96,10 @@ function Assert-PathRefusedBeforeRead([string] $path) { $node.Name -cin @("Stop-UiScenario", "Assert-UiScenarioPath") }, $true)) { . ([scriptblock]::Create($definition.Extent.Text)) } $script:pathGuardReads = 0 + function Get-UiScenarioDriveType { + param([string] $root) + return $driveType + } function Get-Item { param([string] $LiteralPath, [switch] $Force) $script:pathGuardReads++ @@ -166,6 +170,9 @@ try { } } } + Test-Contract "mapped network drive refuses before filesystem access" { + Assert-PathRefusedBeforeRead 'Z:\owned\scenario' ([IO.DriveType]::Network) + } Test-Contract "authority reparse ancestor refused" { $directory = Join-Path $scratch "authority-copy" $null = New-Item -ItemType Directory -Path $directory -Force diff --git a/windows-tests/ui-scenarios/README.md b/windows-tests/ui-scenarios/README.md index fb280e18..d3393e81 100644 --- a/windows-tests/ui-scenarios/README.md +++ b/windows-tests/ui-scenarios/README.md @@ -123,6 +123,8 @@ case-insensitive Windows duplicate detection. Absolute, empty-segment, traversal, device, alternate-stream, trailing-dot/space, invalid Windows character, missing, directory-as-file, and reparse-point paths fail. No reparse-point ancestor is accepted, including for authority or scenario paths. +Caller paths also reject UNC/device/alternate-stream syntax after normalization, +and mapped network drives before reading their contents. The caller must supply owned, privacy-safe packaged data and keep it stable during validation. Static path checks and hashes are not protection against a concurrent hostile filesystem mutation, filesystem attestation, or proof that