diff --git a/tests/README.md b/tests/README.md index d6c443825..20906674c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1273,6 +1273,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_codex_custom_azure_openai_mps` | Select custom Azure OpenAI MPS through the configure picker, then launch Codex without a provider override | Saved provider; completed TUI file task on the selected GPT deployment; normal exit; no routing | | `test_ug_configure_claude_bedrock_mps` | Select the restricted Bedrock MPS in the real configure picker; launch Claude without an explicit model | Saved provider in status; Sonnet 4.6 is selected from the Sonnet + Haiku allow-list and completes a TUI file task; normal exit | | `test_ug_configure_claude_bedrock_mantle_mps` | Select the Mantle MPS in the real configure picker; launch Claude on Opus 5.5 without a provider override | Saved provider and Claude provider header; completed TUI file task on the selected model; normal exit; no routing | +| `test_ug_codex_desktop_macos_skill_calls_mcp`, `test_ug_codex_desktop_windows_skill_calls_mcp` | Reuse the existing CI credential in an isolated CLI token profile, configure Codex with --use-pat, add web-search MCP, and invoke a local skill | Verified CLI identity before configure/add; hidden reference token on screen and in a completed managed-model desktop turn; successful native MCP receipts with the hidden query in that same turn. Uses the saved-token path with inherited bearer overrides removed; browser OAuth and token refresh are not covered | | `test_ug_configure_codex_databricks` | Configure Databricks Hosted; execute the generated auth helper; open Codex TUI and read a file | Generated helper invokes `ug` with clean token stdout; completed assistant answer contains the file value; normal exit and reopen | | `test_ug_configure_codex_openai_mps` | Select OpenAI MPS in the real configure picker; launch Codex | Saved provider in status; completed TUI file task; normal exit | | `test_ug_configure_codex_bedrock_runtime_mps` | Select the Bedrock Runtime MPS in the real configure picker; launch Codex on its GPT-5.6 inference profile without a provider override | Saved provider in status; completed TUI file task on the selected model; normal exit; no routing | @@ -1452,6 +1453,27 @@ and managed-settings coverage remain deferred; a green installation check alone does not establish a successful live task. Required macOS and Windows jobs open the pinned ChatGPT desktop app after managed `ug configure`, send a prompt through the GUI, and assert the exact answer, model, and completed turn from the app's rollout. +Both also invoke a user-authored `$CODEX_HOME/skills` skill whose reference file supplies a hidden +query and completion token, after registering `system.ai.web_search` with `ug mcp add`. +The token must render in the GUI and complete on the managed model; native successful MCP receipts +must carry the query in that same Codex Desktop/Databricks turn. The pinned app versions are +macOS `26.1103.31037` and Windows `26.1103.7773.0`; harness-latest also selects the macOS journey. +`test_codex_desktop_evidence.py` rejects false positives from failed/missing calls, unrelated +turns/queries/servers, wrong origins/models and echoed or incomplete answers. Skill/MCP evidence +selects the single root rollout and retains any child rollouts in the artifacts. Missing or +ambiguous roots fail, and successful child evidence cannot satisfy a failed or incomplete root. +These are offline parser checks; live desktop coverage requires the native CI jobs. +Both inference and skill journeys wait for the typed draft, click Send, and require the submitted user prompt +in the rollout before waiting for an answer. Windows first completes its visible sandbox setup. +The app's saved elevated sandbox mode proves completion even after its short-lived banner disappears; +the tests only read that native setting. Missing or partial TOML is polled until the setup +deadline; component checks cover recovery and deadline failure. Windows artifacts retain early MCP/setup diagnostics. +If the app's setup restart leaves a sandbox-readiness error, the journey uses the visible +Try again control once before sending the prompt; a persistent error fails. No task is resubmitted. +Offline submission checks reject assistant echoes, unrelated prompts and non-message events. +The native MCP approval is answered with Allow once only when the dialog names web_search on +the expected server with the exact hidden query. Offline checks reject other tools, servers, +queries and ambiguous approval controls. The `All integration tests` check requires every selected integration job to pass, including both managed-config lanes for full/live runs; full coverage does not depend on a label or a manual request. @@ -1599,7 +1621,7 @@ verify that neither a sudo capability probe nor a prompting worker is started. | Follow-up turns and conversation resume | Not covered; reopen proves startup, not conversation resume | | Claude/Codex interactive smart routing | First-prompt routing through `first_prompt_and_subagent_no_orch_v0` is covered by the `managed_fixture` banner journeys and CUJ4's preset journeys. CUJ4's managed journeys verify that `smart_routing.enabled` alone leaves Claude's first prompt unrouted and turns on Smart Router Orchestrator for Claude, and that Codex's managed default stays unrouted and unorchestrated; the Claude managed-default skill-toggle cases and a Codex launch-only opt-out case cover those paths; `--enable-smart-routing` has component coverage only. The live subagent-only journeys spawn native children and invoke the installed Smart Router skill to verify routing at launch, off, and off -> on in separate Claude/Codex sessions. The scheduled harness-latest workflow repeats CUJ4, routing hooks, skill toggles, and banners with both latest agents. Plugin-refresh survival and native daemon/background dispatch remain deferred. | | Full allow/deny tool-permission matrix | Not covered; onboarding/trust uses actual TUI choices | -| Desktop Codex app, Isaac itself, auto-upgrades | Not covered by command forwarding or pinned-version tests. Codex desktop's shared-config catalog pointer (repeat configure and a user-owned pointer) is covered by `integration/test_ug_configure_codex_desktop.py`; GUI inference on macOS and Windows by `integration/test_ug_codex_desktop_macos.py` and `integration/test_ug_codex_desktop_windows.py`; desktop tracing and Linux GUI are not | +| Desktop Codex app, Isaac itself, auto-upgrades | Not covered by command forwarding or pinned-version tests. Codex desktop's shared-config catalog pointer (repeat configure and a user-owned pointer) is covered by `integration/test_ug_configure_codex_desktop.py`; GUI inference, local skill execution and MCP web-search invocation on macOS and Windows by `integration/test_ug_codex_desktop_macos.py` and `integration/test_ug_codex_desktop_windows.py`; UC skill downloads through the desktop, connector browser sign-in, desktop tracing and Linux GUI are not | | Native macOS/Windows live TUI, managed settings, resize/signals | Windows fresh-install, CLI, local helpers, and the Claude headless gateway journey are advisory; Codex Windows CI is blocked on npm proxy access. Live PTY/TUI, managed settings, and signal behavior still need separate platform implementation and coverage | | Other agents | Live scope is Claude Code and Codex, plus a non-blocking OpenCode headless case | | OpenCode `--model` / `-m` | Unit/component tests cover raw CLI forwarding, configured model selection, launch arguments, rejection of unknown models before starting the native process, and the existing failure when no default model is available; no dedicated live integration journey | diff --git a/tests/integration/README.md b/tests/integration/README.md index a57a855f7..ee564e41b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -365,6 +365,46 @@ accessibility tree captured before the fatal error in command evidence. Automation: each screen gets 60 seconds and the answer four minutes, with no retries, and it saves a screenshot of the app window as evidence. It also requires the app's model picker to list only the managed model, under its readable name (`5.6 Sol`) rather than the raw id. +Both inference cases wait for the rendered draft, click Send, and require the user prompt in +the native rollout before the answer wait. Failed answer waits also retain rollout evidence. +Each module also runs `test_ug_codex_desktop_*_skill_calls_mcp`. It reuses the same +CI OAuth bearer as the inference cases, saving it in an isolated CLI token profile through +`databricks configure --host --profile codex-desktop-ci` with redacted stdin. +`databricks current-user me` verifies that profile before configure/add. The test removes +bearer and bearer-command overrides and uses the default CLI config location. UG selects +`--profile codex-desktop-ci --use-pat`, its existing saved-token mode also used by the Cowork +connector tests, so the MCP proxy reads the credential from disk without forwarded auth +environment variables. No new PAT is minted; the credential remains the runner's short-lived +OAuth token. The profile is removed with the disposable home and is never uploaded. +Browser sign-in, OAuth-cache lookup and token refresh are not covered. The token must remain +valid for the journey; expiry fails the test. The inference-only cases keep their existing +bearer setup. +After managed configure, +`ug mcp add --agents codex --names system.ai.web_search` registers the real search service. +A user-authored skill in the session's `$CODEX_HOME/skills` points to a reference file containing a fresh query +and completion token. Windows completes sandbox setup through the visible Continue button. +It waits through a missing or partially written config within the same setup deadline, then +reads the elevated mode saved by the app after success, without writing native settings or +depending on the transient success banner, and retains early MCP/setup diagnostics in evidence. +After setup's app-server restart, a visible readiness error gets one Try again recheck before +Send. A persistent error fails; this does not resubmit a task or change sandbox permissions. +Both platforms wait for the typed draft, click Send, and require the user prompt in the native +rollout before the answer wait. The GUI prompt names only the skill. Success requires the token on screen, +a completed Codex Desktop/Databricks turn on the managed model, and native `McpToolCall` +receipts in that same turn with the reference query, successful status and nonempty MCP content. +The skill/MCP check selects the single root rollout while retaining child rollouts as evidence. +Missing or ambiguous roots fail; a child's successful call or answer cannot satisfy the root's checks. +Every matching call is checked; additional successful calls are allowed, including calls made +through code mode. Native approval dialogs must name web_search on the expected server and show +the hidden query before the test clicks Allow once; unexpected or ambiguous approvals fail. +These journeys use the same pinned apps and managed workspace as inference; +the workspace identity must have access to `system.ai.web_search`. They cover local skill +execution and MCP invocation, not UC skill downloads, connector browser sign-in or search ranking. +`../test_codex_desktop_evidence.py` checks hidden reference inputs and rejects missing/failed +receipts, unrelated servers/queries/turns, wrong origins/models and incomplete answers offline. +It also rejects false submission evidence and checks profile setup failure propagation, +removal of inherited credentials, and redaction of token stdin/stdout/stderr from evidence. +These component checks do not establish live authentication or MCP execution. `scripts/run_integration.py` ignores both modules unless `--desktop-app` is passed, which only the desktop CI jobs do, so a local run never drives a developer's installed app. @@ -1147,8 +1187,8 @@ test_ug_codex_v2_routing.py # exact native v2 assignments and route test_ug_configure_claude_lifecycle.py # repeat setup, revert, rejected credentials test_ug_configure_claude_desktop.py # Desktop lifecycle, system.ai/relay inference, installed Cowork MCP; no GUI test_ug_configure_codex_desktop.py # stubbed managed Codex desktop catalog pointer, no GUI -test_ug_codex_desktop_macos.py # ChatGPT desktop app on macOS: GUI prompt, exact answer, model, completed turn -test_ug_codex_desktop_windows.py # the same journey on Windows through UI Automation +test_ug_codex_desktop_macos.py # macOS GUI: inference plus a local skill invoking real MCP web_search +test_ug_codex_desktop_windows.py # the same journeys on Windows through UI Automation test_ug_configure_claude_workspace_switch.py # real skills MCP cleanup across two workspaces test_ug_configure_codex_lifecycle.py # repeat setup, revert, rejected credentials test_ug_configure_scope.py # files written by default, --user, UG_SKIP_OS_SETTINGS, --global; launch after each @@ -1646,7 +1686,7 @@ OpenAI's versioned appcast zip, pinned by version and SHA256 in `integration.yml Accessibility to the runner's shell, so no permission prompt is needed. It uploads `integration-codex-desktop-macos` evidence. To update the app, take a newer `` from `https://persistent.oaistatic.com/codex-app-prod/appcast.xml` and its SHA256. The -harness-latest workflow runs the same journey against the newest appcast build, so it shows +harness-latest workflow runs the same journeys against the newest appcast build, so it shows when a bump is due. The required **Codex desktop inference · Windows** job installs the ChatGPT MSIX that OpenAI keeps at `codex-app-prod/releases//ChatGPT-x64.msix`, pinned by version and SHA256 in diff --git a/tests/integration/test_ug_codex_desktop_macos.py b/tests/integration/test_ug_codex_desktop_macos.py index 286de880a..f39bcda94 100644 --- a/tests/integration/test_ug_codex_desktop_macos.py +++ b/tests/integration/test_ug_codex_desktop_macos.py @@ -4,6 +4,15 @@ from pathlib import Path import pytest +from utils.codex_desktop import ( + MCP_SERVICE, + SKILL_NAME, + assert_desktop_skill_mcp_session, + configure_databricks_token_profile, + wait_for_skill_answer, + wait_for_submitted_prompt, + write_search_skill, +) from utils.evidence import ( agent_sessions, assert_completed_task_model, @@ -28,7 +37,7 @@ def test_ug_codex_desktop_macos_completes_a_gateway_turn(live_session, workspace Expected: the app shows the answer, and its own Codex Desktop rollout, using ug's Databricks provider, records exactly that answer in a completed turn whose model is the managed system.ai GPT default. Onboarding is skipped through its visible buttons with mouse clicks and the prompt - is typed with real keystrokes; nothing is seeded. GUI rendering of other screens, MCP, tracing + is typed with real keystrokes, then submitted with Send; nothing is seeded. GUI rendering of other screens, MCP, tracing and revert while the app runs are not covered. A subsequent click on an absent label fails with the native accessibility tree before the fatal error, captured in command evidence. """ @@ -46,8 +55,15 @@ def test_ug_codex_desktop_macos_completes_a_gateway_turn(live_session, workspace app.click("AXButton", "Go to ChatGPT") # Click the composer so the keystrokes cannot land in another control. app.click("AXTextArea", None, timeout=120) - app.type_line(f"Reply with exactly {marker} and nothing else.") - app.wait_for("AXStaticText", marker, 300) + prompt = f"Reply with exactly {marker} and nothing else." + app.type_line(prompt, submit=False) + app.wait_for("AXTextArea", prompt, 60) + app.click("AXButton", "Send") + try: + wait_for_submitted_prompt(session, prompt) + app.wait_for("AXStaticText", marker, 300) + finally: + session.record("agent-sessions.json", agent_sessions(session, "codex")) # The answer can render before the rollout records the completed turn; keep the app open. deadline = time.monotonic() + 60 while not completed_task_models(session, "codex", marker): @@ -91,3 +107,56 @@ def test_ug_codex_desktop_macos_completes_a_gateway_turn(live_session, workspace assert tree_rows, lines assert all(len(row.split("\t")) == 4 for row in tree_rows), tree_rows assert any(row.split("\t")[0].startswith("AX") for row in tree_rows), tree_rows + + +def test_ug_codex_desktop_macos_skill_calls_mcp(live_session, workspace): + """Scenario: use an authenticated profile, add web_search, and invoke a skill in the desktop GUI. + + Expected: the existing CI credential is saved through the CLI and its identity verified before + configure/add. The app and MCP use that token profile with no inherited bearer override. + Browser OAuth and refresh are not covered. The skill reference supplies the hidden query and token. + The app displays that token and its Databricks-backed completed turn records successful + native MCP web_search receipts with the exact query on the managed model. User onboarding + and Send use visible controls; the rollout must record the submitted prompt before the + answer wait. The expected search is approved with Allow once; no approval/config state is + seeded. UC skill downloads, connector + browser sign-in and built-in web search are outside this journey. + """ + session = live_session + profile = configure_databricks_token_profile(session, workspace) + marker = fresh_id() + query = f"Databricks Unity Catalog documentation {fresh_id()}" + write_search_skill(session.home, query, marker) + use_managed_config_fixture(session, "managed_workspace_default") + session.run( + "configure", + "--agents", + "codex", + "--profile", + profile, + "--use-pat", + "--skip-upgrade", + timeout=240, + ) + session.run("mcp", "add", "--agents", "codex", "--names", MCP_SERVICE, timeout=240) + + with MacApp(session, CHATGPT_APP, "chatgpt") as app: + app.click("AXRadioButton", "Engineering", timeout=180) + app.click("AXButton", "Continue") + app.click("AXButton", "Skip") + app.click("AXButton", "Go to ChatGPT") + try: + app.click("AXTextArea", None, timeout=120) + prompt = f"Use ${SKILL_NAME}. Follow its instructions and reply with only its completion token." + app.type_line(prompt, submit=False) + app.wait_for("AXTextArea", prompt, 60) + app.click("AXButton", "Send") + wait_for_submitted_prompt(session, prompt) + wait_for_skill_answer(app, query, marker, "AXStaticText", "AXButton", 300) + finally: + session.record("agent-sessions.json", agent_sessions(session, "codex")) + deadline = time.monotonic() + 60 + while not completed_task_models(session, "codex", marker): + assert time.monotonic() < deadline, "The app never recorded the completed skill turn" + time.sleep(1) + assert_desktop_skill_mcp_session(session, query, marker, MANAGED_CODEX_MODEL) diff --git a/tests/integration/test_ug_codex_desktop_windows.py b/tests/integration/test_ug_codex_desktop_windows.py index 5b9f7e4f8..18c419c18 100644 --- a/tests/integration/test_ug_codex_desktop_windows.py +++ b/tests/integration/test_ug_codex_desktop_windows.py @@ -3,6 +3,17 @@ import time import pytest +from utils.codex_desktop import ( + MCP_SERVICE, + SKILL_NAME, + assert_desktop_skill_mcp_session, + configure_databricks_token_profile, + finish_windows_readiness_check, + wait_for_skill_answer, + wait_for_submitted_prompt, + wait_for_windows_sandbox, + write_search_skill, +) from utils.evidence import ( agent_sessions, assert_completed_task_model, @@ -33,7 +44,7 @@ def test_ug_codex_desktop_windows_completes_a_gateway_turn(live_session, workspa the answer, and its own Codex Desktop rollout, using ug's Databricks provider, records exactly that answer in a completed turn whose model is the managed system.ai GPT default. Onboarding is skipped through its visible controls with UI Automation and the prompt is typed with real - keystrokes; nothing is seeded. Each screen gets 60 seconds and the answer four minutes, + keystrokes followed by Send; nothing is seeded. Each screen gets 60 seconds and the answer four minutes, without retries. GUI rendering of other screens, MCP, tracing and revert while the app runs are not covered. """ @@ -59,8 +70,15 @@ def test_ug_codex_desktop_windows_completes_a_gateway_turn(live_session, workspa app.click("ControlType.Button", MODEL_PICKER) # Focus the composer so the keystrokes cannot land in another control. app.click("ControlType.Edit", None) - app.type_line(f"Reply with exactly {marker} and nothing else.") - app.wait_for("ControlType.Text", marker, ANSWER_TIMEOUT) + prompt = f"Reply with exactly {marker} and nothing else." + app.type_line(prompt, submit=False) + app.wait_for("ControlType.Edit", prompt) + app.click("ControlType.Button", "Send") + try: + wait_for_submitted_prompt(session, prompt) + app.wait_for("ControlType.Text", marker, ANSWER_TIMEOUT) + finally: + session.record("agent-sessions.json", agent_sessions(session, "codex")) # The answer can render before the rollout records the completed turn; keep the app open. deadline = time.monotonic() + 60 while not completed_task_models(session, "codex", marker): @@ -78,3 +96,64 @@ def test_ug_codex_desktop_windows_completes_a_gateway_turn(live_session, workspa answers = [answer for records in sessions for answer in assistant_answers("codex", records)] assert [answer.strip() for answer in answers] == [marker], answers assert_completed_task_model(session, "codex", marker, MANAGED_CODEX_MODEL) + + +def test_ug_codex_desktop_windows_skill_calls_mcp(live_session, workspace): + """Scenario: use an authenticated profile, add web_search, and invoke a skill in the desktop GUI. + + Expected: the existing CI credential is saved through the CLI and its identity verified before + configure/add. The app and MCP use that token profile with no inherited bearer override. + Browser OAuth and refresh are not covered. The skill reference supplies the hidden query and token. + The app displays that token and its Databricks-backed completed turn records successful + native MCP web_search receipts with the exact query on the managed model. User onboarding, + Windows sandbox setup and Send use visible controls; the rollout must record the prompt + before the answer wait. Setup must save elevated sandbox mode; a readiness error after its + app-server restart gets one visible recheck before Send. The expected search is + approved with Allow once; no approval/config + state is seeded. UC skill downloads, connector + browser sign-in and built-in web search are outside this journey. + """ + session = live_session + profile = configure_databricks_token_profile(session, workspace) + marker = fresh_id() + query = f"Databricks Unity Catalog documentation {fresh_id()}" + write_search_skill(session.home, query, marker) + use_managed_config_fixture(session, "managed_workspace_default") + session.run( + "configure", + "--agents", + "codex", + "--profile", + profile, + "--use-pat", + "--skip-upgrade", + timeout=240, + ) + session.run("mcp", "add", "--agents", "codex", "--names", MCP_SERVICE, timeout=240) + + with WindowsApp(session, CHATGPT_PACKAGE, CHATGPT_EXECUTABLE, "chatgpt") as app: + app.click("ControlType.RadioButton", "Engineering") + app.click("ControlType.Button", "Continue") + app.click("ControlType.Button", "Skip") + app.click("ControlType.Button", "Go to ChatGPT") + try: + app.wait_for("ControlType.Text", "Finish Windows setup to continue") + app.click("ControlType.Button", "Continue") + wait_for_windows_sandbox(app) + app.click("ControlType.Edit", None) + prompt = f"Use ${SKILL_NAME}. Follow its instructions and reply with only its completion token." + app.type_line(prompt, submit=False) + app.wait_for("ControlType.Edit", prompt) + finish_windows_readiness_check(app) + app.click("ControlType.Button", "Send") + wait_for_submitted_prompt(session, prompt) + wait_for_skill_answer( + app, query, marker, "ControlType.Text", "ControlType.Button", ANSWER_TIMEOUT + ) + finally: + session.record("agent-sessions.json", agent_sessions(session, "codex")) + deadline = time.monotonic() + 60 + while not completed_task_models(session, "codex", marker): + assert time.monotonic() < deadline, "The app never recorded the completed skill turn" + time.sleep(1) + assert_desktop_skill_mcp_session(session, query, marker, MANAGED_CODEX_MODEL) diff --git a/tests/integration/utils/codex_desktop.py b/tests/integration/utils/codex_desktop.py new file mode 100644 index 000000000..317c6ca84 --- /dev/null +++ b/tests/integration/utils/codex_desktop.py @@ -0,0 +1,205 @@ +"""User-authored skill input and native rollout evidence for the desktop journeys.""" + +import time +import tomllib +from pathlib import Path + +from .agents.codex import completed_task_models +from .evidence import agent_sessions, is_child_session + +MCP_SERVICE = "system.ai.web_search" +MCP_SERVER = "system-ai-web_search" +SKILL_NAME = "ug-desktop-search" +SEARCH_APPROVAL = f'Allow the {MCP_SERVER} MCP server to run tool "web_search"?' + + +def configure_databricks_token_profile(session, workspace: str) -> str: + """Reuse the suite's credential through the saved-token path used by desktop connectors.""" + token = session.env.pop("DATABRICKS_BEARER").strip() + assert token, "The desktop profile requires the existing integration bearer" + # Use the default profile location so MCP startup needs no forwarded auth environment. + for name in ("DATABRICKS_BEARER_COMMAND", "DATABRICKS_CONFIG_FILE"): + session.env.pop(name, None) + profile = "codex-desktop-ci" + cli = session.which("databricks") + session.run( + "configure", + "--host", + workspace, + "--profile", + profile, + binary=cli, + input_text=token + "\n", + timeout=30, + ) + session.run("current-user", "me", "--profile", profile, binary=cli, timeout=30) + return profile + + +def finish_windows_readiness_check(app, timeout: float = 60) -> None: + """Setup restarts the app-server; finish a failed readiness check before sending any task.""" + error = "Couldn’t check Windows setup" + tree = app.read_tree() + if not any(row[0] == "ControlType.Text" and error in row[1:] for row in tree): + return + app.session.record("windows-readiness-recheck.json", {"tree": tree}) + app.click("ControlType.Button", "Try again") + deadline = time.monotonic() + timeout + while True: + tree = app.read_tree() + if not any(row[0] == "ControlType.Text" and error in row[1:] for row in tree): + return + if time.monotonic() >= deadline: + app.session.record( + "chatgpt-tree.json", {"waiting_for": "Sandbox readiness", "tree": tree} + ) + raise AssertionError("Windows sandbox readiness still failed after its visible recheck") + time.sleep(2) + + +def wait_for_windows_sandbox(app, timeout: float = 180) -> None: + """The app saves this mode after successful setup; its success banner is transient.""" + deadline = time.monotonic() + timeout + path = app.session.home / ".codex/config.toml" + while True: + try: + mode = tomllib.loads(path.read_text(encoding="utf-8")).get("windows", {}).get("sandbox") + except (FileNotFoundError, tomllib.TOMLDecodeError): + mode = None + if mode == "elevated": + app.session.record("windows-sandbox.json", {"mode": mode}) + return + if time.monotonic() >= deadline: + app.session.record( + "chatgpt-tree.json", + {"waiting_for": "Windows sandbox setup", "tree": app.read_tree()}, + ) + raise AssertionError(f"The app never completed elevated Windows sandbox setup: {mode}") + time.sleep(1) + + +def wait_for_skill_answer( + app, query: str, marker: str, text_role: str, button_role: str, timeout: float +) -> None: + """Approve only the requested search through the GUI, then await its hidden answer.""" + deadline = time.monotonic() + timeout + while True: + tree = app.read_tree() + labels = [label for row in tree if row[0] == text_role for label in row[1:]] + approvals = [ + index + for index, row in enumerate(tree) + if row[0] == button_role and "Allow once" in row[1:] + ] + if approvals: + app.session.record("mcp-approval-tree.json", {"tree": tree}) + titles = [ + index + for index, row in enumerate(tree) + if row[0] == text_role and SEARCH_APPROVAL in row[1:] + ] + assert len(approvals) == len(titles) == 1 and any( + row[0] == text_role and query in row[1:] for row in tree[titles[0] : approvals[0]] + ), "Unexpected tool approval; expected only web_search with the skill's query" + app.click(button_role, "Allow once") + elif marker in labels: + return + if time.monotonic() >= deadline: + app.session.record("chatgpt-tree.json", {"waiting_for": marker, "tree": tree}) + raise AssertionError("The desktop app never displayed the completed skill answer") + time.sleep(2) + + +def wait_for_submitted_prompt(session, prompt: str, timeout: float = 60) -> None: + """Require the prompt in the native rollout before waiting for its answer.""" + deadline = time.monotonic() + timeout + while True: + sessions = agent_sessions(session, "codex") + if any( + row.get("type") == "response_item" + and (payload := row.get("payload", {})).get("type") == "message" + and payload.get("role") == "user" + and any( + part.get("type") == "input_text" and part.get("text", "").strip() == prompt + for part in payload.get("content", []) + ) + for records in sessions.values() + for row in records + ): + return + if time.monotonic() >= deadline: + session.record("agent-sessions.json", sessions) + raise AssertionError("The desktop app never recorded the submitted prompt") + time.sleep(1) + + +def write_search_skill(home: Path, query: str, marker: str) -> Path: + """An ordinary user skill, not generated ug state or a substitute MCP server.""" + # CODEX_HOME is explicit in the isolated session, including on native Windows. + skill = home / ".codex/skills" / SKILL_NAME + reference = skill / "references/task.md" + reference.parent.mkdir(parents=True) + (skill / "SKILL.md").write_text( + f"---\nname: {SKILL_NAME}\n" + "description: Search with the configured Unity Gateway MCP service.\n---\n" + "Read references/task.md beside this skill. It contains the search query and completion " + f"token. Call the {MCP_SERVER} MCP server's web_search tool with that exact query. " + "After the tool succeeds, reply with only the completion token. " + "Do not use built-in web search or subagents.\n", + encoding="utf-8", + ) + reference.write_text(f"Query: {query}\nCompletion token: {marker}\n", encoding="utf-8") + return skill + + +def assert_skill_mcp_turn(records: list[dict], query: str, marker: str, model: str) -> None: + """Tie the hidden skill values and successful MCP receipts to the app's completed turn.""" + origins = [ + (row["payload"].get("originator"), row["payload"].get("model_provider")) + for row in records + if row.get("type") == "session_meta" + ] + assert origins == [("Codex Desktop", "Databricks")], origins + assert completed_task_models(records, marker) == {model} + turns = [ + row["payload"]["turn_id"] + for row in records + if row.get("type") == "event_msg" + and row.get("payload", {}).get("type") == "task_complete" + and row["payload"].get("last_agent_message", "").strip() == marker + ] + assert len(turns) == 1, f"Expected one completed desktop answer with the skill token: {turns}" + calls = [ + item + for row in records + if row.get("type") == "event_msg" + and (payload := row.get("payload", {})).get("type") == "item_completed" + and payload.get("turn_id") == turns[0] + and (item := payload.get("item", {})).get("type") == "McpToolCall" + and item.get("server") == MCP_SERVER + and item.get("tool") == "web_search" + ] + assert calls, "No native web_search receipt in the desktop's completed skill turn" + for call in calls: + assert call.get("arguments", {}).get("query") == query, call + assert call.get("status") == "completed" and not call.get("error"), call + result = call.get("result") + assert isinstance(result, dict) and not result.get("isError"), call + assert any( + part.get("type") == "text" + and isinstance(part.get("text"), str) + and part["text"].strip() + for part in result.get("content", []) + ), call + + +def assert_desktop_skill_mcp_session(session, query: str, marker: str, model: str) -> None: + sessions = agent_sessions(session, "codex") + session.record("agent-sessions.json", sessions) + roots = { + path: records + for path, records in sessions.items() + if not is_child_session("codex", path, records) + } + assert len(roots) == 1, f"Expected one desktop root session, got {list(roots)}" + assert_skill_mcp_turn(next(iter(roots.values())), query, marker, model) diff --git a/tests/integration/utils/macos_app.py b/tests/integration/utils/macos_app.py index 29913a877..6ff71e55c 100644 --- a/tests/integration/utils/macos_app.py +++ b/tests/integration/utils/macos_app.py @@ -86,14 +86,15 @@ def click(self, role: str, label: str | None, timeout: float = 60) -> None: "click", str(self.proc.pid), role, label or "", binary=self.client, timeout=60 ) - def type_line(self, text: str) -> None: - """Type into the focused control with real keystrokes, then press Return.""" + def type_line(self, text: str, *, submit: bool = True) -> None: + """Type with real keystrokes, optionally pressing Return.""" + enter = "key code 36\n" if submit else "" script = ( "on run argv\n" 'tell application "System Events"\n' "set frontmost of (first process whose unix id is (item 1 of argv as integer)) to true\n" "keystroke (item 2 of argv)\n" - "key code 36\n" + f"{enter}" "end tell\n" "end run" ) diff --git a/tests/integration/utils/windows_app.py b/tests/integration/utils/windows_app.py index 9e02c8ffb..355540ae4 100644 --- a/tests/integration/utils/windows_app.py +++ b/tests/integration/utils/windows_app.py @@ -56,7 +56,17 @@ def __exit__(self, *exc): finally: stop_process(self.proc) log = self.session.redact(self.log.read_text(errors="replace")) - self.session.record(f"{self.name}-log.json", {"log": log[-20000:]}) + self.session.record( + f"{self.name}-log.json", + { + "log": log[-20000:], + "mcp_and_setup": "\n".join( + line + for line in log.splitlines() + if "mcp" in line.lower() or "sandbox" in line.lower() + )[-20000:], + }, + ) def uia(self, command: str, *args: str) -> str: assert self.proc.poll() is None, f"{self.name} exited with {self.proc.returncode}" @@ -106,6 +116,6 @@ def click(self, control: str, name: str | None, timeout: float = STEP_TIMEOUT) - self.wait_for(control, name, timeout) self.uia("invoke", control, name or "") - def type_line(self, text: str) -> None: - """Type into the focused control with real keystrokes, then press Enter.""" - self.uia("type", text) + def type_line(self, text: str, *, submit: bool = True) -> None: + """Type with real keystrokes, optionally pressing Enter.""" + self.uia("type", text, "enter" if submit else "") diff --git a/tests/integration/utils/windows_uia.ps1 b/tests/integration/utils/windows_uia.ps1 index bad599ad9..5b81a6823 100644 --- a/tests/integration/utils/windows_uia.ps1 +++ b/tests/integration/utils/windows_uia.ps1 @@ -1,6 +1,6 @@ # A UI Automation client for one app: `tree PID` prints its windows' enabled, on-screen elements, -# `invoke PID TYPE NAME` activates the first matching one (any name if NAME is empty), `type PID TEXT` -# types into the focused control with real keystrokes and presses Enter, and `screenshot PID PATH` +# `invoke PID TYPE NAME` activates the first matching one (any name if NAME is empty), `type PID TEXT [enter]` +# types into the focused control with real keystrokes and optionally presses Enter, and `screenshot PID PATH` # saves a PNG of its window. param( [Parameter(Mandatory)][ValidateSet('tree', 'invoke', 'type', 'screenshot')][string]$Command, @@ -10,6 +10,7 @@ param( ) $ErrorActionPreference = 'Stop' +[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes, System.Windows.Forms, System.Drawing $automation = [Windows.Automation.AutomationElement] $byProcess = [Windows.Automation.PropertyCondition]::new($automation::ProcessIdProperty, $ProcessId) @@ -58,7 +59,8 @@ switch ($Command) { $focused = $automation::FocusedElement.Current.ProcessId if ($focused -ne $ProcessId) { throw "Keyboard focus is in process $focused, not $ProcessId" } # SendKeys treats these characters as commands unless braced. - [Windows.Forms.SendKeys]::SendWait(($First -replace '[+^%~(){}\[\]]', '{$0}') + '{ENTER}') + [Windows.Forms.SendKeys]::SendWait(($First -replace '[+^%~(){}\[\]]', '{$0}')) + if ($Second -eq 'enter') { [Windows.Forms.SendKeys]::SendWait('{ENTER}') } } 'screenshot' { $rect = $windows[0].Current.BoundingRectangle diff --git a/tests/test_codex_desktop_evidence.py b/tests/test_codex_desktop_evidence.py new file mode 100644 index 000000000..fab048695 --- /dev/null +++ b/tests/test_codex_desktop_evidence.py @@ -0,0 +1,369 @@ +"""Reject false positives in the native desktop skill/MCP evidence checks.""" + +import copy +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.integration.utils import codex_desktop +from tests.integration.utils.codex_desktop import ( + MCP_SERVER, + SKILL_NAME, + assert_skill_mcp_turn, + write_search_skill, +) +from tests.integration.utils.harness import UserSession + +QUERY = "Databricks Unity Catalog documentation a13b42" +MARKER = "b39c21" +MODEL = "system.ai.gpt-5-6-sol" + + +@pytest.mark.parametrize("failure", [None, "configure", "identity"]) +def test_token_profile_removes_bypasses_and_stops_on_failure(tmp_path, failure): + calls = [] + session = SimpleNamespace( + env={ + "HOME": str(tmp_path), + "DATABRICKS_BEARER": "ci-token", + "DATABRICKS_BEARER_COMMAND": "credential-broker", + "DATABRICKS_CONFIG_FILE": "other-profile-file", + }, + which=lambda name: tmp_path / name, + ) + + def run(*args, **kwargs): + assert session.env == {"HOME": str(tmp_path)} + assert kwargs["binary"] == tmp_path / "databricks" + assert "ci-token" not in args + calls.append(args) + if args[0] == "configure": + assert kwargs["input_text"] == "ci-token\n" + if failure == "configure": + raise AssertionError("Profile setup failed") + elif failure == "identity": + raise AssertionError("Identity verification failed") + + session.run = run + if failure: + with pytest.raises(AssertionError, match="failed"): + codex_desktop.configure_databricks_token_profile(session, "https://workspace.example") + else: + profile = codex_desktop.configure_databricks_token_profile( + session, "https://workspace.example" + ) + assert calls == [ + ("configure", "--host", "https://workspace.example", "--profile", profile), + ("current-user", "me", "--profile", profile), + ] + assert len(calls) == (1 if failure == "configure" else 2) + + +def test_token_profile_credential_is_redacted_after_removing_session_bearer(tmp_path, monkeypatch): + token = "inert-ci-oauth-token" + monkeypatch.setenv("DATABRICKS_BEARER", token) + root, project = tmp_path / "session", tmp_path / "project" + project.mkdir() + session = UserSession(root, project, Path(sys.executable), tmp_path / "artifacts") + assert "DATABRICKS_BEARER" not in session.env + result = session.run( + "-c", + "import sys; token = sys.stdin.read(); print(token); print(token, file=sys.stderr)", + input_text=token + "\n", + ) + evidence = (session.artifacts / "command-1.json").read_text() + assert token not in evidence + assert json.loads(evidence)["stdin"] == "\n" + assert result.stdout.strip() == result.stderr.strip() == "" + + +@pytest.mark.parametrize("state", ["ready", "recovers", "fails"]) +def test_windows_readiness_recheck_is_visible_and_bounded(state): + error = [["ControlType.Text", "Couldn’t check Windows setup", ""]] + tree = [] if state == "ready" else list(error) + clicks = [] + + def click(role, label): + clicks.append((role, label)) + if state == "recovers": + tree.clear() + + app = SimpleNamespace( + read_tree=lambda: tree, click=click, session=SimpleNamespace(record=lambda *args: None) + ) + if state == "fails": + with pytest.raises(AssertionError, match="readiness still failed"): + codex_desktop.finish_windows_readiness_check(app, timeout=0) + else: + codex_desktop.finish_windows_readiness_check(app, timeout=0) + assert clicks == ([] if state == "ready" else [("ControlType.Button", "Try again")]) + + +@pytest.mark.parametrize("mode", [None, "unelevated", "elevated"]) +def test_windows_setup_requires_the_native_elevated_mode(tmp_path, mode): + config = tmp_path / ".codex/config.toml" + config.parent.mkdir() + config.write_text(f'[windows]\nsandbox = "{mode}"\n' if mode else "") + evidence = {} + app = SimpleNamespace( + read_tree=lambda: [], + session=SimpleNamespace( + home=tmp_path, record=lambda name, value: evidence.update({name: value}) + ), + ) + if mode == "elevated": + codex_desktop.wait_for_windows_sandbox(app, timeout=0) + assert evidence["windows-sandbox.json"] == {"mode": "elevated"} + else: + with pytest.raises(AssertionError, match="never completed elevated Windows sandbox setup"): + codex_desktop.wait_for_windows_sandbox(app, timeout=0) + assert "chatgpt-tree.json" in evidence + + +@pytest.mark.parametrize("initial", [None, "[windows"]) +@pytest.mark.parametrize("recovers", [False, True]) +def test_windows_setup_waits_for_missing_or_partial_config( + tmp_path, monkeypatch, initial, recovers +): + config = tmp_path / ".codex/config.toml" + config.parent.mkdir() + if initial is not None: + config.write_text(initial) + elapsed = [0] + + def sleep(seconds): + elapsed[0] += seconds + if recovers: + config.write_text('[windows]\nsandbox = "elevated"\n') + + monkeypatch.setattr(codex_desktop.time, "monotonic", lambda: elapsed[0]) + monkeypatch.setattr(codex_desktop.time, "sleep", sleep) + app = SimpleNamespace( + read_tree=lambda: [], + session=SimpleNamespace(home=tmp_path, record=lambda *args: None), + ) + if recovers: + codex_desktop.wait_for_windows_sandbox(app, timeout=2) + assert elapsed[0] == 1 + else: + with pytest.raises(AssertionError, match="never completed elevated Windows sandbox setup"): + codex_desktop.wait_for_windows_sandbox(app, timeout=2) + assert elapsed[0] == 2 + + +@pytest.mark.parametrize( + ("text_role", "button_role"), + [("AXStaticText", "AXButton"), ("ControlType.Text", "ControlType.Button")], +) +@pytest.mark.parametrize("invalid", [None, "server", "tool", "query", "echoed_query", "ambiguous"]) +def test_gui_approves_only_the_requested_search(monkeypatch, text_role, button_role, invalid): + approval = codex_desktop.SEARCH_APPROVAL + if invalid == "server": + approval = approval.replace(MCP_SERVER, "other-server") + if invalid == "tool": + approval = approval.replace('"web_search"', '"other-tool"') + tree = [ + [text_role, "", approval], + [text_role, "", "query"], + [text_role, "", "other query" if invalid in {"query", "echoed_query"} else QUERY], + [button_role, "Allow once", ""], + ] + if invalid == "ambiguous": + tree.append([button_role, "Allow once", ""]) + if invalid == "echoed_query": + tree.insert(0, [text_role, "", QUERY]) + clicks = [] + + def click(role, label): + clicks.append((role, label)) + tree[:] = [[text_role, "", MARKER]] + + app = SimpleNamespace( + read_tree=lambda: tree, + click=click, + session=SimpleNamespace(record=lambda *args: None), + ) + monkeypatch.setattr(codex_desktop.time, "sleep", lambda seconds: None) + if invalid: + with pytest.raises(AssertionError, match="Unexpected tool approval"): + codex_desktop.wait_for_skill_answer(app, QUERY, MARKER, text_role, button_role, 1) + assert not clicks + else: + codex_desktop.wait_for_skill_answer(app, QUERY, MARKER, text_role, button_role, 1) + assert clicks == [(button_role, "Allow once")] + + +@pytest.mark.parametrize( + ("kind", "role", "text", "submitted"), + [ + ("response_item", "user", "Use the skill.\n", True), + ("response_item", "assistant", "Use the skill.", False), + ("response_item", "user", "Another prompt.", False), + ("event_msg", "user", "Use the skill.", False), + ], +) +def test_submission_requires_the_native_user_prompt(monkeypatch, kind, role, text, submitted): + sessions = { + "desktop": [ + { + "type": kind, + "payload": { + "type": "message", + "role": role, + "content": [{"type": "input_text", "text": text}], + }, + } + ] + } + monkeypatch.setattr(codex_desktop, "agent_sessions", lambda *args: sessions) + evidence = {} + session = SimpleNamespace(record=lambda name, value: evidence.update({name: value})) + if submitted: + codex_desktop.wait_for_submitted_prompt(session, "Use the skill.", timeout=0) + else: + with pytest.raises(AssertionError, match="never recorded the submitted prompt"): + codex_desktop.wait_for_submitted_prompt(session, "Use the skill.", timeout=0) + assert evidence["agent-sessions.json"] == sessions + + +@pytest.fixture +def records(): + return [ + { + "type": "session_meta", + "payload": {"originator": "Codex Desktop", "model_provider": "Databricks"}, + }, + {"type": "turn_context", "payload": {"turn_id": "turn-1", "model": MODEL}}, + { + "type": "event_msg", + "payload": { + "type": "item_completed", + "turn_id": "turn-1", + "item": { + "type": "McpToolCall", + "id": "call-1", + "server": MCP_SERVER, + "tool": "web_search", + "arguments": {"query": QUERY}, + "status": "completed", + "result": {"content": [{"type": "text", "text": "Databricks documentation"}]}, + }, + }, + }, + { + "type": "event_msg", + "payload": { + "type": "task_complete", + "turn_id": "turn-1", + "last_agent_message": MARKER, + }, + }, + ] + + +def test_skill_values_are_only_in_the_reference_file(tmp_path): + skill = write_search_skill(tmp_path, QUERY, MARKER) + + assert skill == tmp_path / ".codex/skills" / SKILL_NAME + instructions = (skill / "SKILL.md").read_text() + assert "references/task.md" in instructions and MCP_SERVER in instructions + assert QUERY not in instructions and MARKER not in instructions + assert (skill / "references/task.md").read_text() == ( + f"Query: {QUERY}\nCompletion token: {MARKER}\n" + ) + + +def test_native_mcp_receipts_allow_extra_successful_calls(records): + assert_skill_mcp_turn(records, QUERY, MARKER, MODEL) + extra_call = copy.deepcopy(records[2]) + extra_call["payload"]["item"]["id"] = "call-2" + records.insert(3, extra_call) + assert_skill_mcp_turn(records, QUERY, MARKER, MODEL) + + +@pytest.fixture +def desktop_session(monkeypatch, records): + child = copy.deepcopy(records) + child[0]["payload"]["source"] = {"subagent": {"thread_spawn": {"parent_thread_id": "root"}}} + sessions = {"child.jsonl": child, "root.jsonl": records} + evidence = {} + session = SimpleNamespace( + sessions=sessions, + evidence=evidence, + record=lambda name, value: evidence.update({name: value}), + ) + monkeypatch.setattr(codex_desktop, "agent_sessions", lambda *args: sessions) + return session + + +def test_desktop_root_allows_child_rollouts_and_records_all_evidence(desktop_session): + codex_desktop.assert_desktop_skill_mcp_session(desktop_session, QUERY, MARKER, MODEL) + assert desktop_session.evidence["agent-sessions.json"] == desktop_session.sessions + + +@pytest.mark.parametrize("root_count", [0, 2]) +def test_missing_or_ambiguous_desktop_roots_fail(desktop_session, root_count): + if root_count == 0: + del desktop_session.sessions["root.jsonl"] + else: + desktop_session.sessions["another-root.jsonl"] = desktop_session.sessions["root.jsonl"] + with pytest.raises(AssertionError, match="Expected one desktop root session"): + codex_desktop.assert_desktop_skill_mcp_session(desktop_session, QUERY, MARKER, MODEL) + + +@pytest.mark.parametrize( + ("index", "changes"), + [ + (0, {"originator": "codex_cli_rs"}), + (0, {"model_provider": "OpenAI"}), + (1, {"model": "another-model"}), + (1, {"turn_id": "another-turn"}), + (2, {"turn_id": "earlier-turn"}), + (2, {"type": "item_started"}), + (3, {"type": "agent_message"}), + (3, {"last_agent_message": f"I could not finish {MARKER}"}), + ], +) +def test_wrong_origin_model_turn_or_answer_fails(records, desktop_session, index, changes): + records[index]["payload"].update(changes) + with pytest.raises(AssertionError): + codex_desktop.assert_desktop_skill_mcp_session(desktop_session, QUERY, MARKER, MODEL) + + +@pytest.mark.parametrize( + "changes", + [ + {"type": "WebSearch"}, + {"server": "another-server"}, + {"tool": "another-tool"}, + {"arguments": {"query": "an unrelated query"}}, + {"status": "inProgress"}, + {"status": "failed"}, + {"error": {"message": "permission denied"}}, + {"result": None}, + {"result": {"isError": True, "content": [{"type": "text", "text": "error"}]}}, + {"result": {"content": []}}, + {"result": {"content": [{"type": "text", "text": " "}]}}, + ], +) +def test_every_matching_mcp_receipt_must_succeed(records, changes): + records[2]["payload"]["item"].update(changes) + with pytest.raises(AssertionError): + assert_skill_mcp_turn(records, QUERY, MARKER, MODEL) + + +def test_success_cannot_hide_another_failed_matching_call(records): + failed_call = copy.deepcopy(records[2]) + failed_call["payload"]["item"].update(id="failed-call", status="failed") + records.insert(3, failed_call) + with pytest.raises(AssertionError): + assert_skill_mcp_turn(records, QUERY, MARKER, MODEL) + + +def test_final_answer_alone_cannot_prove_tool_use(records, desktop_session): + del records[2] + with pytest.raises(AssertionError, match="No native web_search receipt"): + codex_desktop.assert_desktop_skill_mcp_session(desktop_session, QUERY, MARKER, MODEL) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index b9909e05c..7487ac5fa 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -148,7 +148,8 @@ def test_codex_desktop_macos_ci_is_blocking_and_installs_a_pinned_app(): assert "shasum -a 256 -c" in job assert "--desktop-app" in job assert _collect_integration_cases(_ci_marker_expression(job, "codex")) == { - "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_completes_a_gateway_turn" + "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_completes_a_gateway_turn", + "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_skill_calls_mcp", } @@ -166,7 +167,8 @@ def test_codex_desktop_windows_ci_is_blocking_and_installs_a_pinned_app(): assert "-ne $env:CHATGPT_APP_SHA256" in job assert "--desktop-app" in job assert _collect_integration_cases(_ci_marker_expression(job, "codex")) == { - "test_ug_codex_desktop_windows.py::test_ug_codex_desktop_windows_completes_a_gateway_turn" + "test_ug_codex_desktop_windows.py::test_ug_codex_desktop_windows_completes_a_gateway_turn", + "test_ug_codex_desktop_windows.py::test_ug_codex_desktop_windows_skill_calls_mcp", } @@ -187,7 +189,8 @@ def test_latest_harness_runs_the_newest_signed_chatgpt_app(): assert '"TeamIdentifier=$OPENAI_TEAM_ID"' in workflow assert "${{ matrix.suite == 'desktop' && '--desktop-app' || '' }}" in workflow assert _collect_integration_cases(entry.group(2)) == { - "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_completes_a_gateway_turn" + "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_completes_a_gateway_turn", + "test_ug_codex_desktop_macos.py::test_ug_codex_desktop_macos_skill_calls_mcp", }