diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 428c88528..426dfa45c 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -141,8 +141,15 @@ async def load_by_signed_url(env: MCPServerEnv | WebsiteEnv, environment_artifac if urls[path] is None: raise RuntimeError(f"{deployed_by}, so its server fetches {file.object_url} itself, and {type(stores[path]).__name__} " "can't sign a URL for it; loading into such an env needs an object store that signs URLs (S3, Cloud Storage)") - await protocol_v1.reset_data(base_url, timeout=timeout) - await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, urls), timeout=timeout) + headers = data_plane_headers(env, base_url) + await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) + await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, urls), timeout=timeout, headers=headers) + + +def data_plane_headers(env: MCPServerEnv | WebsiteEnv, url: str) -> dict[str, str] | None: + """The headers a call to ``url`` sends of ``env``'s record's control token; None, which leaves the call to a run's + or a multi env's grant, when the record gives it none: no record, no token, or an address outside the record's.""" + return (env._deployed.control_headers_for(url) or None) if env._deployed is not None else None async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> bool: @@ -161,7 +168,8 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo child = env._deployed.child_env(name, mcp=mcp) if env._deployed is not None and env._deployed.environment_card else None if child is not None and child.serves(METHOD_RESET) is False: return False - await protocol_v1.reset_data(base_url, timeout=DATA_PLANE_LOAD_TIMEOUT_S) + await protocol_v1.reset_data(base_url, timeout=DATA_PLANE_LOAD_TIMEOUT_S, + headers=data_plane_headers(env, base_url)) elif builtin is None: return False else: diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index bb959deb3..44f5c02dd 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -21,7 +21,8 @@ from agent_env.env import legacy_protocol from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, + reset_server_data, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -186,8 +187,10 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif # reset_data previously took the client default (30s), tighter than the load it # precedes -- dropping and recreating a large service's schemas can plausibly # exceed that. Give it the same budget as the load it belongs to. - await protocol_v1.reset_data(base_url, timeout=timeout) - await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=timeout) + headers = data_plane_headers(self, base_url) + await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) + await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=timeout, + headers=headers) finally: if folder is not None: await self._remove_from_container(folder) diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index e8c7f0ca1..3239c4ed3 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional import yaml +from agentenv_protocol import client as protocol_v1 logger = logging.getLogger(__name__) @@ -461,11 +462,13 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif # MCP servers are checked first; if a website shares the same environment_name, the MCP server wins. for env in self.mcp_server_envs: if env.environment_name == environment_artifact.environment_name: - await env.load_environment_artifact(environment_artifact) + with _record_grant(self._deployed): + await env.load_environment_artifact(environment_artifact) return for env in self.website_envs: if env.environment_name == environment_artifact.environment_name: - await env.load_environment_artifact(environment_artifact) + with _record_grant(self._deployed): + await env.load_environment_artifact(environment_artifact) return if (computer := self._computers.holding(environment_artifact.environment_name)) is not None: await computer.load_environment_artifact(environment_artifact) @@ -1067,7 +1070,8 @@ async def reset(self, deployed: DeployedEnv) -> None: for env in services.mcp_server_envs), *((f"website {env.environment_name!r}", reset_server_data(env, mcp=False)) for env in services.website_envs)] - results = await _multi_computers.each(self.id, "reset", resets) + with _record_grant(deployed): + results = await _multi_computers.each(self.id, "reset", resets) for (child, _), result in zip(resets, results): if result is False: logger.info(f"multi env '{self.id}': {child} serves no reset and takes no load, so it stays as it is") @@ -1122,6 +1126,14 @@ def _gateway_state_provider(env: MultiEnv) -> DatabaseStateProvider | None: return env._env_provider._state_provider if isinstance(env._env_provider, EnvironmentGatewayProvider) else None +def _record_grant(deployed: Optional[DeployedEnv]): + """The grant of ``deployed``'s control token to calls under its address and its gateway's, for a child env that a + name it shares with another keeps from the record (``_hand_record_to_child_envs``), whose calls go to the gateway.""" + if deployed is None: + return protocol_v1.authorized({}) + return protocol_v1.authorized({url: deployed.control_headers for url in (deployed.environment_url, gateway_url_of(deployed))}) + + def _hand_record_to_child_envs(env: MultiEnv, deployed: DeployedEnv) -> None: """Give each child env the record its loads read the card from; a name an MCP server and a website share keeps the live probe, since the card can't tell them apart by name.""" shared = {c.environment_name for c in env.mcp_server_envs} & {c.environment_name for c in env.website_envs} diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 001ab5d46..b7a01c7d2 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -44,7 +44,8 @@ from agent_env.env import legacy_protocol from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, + reset_server_data, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -321,9 +322,10 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif try: staged = {path: f"file://{await self._copy_artifact_into_container(file, paths[path])}" for path, file in files.items()} - await protocol_v1.reset_data(base_url) + headers = data_plane_headers(self, base_url) + await protocol_v1.reset_data(base_url, headers=headers) await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), - timeout=DATA_PLANE_LOAD_TIMEOUT_S) + timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers) finally: if folder is not None: await self._remove_from_container(folder) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 06ecbcfd3..d2425becf 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -176,8 +176,9 @@ def sources(names: list[str]) -> str: TOOL_DISABLE_ACTION = "disable" TOOL_ENABLE_ACTION = "enable" # The harness's routes that take the deploy's control token: the ones that change tool access, triggers and -# notifications. Every other route stays open for now, the clock's writes included: callers outside agent-env set the -# clock over plain HTTP and don't send the token yet. +# notifications. The gateway also takes it for the data plane and for an MCP server's own endpoint behind its proxy +# (Gateway._takes_control_token). Every other route stays open for now, the clock's writes included: callers outside +# agent-env set the clock over plain HTTP and don't send the token yet. CONTROL_ROUTES = frozenset({ f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}", "/triggers/register", "/triggers/remove", "/triggers/clear", diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index a7322b676..9bba5539a 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -86,6 +86,8 @@ called_tools, fetch_child_card, forward, + is_child_mcp_path, + reaches, listed_url, lowered, operations, @@ -178,7 +180,8 @@ def __init__( self.rest_proxy_urls = rest_proxy_urls or {} self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} self._listed_by_name = frozenset(listed_by_name) - self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) + self._mcp_endpoints = mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) + self._token_paths = token_paths(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) self._site_cards: dict[str, SiteCard] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) self._browsers = BrowserRoutes(self.rest_proxy_urls, self.REST_PROXY_TIMEOUT, self._log_event, @@ -545,8 +548,9 @@ def _asgi_app(self): async def app(scope, receive, send): if self._control_bearer and scope["type"] in ("http", "websocket"): headers = scope.get("headers", ()) - if scope["type"] == "http" and scope["path"] in CONTROL_ROUTES and not hmac.compare_digest( + if scope["type"] == "http" and self._takes_control_token(scope) and not hmac.compare_digest( dict(headers).get(b"authorization", b""), self._control_bearer): + logger.warning("refused %s %s: it takes the env's control token", scope["method"], scope["path"]) response = self._json_response({"ok": False, "error": "this route takes the env's control token"}, 401) await response(scope, receive, send) return @@ -569,6 +573,19 @@ async def app(scope, receive, send): return app + def _takes_control_token(self, scope: dict) -> bool: + """Whether a request takes the deploy's control token: a control route, the env's data plane, or one of + ``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``reaches``), + after dot-segment normalisation. A site's /website route never reaches its data plane.""" + path = scope["path"] + if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): + return True + prefix, _, rest = path.lstrip("/").partition("/") + service, _, rest = rest.partition("/") + guarded = self._token_paths.get(service, ()) if prefix == "svc" else () + raw, base = scope.get("raw_path") or path.encode(), self.rest_proxy_urls.get(service, "") + return any(reaches(base, raw, path, rest, endpoint) for endpoint in guarded) + @asynccontextmanager async def _session_lifespan(self, mcp: FastMCP): """Once per inbound MCP session: the session's key joins the caller, and a connect gets to heal a failed @@ -784,12 +801,12 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | if base_url is None: return self._json_response({"error": f"Unknown service: {service_name}"}, 404) role = self._request_role(request) - site_mcp = self._site_mcp_endpoints.get(service_name) if url_map is self.rest_proxy_urls else None - mcp_server = url_map is self.rest_proxy_urls and (service_name.startswith("mcp-") or site_mcp is not None) - mcp_endpoint = site_mcp or "mcp" + mcp_endpoint = self._mcp_endpoints.get(service_name) if url_map is self.rest_proxy_urls else None + mcp_server = mcp_endpoint is not None + mcp_endpoint = mcp_endpoint or "mcp" if url_map is self.rest_proxy_urls and (cdp := await self._browsers.cdp_answer(request, base_url, role)) is not None: return cdp - if mcp_server and request.method == "POST" and reaches_child_mcp(request, mcp_endpoint): + if mcp_server and request.method == "POST" and reaches_child_mcp(request, base_url, mcp_endpoint): if (refusal := await self._refuse_disabled_calls(await request.body(), role)) is not None: return refusal if request.method in _SAFE_METHODS: @@ -1741,6 +1758,36 @@ def site_mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, return endpoints +def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str], + rest_proxy_urls: dict[str, str]) -> dict[str, str]: + """The MCP endpoint, as a path under its /svc// route, of each backing route that is an MCP server's: a site + that is an internal MCP server too (``site_mcp_endpoints``), and each ``mcp-`` route at the path its server's + ``mcp_url`` gives, ``/mcp`` when none is under the route's address. Where the proxy stamps the role and applies the + tool rules, and where the control token is taken.""" + endpoints = site_mcp_endpoints(servers, website_urls, rest_proxy_urls) + for key, base in rest_proxy_urls.items(): + if key in endpoints or not key.startswith("mcp-"): + continue + name = route_environment_name(key, base, servers, website=False) + found = [resolved.route_path for server in servers if server.name == name + if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server + and resolved.route_path is not None] + endpoints[key] = found[0] if found else "/mcp" + return endpoints + + +def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], + rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: + """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, + extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and + triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here.""" + endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls) + return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ())) + for key, base in rest_proxy_urls.items() + if key in website_urls or route_environment_name(key, base, servers, website=False) != key + or not any(server.name == key for server in servers)} + + class CardURL(NamedTuple): """A URL a site's card names, as the site at an address serves it: its path and query; the path a request on the site's route reaches it at, its path under the address's (None when it isn't under it); and whether it is on the diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 5a12faab3..1fecba0c0 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -105,7 +105,7 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx. return await _done(_json_response({"error": str(e)}, 400), on_done) content = await request.body() # /svc and /website request bodies are small (JSON-RPC, forms) headers = upstream_headers(request, role, withheld=withheld, site_names=site_names) - if mcp_server and request.method == "POST" and reaches_child_mcp(request, mcp_endpoint): + if mcp_server and request.method == "POST" and reaches_child_mcp(request, base_url, mcp_endpoint): # The gateway is the trust boundary for identity: a child's proxied MCP endpoint stays reachable, but the # role in its JSON-RPC `_meta` is the gateway's to write, like the header above. content = stamp_jsonrpc_role(content, role) @@ -274,17 +274,26 @@ def is_child_mcp_path(decoded_rest: str, endpoint: str = "mcp") -> bool: return normalized == at or normalized.startswith(at + "/") -def reaches_child_mcp(request: HTTPConnection, endpoint: str = "mcp") -> bool: - """Whether a /svc request reaches the backing server's MCP endpoint (:func:`is_child_mcp_path`), by the path as - routed here or by the one the server receives: the raw path forwarded, its dot segments removed by httpx and then - decoded, which differ when an encoded slash sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``).""" - if is_child_mcp_path(request.path_params.get("path", ""), endpoint): +def reaches_child_mcp(request: HTTPConnection, base_url: str, endpoint: str = "mcp") -> bool: + """Whether a /svc request reaches the MCP endpoint of the backing server at ``base_url`` (:func:`reaches`).""" + return reaches(base_url, request.scope["raw_path"], request.scope["path"], request.path_params.get("path", ""), endpoint) + + +def reaches(base_url: str, raw_path: bytes, decoded_path: str, decoded_rest: str, endpoint: str) -> bool: + """Whether a /svc request whose routed remainder is ``decoded_rest`` reaches ``endpoint`` of the backing server at + ``base_url`` (:func:`is_child_mcp_path`): by that remainder, or by the path the server receives, the base's path and + the raw remainder forwarded, its dot segments removed by httpx and then decoded. They differ when an encoded slash + sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``) or a dot segment climbs into a base's path + (``../prefix/mcp`` reaches base ``/prefix``'s ``/mcp``). False for a request-target forward() refuses.""" + if is_child_mcp_path(decoded_rest, endpoint): return True try: - sent = httpx.URL(upstream_url("http://child", request)).path + raw_rest = raw_remainder(raw_path, decoded_path, decoded_rest) if decoded_rest else b"" + sent = httpx.URL(f"{base_url.rstrip('/')}/{raw_rest.decode('latin-1')}").path + base = httpx.URL(base_url).path.rstrip("/") except (ValueError, httpx.InvalidURL): - return False # forward() refuses a request-target it can't build a URL for - return is_child_mcp_path(sent, endpoint) + return False + return is_child_mcp_path(sent, f"{base}/{endpoint.lstrip('/')}") def stamp_jsonrpc_role(body: bytes, role: str) -> bytes: diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index ec9747648..3bee24e39 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Optional import httpx +from agentenv_protocol.client import control_headers from agentenv_protocol.website import ( SESSION_SECRET_ENV, SITE_READY_TIMEOUT_S, @@ -983,7 +984,7 @@ async def _stand_up(_spec): if card is None: raise RuntimeError("Tunnel did not become ready in time") if website_configs: - await _wait_for_websites(gateway_url, website_configs) + await _wait_for_websites(gateway_url, website_configs, headers=control_headers(control_token)) card = await self._wait_for_tunnel(gateway_url, timeout=120) or card card_read_at_utc = datetime.now(timezone.utc).isoformat() await self._refuse_tool_name_clashes(gateway_url) @@ -1802,8 +1803,10 @@ class WebsiteContractError(RuntimeError): async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConfig], *, - timeout_s: float = WEBSITE_READY_TIMEOUT_S, poll_s: float = WEBSITE_READY_POLL_S) -> None: - """Wait until each site is ready, as its card says through ``/svc//``: by ``website/v1``'s ``ready``, or, + headers: dict[str, str] | None = None, timeout_s: float = WEBSITE_READY_TIMEOUT_S, + poll_s: float = WEBSITE_READY_POLL_S) -> None: + """Wait until each site is ready, as its card says through ``/svc//``, asked with the gateway's control + ``headers``, since its data plane takes them: by ``website/v1``'s ``ready``, or, for a card without it or a site serving none, by its stored health path answering 200, which is deprecated. A site whose answer says how long it may take (``timeout_s``) is waited that long, or its own ``ready_timeout_s`` if longer, and ``WEBSITE_READY_MARGIN_S``; any other its ``ready_timeout_s`` and the margin, or ``timeout_s`` if @@ -1817,7 +1820,7 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf warned: set[str] = set() while waiting: for name, wc in list(waiting.items()): - ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path) + ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path, headers=headers) if declared is not None: limit[name] = max(declared, own[name]) + WEBSITE_READY_MARGIN_S if ready is None and name not in warned: @@ -1835,7 +1838,8 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf await asyncio.sleep(poll_s) -async def _website_ready(base_url: str, health_path: str) -> tuple[Optional[bool], str, Optional[float]]: +async def _website_ready(base_url: str, health_path: str, *, + headers: dict[str, str] | None = None) -> tuple[Optional[bool], str, Optional[float]]: """Whether the site at ``base_url`` is ready, why not, and how long it says it may take: True or False by the ``WebsiteReadiness`` its card's ``website/v1`` ``ready`` answers, as the protocol's ``website_readiness`` reads it, an answer that isn't one False; with no card or no ``ready`` on it, None once its health path answers 200 and False @@ -1843,7 +1847,7 @@ async def _website_ready(base_url: str, health_path: str) -> tuple[Optional[bool async with httpx.AsyncClient() as client: try: card = _card_of(await client.get(base_url + WELL_KNOWN_PATH, timeout=10), base_url) - readiness = None if card is None else await website_readiness(base_url, card, timeout=10) + readiness = None if card is None else await website_readiness(base_url, card, timeout=10, headers=headers) if readiness is not None: return readiness.ready, "" if readiness.ready else _not_ready_why(readiness), readiness.timeout_s health = await client.get(base_url + health_path, timeout=10) diff --git a/src/agent_env/task_step/task_steps/run_code.py b/src/agent_env/task_step/task_steps/run_code.py index 2652f4f01..59401cfdc 100644 --- a/src/agent_env/task_step/task_steps/run_code.py +++ b/src/agent_env/task_step/task_steps/run_code.py @@ -2,12 +2,12 @@ The script defines ``run(input) -> JSON``, where ``input = {"args", "results"}``. The result is stored at ``context.metadata["script_results"][result_id]``. -A script run for an env also gets ``input["env"] = {"id", "url", "mcp_url"}``: the env's +A script run for an env also gets ``input["env"] = {"id", "url", "mcp_url", "headers"}``: the env's address, where its card is at ``/.well-known/agent-env.json``, and its MCP endpoint, as the script -reaches them: on the env's host as that host does (``Sandbox.url_on_host``), since a host can't -always call its own public URL; inside a computer, the driver on loopback, with the ``headers`` that -carry the computer's control token to its framework-only routes. A script run in an agent's sandbox -never gets the token. +reaches them, and the ``headers`` that carry the env's control token to its framework-only routes and +its data plane: on the env's host as that host does (``Sandbox.url_on_host``), since a host can't +always call its own public URL; inside a computer, the driver on loopback. A script run in an agent's +sandbox never gets the token. ``script_artifact_id`` is either a single-file ``FileArtifact`` or a ``FileArtifactUniverse`` whose ``.py`` members are staged side by side so the entry @@ -538,12 +538,14 @@ def _is_safe_member(name: str) -> bool: def _env_on_host(deployed: DeployedSandboxEnv, sandbox: Any) -> dict: - """The env's id, address and MCP endpoint as a script on its host reaches them.""" + """The env's id, address and MCP endpoint as a script on its host reaches them, and the headers its control token + travels in.""" url, mcp_url = deployed.environment_url, deployed.mcp_url return { "id": deployed.env_id, "url": url and sandbox.url_on_host(url), "mcp_url": mcp_url and sandbox.url_on_host(mcp_url), + "headers": deployed.control_headers, } diff --git a/tst/integration/env/envs/multi_env_local_test.py b/tst/integration/env/envs/multi_env_local_test.py index 92535eede..50c369c60 100644 --- a/tst/integration/env/envs/multi_env_local_test.py +++ b/tst/integration/env/envs/multi_env_local_test.py @@ -53,14 +53,14 @@ async def test_a_multi_env_deploys_restores_loads_and_tears_down_on_local_compos assert isinstance(restored, MultiEnv) and restored._sandbox.work_dir == work_dir await restored.load_environment_universe_artifact(_items_universe(uid)) base_url = await legacy_protocol.v1_base_url(deployed, deployed.gateway_url, "items") - assert (await protocol_v1.get_data(base_url)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} + assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} slack_state = f"{legacy_protocol.environment_base_url(deployed.gateway_url, 'slack')}/export-state" assert len(httpx.get(slack_state, timeout=30).json()["channels"]) == 10 # A multi env without computers resets its servers, each as it resets alone, and fails if one doesn't. await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( TaskStepContext(deployed_envs=[deployed])) - assert (await protocol_v1.get_data(base_url)).model_dump()["parts"][0]["data"] == {"items": []} + assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": []} assert httpx.get(slack_state, timeout=30).json()["channels"] == [] await restored.close() diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 2cee30a98..2c1c779b5 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -984,7 +984,8 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) build_step = BuildMcpCliTaskStep(id=f"build-cli-{email_env.id}", version=None, env_id=email_env.id, command_name="email") ctx = TaskStepContext(deployed_envs=[result]) - await build_step.execute(ctx) + with protocol_v1.authorized({result.environment_url: result.control_headers}): + await build_step.execute(ctx) cli_ref = ctx.metadata["cli_artifact"] cli_artifact = Artifact.get(cli_ref["id"], version=cli_ref["version"]) @@ -1915,7 +1916,7 @@ async def test_gateway_changes_name_the_request_behind_a_write_through_the_rest_ async with httpx.AsyncClient() as client: seen = (await client.get(f"{gw}/changes", params={"limit": 10000}, timeout=30)).json()["changes"] after_id = seen[-1]["id"] if seen else 0 - async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "carol@example.com"}, timeout=60) as as_carol: + async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "carol@example.com", **result.control_headers}, timeout=60) as as_carol: async with streamable_http_client(f"{gw}/svc/mcp-email/mcp", http_client=as_carol) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() @@ -1952,7 +1953,7 @@ async def test_a_tool_disabled_for_a_role_is_refused_through_svc_as_on_the_gatew after_id = seen[-1]["id"] if seen else 0 answers = {} for role in ("carol@example.com", "dana@example.com"): - async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: role}, timeout=60) as as_role: + async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: role, **result.control_headers}, timeout=60) as as_role: for path in ("/mcp", "/svc/mcp-email/mcp"): async with streamable_http_client(f"{gw}{path}", http_client=as_role) as (read, write, _): async with ClientSession(read, write) as session: @@ -2187,7 +2188,7 @@ async def recording_send(self, request, *args, **kwargs): monkeypatch.undo() routes = [r for r in sent if r[1].startswith("/svc/")] assert routes == [("POST", "/svc/mcp-renameditems/agentenv")] * 2, routes - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["r"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["r"]} finally: await env.close() @@ -2238,23 +2239,23 @@ async def test_gateway_with_agentenv_environment(agentenv_items_env, monkeypatch } }, } - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} - await protocol_v1.add_data(base, [DataPart(data={"items": ["a", "b"]})]) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["a", "b"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} + await protocol_v1.add_data(base, [DataPart(data={"items": ["a", "b"]})], headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["a", "b"]} - await protocol_v1.reset_data(base) - await protocol_v1.add_data(base, [FilePart(file={"uri": "file:///data/seed.json", "mimeType": "application/json"})]) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["seeded-a", "seeded-b"]} + await protocol_v1.reset_data(base, headers=result.control_headers) + await protocol_v1.add_data(base, [FilePart(file={"uri": "file:///data/seed.json", "mimeType": "application/json"})], headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["seeded-a", "seeded-b"]} - async with httpx.AsyncClient(timeout=30) as client: + async with httpx.AsyncClient(timeout=30, headers=result.control_headers) as client: r = await client.post(f"{base}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/add", "params": {"parts": []}}) assert r.status_code == 200 and r.json()["error"]["code"] == -32602 r = await client.post(f"{base}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/bogus"}) assert r.json()["error"]["code"] == -32601 - await protocol_v1.reset_data(base) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} - await protocol_v1.add_data(base, [DataPart(data={"items": ["x"]})]) + await protocol_v1.reset_data(base, headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} + await protocol_v1.add_data(base, [DataPart(data={"items": ["x"]})], headers=result.control_headers) async def assert_tool_sees_items(session, tools_result, tool_names): assert "list_items" in tool_names, f"Expected list_items tool, got {tool_names}" @@ -2292,7 +2293,7 @@ async def assert_tool_sees_items(session, tools_result, tool_names): ), ) await env.load_environment_artifact(items_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["y", "z"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["y", "z"]} # Non-JSON EnvironmentArtifact loads via the file:// FilePart branch (no content_type special-case). csv_artifact = EnvironmentArtifact.put( @@ -2307,11 +2308,11 @@ async def assert_tool_sees_items(session, tools_result, tool_names): ), ) await env.load_environment_artifact(csv_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} # A file name with shell syntax in it is staged and loaded as it is, and runs nothing on the sandbox host. await _load_shell_named_artifact(env, "items", monkeypatch, tmp_path) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["shell-named"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["shell-named"]} assert not (tmp_path / "pwned").exists() # Environment Card extension end-to-end (urn:agentenv:set-errors/v1): a server-local @@ -2340,7 +2341,7 @@ async def _count_list_items_errors(n: int) -> int: assert await _count_list_items_errors(20) == 0 # no injection yet ack = await protocol_v1.invoke_extension( base, card, "urn:agentenv:set-errors/v1", - {"tool_name": "list_items", "error_rate": 0.75}, + {"tool_name": "list_items", "error_rate": 0.75}, headers=result.control_headers, ) assert ack == {"tool_name": "list_items", "error_rate": 0.75} assert await _count_list_items_errors(20) >= 1 # P(0 errors) = 0.25**20 ~ 1e-12 @@ -2390,13 +2391,13 @@ async def assert_list_items_disabled(session, tools_result, tool_names): ack = await protocol_v1.invoke_extension( result.gateway_url, items_child, "urn:agentenv:set-errors/v1", - {"tool_name": "list_items", "error_rate": 0.5}, + {"tool_name": "list_items", "error_rate": 0.5}, headers=result.control_headers, ) assert ack == {"tool_name": "list_items", "error_rate": 0.5} - await protocol_v1.reset_data(result.gateway_url) - await protocol_v1.add_data(result.gateway_url, [DataPart(data={"items": ["env-added"]})]) - assert (await protocol_v1.get_data(result.gateway_url)).parts[0].data == {"items": ["env-added"]} + await protocol_v1.reset_data(result.gateway_url, headers=result.control_headers) + await protocol_v1.add_data(result.gateway_url, [DataPart(data={"items": ["env-added"]})], headers=result.control_headers) + assert (await protocol_v1.get_data(result.gateway_url, headers=result.control_headers)).parts[0].data == {"items": ["env-added"]} finally: await env.close() @@ -2424,7 +2425,7 @@ async def test_snapshot_agent_state_v1_and_fallback(multi_env_items_email, items env_card = await protocol_v1.get_card(result.gateway_url) assert protocol_v1.find_child(env_card, "items") is not None assert protocol_v1.find_child(env_card, "email") is None - async with httpx.AsyncClient(timeout=30) as client: + async with httpx.AsyncClient(timeout=30, headers=result.control_headers) as client: r = await client.post(f"{result.gateway_url}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/reset"}) assert "exactly one" in r.json()["error"]["message"] @@ -2491,7 +2492,7 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm base = f"{result.gateway_url}/svc/webitems" assert await protocol_v1.supports_v1(base) is True - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} assert result.environment_card_url == f"{result.gateway_url}/.well-known/agent-env.json" env_card = await protocol_v1.get_card(result.gateway_url) @@ -2510,7 +2511,7 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm ), ) await env.load_environment_artifact(web_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["w1", "w2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["w1", "w2"]} # Non-JSON EnvironmentArtifact loads via the file:// FilePart branch. csv_artifact = EnvironmentArtifact.put( @@ -2525,10 +2526,10 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm ), ) await env.load_environment_artifact(csv_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} await _load_shell_named_artifact(env, "webitems", monkeypatch, tmp_path) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["shell-named"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["shell-named"]} assert not (tmp_path / "pwned").exists() finally: await env.close() diff --git a/tst/unit/env/envs/test_env_card_loads.py b/tst/unit/env/envs/test_env_card_loads.py index 4e2b4301a..37c385d18 100644 --- a/tst/unit/env/envs/test_env_card_loads.py +++ b/tst/unit/env/envs/test_env_card_loads.py @@ -18,8 +18,10 @@ import httpx import pytest from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI +from agentenv_protocol import client as protocol_v1 from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv +from agent_env.env.envs import _deployment from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv @@ -56,6 +58,33 @@ async def test_a_child_env_on_the_card_loads_through_its_v1_data_plane_with_no_c assert json.loads(sent[1].content)["params"]["parts"][0]["file"]["uri"] == "file:///data/email.json" +@pytest.mark.asyncio +async def test_a_load_sends_its_records_control_token_with_no_run_to_grant_it(sent): + deployed = _record(await _card("mcp-email", "shop")) + deployed.control_token = "s3cret" + web = _website() + web._sandbox, web._gateway_url, web._deployed = MagicMock(), _GW, deployed + web._copy_artifact_into_container = AsyncMock(return_value="/tmp/data/shop.json") + web._env_provider = MagicMock(spec=EnvironmentGatewayProvider, install_changelog_triggers=AsyncMock()) + + await _mcp(deployed).load_environment_artifact(_artifact("email")) + await web.load_environment_artifact(_artifact("shop")) + + assert [(str(r.url), r.headers.get("authorization")) for r in sent] == [ + *[(f"{_CARD}/svc/mcp-email/agentenv", "Bearer s3cret")] * 2, *[(f"{_CARD}/svc/shop/agentenv", "Bearer s3cret")] * 2] + + +def test_a_record_with_no_token_for_an_address_leaves_a_call_there_to_a_grant(): + env = _mcp(_record(None)) + env._deployed.control_token = "s3cret" + at_gateway, at_card = f"{_GW}/svc/mcp-email", f"{_CARD}/svc/mcp-email" + with protocol_v1.authorized({_GW: {"Authorization": "Bearer granted"}}): + sent = protocol_v1.request_headers(at_gateway, _deployment.data_plane_headers(env, at_gateway)) + assert sent == {"Authorization": "Bearer granted"} + assert _deployment.data_plane_headers(env, at_card) == {"Authorization": "Bearer s3cret"} + assert _deployment.data_plane_headers(_mcp(None), at_card) is None + + @pytest.mark.asyncio async def test_a_child_env_missing_from_the_card_takes_the_legacy_path(sent): # email serves no card, so the gateway leaves it out of the composed card. @@ -233,6 +262,27 @@ async def test_multi_env_withholds_the_record_from_a_name_an_mcp_server_and_a_we assert [email._deployed, slack._deployed, slack_web._deployed] == [deployed, None, None] +@pytest.mark.asyncio +async def test_a_name_an_mcp_server_and_a_website_share_loads_and_resets_with_the_records_token(sent): + deployed = _record(await _card("mcp-email", "slack")) + deployed.control_token = "s3cret" + provider = MagicMock(get_sandbox=AsyncMock(return_value=MagicMock(mode="vm"))) + with patch("agent_env.env.env.Env.get", side_effect=lambda *_: MultiEnv( + id="multi", version=1, mcp_server_envs=[_mcp_env(), _mcp_env("slack")], website_envs=[_website("slack")])), \ + patch("agent_env.providers.get_env_sandbox_provider", return_value=provider), \ + patch.object(EnvironmentGatewayProvider, "install_changelog_triggers", AsyncMock()): + env = await MultiEnv.from_deployed_env(deployed) + slack = env.mcp_server_envs[1] + slack._copy_artifact_into_container = AsyncMock(return_value="/data/slack.json") + slack._staged_artifact_size = AsyncMock(return_value=None) + await env.load_environment_artifact(_artifact("slack")) + await env.reset(deployed) + + assert sorted((r.url.path, r.headers.get("authorization")) for r in sent if r.url.path.endswith("/agentenv")) == [ + ("/sandbox/sb-1-18765/svc/mcp-email/agentenv", "Bearer s3cret"), + *[("/svc/mcp-slack/agentenv", "Bearer s3cret")] * 3, ("/svc/slack/agentenv", "Bearer s3cret")] + + async def _card(*keys: str) -> dict: """The real gateway's composed card, with a v1 backing server behind each key.""" gw = Gateway(host="127.0.0.1", port=0, server_name="env1234", internal_mcp_servers=[], diff --git a/tst/unit/env/envs/test_server_reset.py b/tst/unit/env/envs/test_server_reset.py index a73236dc1..42adeec6b 100644 --- a/tst/unit/env/envs/test_server_reset.py +++ b/tst/unit/env/envs/test_server_reset.py @@ -66,6 +66,15 @@ async def test_a_server_serving_data_reset_is_emptied_through_it_and_its_changel server.triggers.assert_awaited_once_with("email") +@pytest.mark.asyncio +async def test_a_reset_sends_its_records_control_token_with_no_run_to_grant_it(server): + env = _server(_card("data/reset", "data/add", "data/get")) + env._deployed.control_token = "s3cret" + + assert await _deployment.reset_server_data(env, mcp=True) is True + assert [request.headers.get("authorization") for request in server.requests] == ["Bearer s3cret"] + + @pytest.mark.asyncio async def test_a_server_whose_card_declares_no_data_reset_takes_no_load_and_is_left_as_it_is(server): assert await _deployment.reset_server_data(_server(_card()), mcp=True) is False diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index b561ba1c9..dce2ab210 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -1,11 +1,18 @@ -"""The gateway's control routes take the deploy's control token when it has one; every other route stays open.""" +"""The gateway's control routes, the data plane and an MCP server's own endpoint behind its proxy take the deploy's +control token when it has one; every other route stays open.""" from __future__ import annotations +import asyncio +import logging +from urllib.parse import unquote, urlsplit + import httpx import pytest from starlette.responses import JSONResponse -from agent_env.env.gateway.constants import CONTROL_ROUTES +from agent_env.env.gateway import InternalMCPServer +from agent_env.env.gateway import rest_proxy as rest_proxy_module +from agent_env.env.gateway.constants import CONTROL_ROUTES, WELL_KNOWN_PATH from agent_env.env.gateway.gateway import Gateway TOKEN = "s3cret" @@ -19,6 +26,20 @@ "/clock/set-time": "PUT", "/clock/advance": "PUT", "/clock/clear": "POST"} +# Behind the proxy: the data plane, its extensions, each service's included, and an MCP server's own endpoint, however +# the path is spelt on its way to the gateway. +GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"), + ("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"), + ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), + ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"), + ("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"), + ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")] +PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), + ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), + ("GET", "/website/shop/index.html")] +_RealAsyncClient = httpx.AsyncClient + + def _client(token: str | None) -> httpx.AsyncClient: gw = Gateway(host="127.0.0.1", port=0, server_name="unit", internal_mcp_servers=[], control_token=token) return httpx.AsyncClient(transport=httpx.ASGITransport(app=gw._asgi_app()), base_url="http://gateway") @@ -88,3 +109,130 @@ async def test_a_gateway_started_without_a_token_checks_none(path): async with _client(None) as client: response = await client.request(CONTROL[path], path, json={}) assert response.status_code != 401 + + +class _Backing: + """The services behind the gateway's proxy: each answers every request, and the paths reaching them are kept.""" + + def __init__(self, monkeypatch): + self.paths: list[str] = [] + self.authorization: list[str | None] = [] + transport = httpx.MockTransport(self._answer) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + + def _answer(self, request: httpx.Request) -> httpx.Response: + self.paths.append(request.url.path) + self.authorization.append(request.headers.get("authorization")) + + async def body(): + yield b'{"jsonrpc": "2.0", "id": 1, "result": {}}' + + return httpx.Response(200, headers={"content-type": "application/json"}, content=body()) + + +async def _proxied(token: str | None, method: str, path: str, headers: dict[str, str], **children) -> int: + """The status of one request through the gateway's served app, its path as uvicorn hands it: percent-decoded, its + dot segments and doubled slashes kept; an MCP server and a website behind it unless ``children`` say otherwise.""" + gw = Gateway(host="127.0.0.1", port=0, server_name="unit", control_token=token, **{ + "internal_mcp_servers": [], "rest_proxy_urls": {"mcp-svc": "http://svc", "shop": "http://shop-backend"}, + "website_urls": {"shop": "http://shop-backend"}, **children}) + url = urlsplit(path) + scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method, + "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"", + "root_path": "", "server": ("gateway", 80), "client": ("agent", 1), + "headers": [(b"content-type", b"application/json"), + *((name.lower().encode(), value.encode()) for name, value in headers.items())]} + received, started = False, [] + + async def receive(): + nonlocal received + if not received: + received = True + return {"type": "http.request", "body": b'{"jsonrpc": "2.0", "id": 1, "method": "data/reset"}', + "more_body": False} + await asyncio.Event().wait() + + async def send(message): + if message["type"] == "http.response.start": + started.append(message["status"]) + + await gw._asgi_app()(scope, receive, send) + return started[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +@pytest.mark.parametrize("headers", [{}, {"Authorization": "Bearer wrong"}], ids=["none", "wrong"]) +async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse_a_call_without_the_token( + monkeypatch, caplog, method, path, headers): + backing = _Backing(monkeypatch) + caplog.set_level(logging.WARNING, logger="agent_env.env.gateway.gateway") + assert await _proxied(TOKEN, method, path, headers) == 401 + assert backing.paths == [] + assert [r.getMessage() for r in caplog.records] == [f"refused {method} {unquote(path)}: it takes the env's control token"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_answer_a_call_with_the_token( + monkeypatch, method, path): + _Backing(monkeypatch) + assert await _proxied(TOKEN, method, path, {"Authorization": f"Bearer {TOKEN}"}) != 401 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", PROXIED_OPEN) +async def test_the_rest_of_the_proxy_stays_open(monkeypatch, method, path): + backing = _Backing(monkeypatch) + assert await _proxied(TOKEN, method, path, {}) != 401 + assert backing.paths + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy(monkeypatch, method, path): + _Backing(monkeypatch) + assert await _proxied(None, method, path, {}) != 401 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", [("POST", "/svc/mcp-x/rpc"), ("GET", "/svc/mcp-x/rpc/"), ("POST", "/svc/mcp-x/agentenv")]) +async def test_an_mcp_server_whose_endpoint_is_elsewhere_is_guarded_at_its_endpoint(monkeypatch, method, path): + backing = _Backing(monkeypatch) + servers = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url="http://x:8000/rpc")], + "rest_proxy_urls": {"mcp-x": "http://x:8000"}, "website_urls": {}} + assert await _proxied(TOKEN, method, path, {}, **servers) == 401 + assert await _proxied(TOKEN, method, path, {"Authorization": f"Bearer {TOKEN}"}, **servers) != 401 + assert backing.paths == [path.removeprefix("/svc/mcp-x")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/svc/desk/agentenv/ext/browser/cdp", "/svc/desk/mcp"]) +async def test_a_child_env_of_its_own_takes_its_own_token_through_the_proxy(monkeypatch, path): + backing = _Backing(monkeypatch) + computer = {"internal_mcp_servers": [InternalMCPServer(name="desk", mcp_url="http://desk:18765/mcp")], + "rest_proxy_urls": {"desk": "http://desk:18765"}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", path, {"Authorization": "Bearer the-computers"}, **computer) != 401 + forwarded = [(at, sent) for at, sent in zip(backing.paths, backing.authorization) if at != WELL_KNOWN_PATH] + assert forwarded == [(path.removeprefix("/svc/desk"), "Bearer the-computers")] + + +@pytest.mark.asyncio +async def test_a_live_websites_data_plane_is_guarded_though_it_has_no_website_route(monkeypatch): + backing = _Backing(monkeypatch) + live = {"rest_proxy_urls": {"github": "http://github-website-backend:8000"}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {}, **live) == 401 + assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {"Authorization": f"Bearer {TOKEN}"}, **live) != 401 + assert backing.paths == ["/agentenv"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("base", ["http://x:8000/prefix", "http://x:8000/%70refix"], ids=["plain", "encoded"]) +@pytest.mark.parametrize("path", ["/svc/mcp-x/../prefix/mcp", "/svc/mcp-x/../prefix/agentenv", "/svc/mcp-x/mcp", + "/svc/mcp-x/public%2Fx/../agentenv"]) +async def test_a_server_behind_a_base_path_is_guarded_however_a_path_climbs_into_it(monkeypatch, base, path): + backing = _Backing(monkeypatch) + prefixed = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url=f"{base}/mcp")], + "rest_proxy_urls": {"mcp-x": base}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", path, {}, **prefixed) == 401 + assert backing.paths == [] diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 2ed6ac102..5df7f530e 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -530,6 +530,20 @@ async def _post_rpc(gw: Gateway, payload, role: str, path: str = "/svc/mcp-svc/m return status, json.loads(b"".join(chunks)) +@pytest.mark.asyncio +async def test_a_server_whose_mcp_endpoint_is_elsewhere_has_the_role_stamped_there_and_not_at_mcp(monkeypatch): + up = _Upstream([b"{}"]) + up.install(monkeypatch) + gw = Gateway(host="127.0.0.1", port=0, server_name="t", rest_proxy_urls={"mcp-svc": UPSTREAM}, + internal_mcp_servers=[InternalMCPServer(name="svc", mcp_url=f"{UPSTREAM}/rpc")]) + body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}).encode() + for path in ("/svc/mcp-svc/rpc", "/svc/mcp-svc/mcp"): + await _call(gw, path, method="POST", body=body, headers={ + "AgentEnv-Role": "alice@example.com", "content-type": "application/json", "content-length": str(len(body))}) + assert [json.loads(r.content)["params"].get("_meta") for r in up.requests] == [ + {AGENT_ENV_ROLE_META_KEY: "alice@example.com"}, None] + + @pytest.mark.asyncio async def test_a_tool_disabled_for_the_caller_is_refused_as_mcp_refuses_it_and_never_reaches_the_child(monkeypatch): up = _Upstream([b"{}"]) diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 09b274198..4b0f2b7d0 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2646,14 +2646,15 @@ def _sites(handler): @pytest.mark.asyncio -async def test_a_deploy_waits_for_a_site_until_its_cards_ready_says_so_and_never_reads_its_health_path(): - asked = [] +async def test_a_deploy_waits_for_a_site_until_its_cards_ready_says_so_with_its_token_and_never_reads_its_health_path(): + asked, sent = [], [] def site(request: httpx.Request) -> httpx.Response: asked.append(request.url.path) if request.url.path == "/svc/mail/.well-known/agent-env.json": return httpx.Response(200, json=_READY_CARD) if request.url.path == "/svc/mail/agentenv/ext/website/ready": + sent.append(request.headers.get("authorization")) ready = asked.count(request.url.path) >= 2 return httpx.Response(200, json={"ready": ready, "problems": [] if ready else ["seeded"]}) return httpx.Response(500) @@ -2661,8 +2662,9 @@ def site(request: httpx.Request) -> httpx.Response: with _sites(site): await env_gateway_provider._wait_for_websites( "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail", - health_path="/healthz")], poll_s=0) + health_path="/healthz")], headers={"Authorization": "Bearer s3cret"}, poll_s=0) assert asked.count("/svc/mail/agentenv/ext/website/ready") == 2 and "/svc/mail/healthz" not in asked + assert sent == ["Bearer s3cret"] * 2 @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_run_code.py b/tst/unit/task_step/test_run_code.py index b21f10b7f..5527367e1 100644 --- a/tst/unit/task_step/test_run_code.py +++ b/tst/unit/task_step/test_run_code.py @@ -296,19 +296,19 @@ async def exec_with_output(self, *args): @pytest.mark.parametrize("deployed,expected", [ (DeployedGatewayEnv(env_id="gw", env_version=1, sandbox_id="sb", gateway_url="https://abc.modal.host", environment_card_url=_CARD_URL, environment_card={"name": "gw"}), - {"id": "gw", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "gw", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), (DeployedSandboxEnv(env_id="srv", env_version=1, sandbox_id="sb", env_provider_type="server", environment_card_url=_CARD_URL, environment_card={"name": "srv"}), - {"id": "srv", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "srv", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), # a plugin's env, whose card puts its MCP endpoint elsewhere (DeployedSandboxEnv(env_id="plug", env_version=1, sandbox_id="sb", env_provider_type="a-plugins", environment_card_url=_CARD_URL, environment_card={"name": "plug", "additionalInterfaces": [{"transport": "mcp", "url": "/v2/mcp"}]}), - {"id": "plug", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/v2/mcp"}), + {"id": "plug", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/v2/mcp", "headers": {}}), # a record from before the card, which kept only its MCP URL (DeployedSandboxEnv(env_id="old", env_version=1, sandbox_id="sb", mcp_url="https://abc.modal.host/mcp"), - {"id": "old", "url": None, "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "old", "url": None, "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), ]) def test_an_envs_address_and_mcp_endpoint_come_from_its_card_whatever_deployed_it(deployed, expected): sandbox = _ContainerEnvSandbox({18765: "https://abc.modal.host"}) @@ -328,11 +328,11 @@ def _written_input(sandbox): @pytest.mark.asyncio -async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_as_that_host_reaches_them(): +async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_as_that_host_reaches_them_and_its_token(): ctx = _make_context(with_agent=False) ctx.deployed_envs[0] = DeployedGatewayEnv( env_id="env-x", env_version=1, sandbox_id="sb-env", gateway_url="https://abc.modal.host", - environment_card_url=_CARD_URL, environment_card={"name": "env-x"}, + environment_card_url=_CARD_URL, environment_card={"name": "env-x"}, control_token="s3cret", ) with _sandbox([(0, "", ""), (0, "2", ""), (0, "{}", "")], agent_mode=False) as sandbox: sandbox.tunnel_urls = {18765: "https://abc.modal.host"} @@ -340,6 +340,7 @@ async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_a assert _written_input(sandbox)["env"] == { "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", + "headers": {"Authorization": "Bearer s3cret"}, } @@ -362,7 +363,7 @@ async def test_a_script_for_an_env_in_a_container_sandbox_runs_in_that_container ctx = await _step(env_id="env-x").execute(ctx) assert ctx.metadata["script_results"]["run-code-1"] == { - "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", + "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}, } assert not any(command[0] in ("sudo", "docker") for command in sandbox.commands)