diff --git a/rest/python/server/integration_test.py b/rest/python/server/integration_test.py index 182a26d..0b91dcb 100644 --- a/rest/python/server/integration_test.py +++ b/rest/python/server/integration_test.py @@ -15,6 +15,7 @@ """Integration tests for the UCP SDK Server.""" import asyncio +import base64 from collections.abc import AsyncGenerator import datetime import json @@ -28,6 +29,7 @@ from absl import flags from absl.testing import absltest import config +from cryptography.hazmat.primitives.asymmetric import ec import db import dependencies from fastapi.testclient import TestClient @@ -3057,6 +3059,201 @@ def test_create_ignores_a_client_sent_ap2_member(self) -> None: "a client sent ap2 member must not be stored either", ) + def test_signed_request_without_keyid_does_not_match_a_key_without_kid( + self, + ) -> None: + """A key published with no kid is never the key a signature names. + + A Signature-Input without a keyid parameter reads as None, and so did + the kid of a published key that omits the member, so the lookup matched + the two and a signature made with such a key verified. kid is a required + string member of every signing_keys[] entry, so the key cannot be found + by kid and the signature skips as key_not_found: logged and allowed by + default, 401 when enforcing. The insecure carve out is set only so that + no name resolution runs; the profile fetch itself is mocked. + """ + key = ec.generate_private_key(ec.SECP256R1()) + jwk = ucp_signing.jwk_from_public_key(key.public_key(), "k1") + del jwk["kid"] + body = json.dumps( + {"line_items": [{"item": {"id": "rose"}, "quantity": 1}]} + ).encode() + covered = [ + "@method", + "@authority", + "@path", + "content-digest", + "content-type", + "idempotency-key", + "ucp-agent", + ] + params = "(" + " ".join(f'"{c}"' for c in covered) + ");created=1893456000" + + def post() -> httpx.Response: + headers = self._get_headers() | { + "Content-Type": "application/json", + "Content-Digest": ucp_signing.content_digest(body), + } + lower = {k.lower(): v for k, v in headers.items()} + derived = { + "@method": "POST", + "@authority": "testserver", + "@path": "/checkout-sessions", + } + base = ucp_signing.build_signature_base( + covered, params, lambda name: derived.get(name, lower.get(name)) + ) + signature = ucp_signing._raw_sign(key, base) + headers["Signature-Input"] = f"sig1={params}" + headers["Signature"] = ( + "sig1=:" + base64.b64encode(signature).decode("ascii") + ":" + ) + return self.client.post( + "/checkout-sessions", headers=headers, content=body + ) + + ucp_signing.clear_key_cache() + config.FLAGS.allow_insecure_profile_urls = True + try: + with self.client, respx.mock: + respx.get("https://agent.example/profile").respond( + 200, json={"signing_keys": [jwk]} + ) + config.FLAGS.require_signatures = True + response = post() + self.assertEqual(response.status_code, 401, response.text) + self.assertEqual( + response.json()["messages"][0]["code"], "key_not_found" + ) + config.FLAGS.require_signatures = False + with self.assertLogs(dependencies.logger, level="WARNING") as logs: + response = post() + self.assertEqual(response.status_code, 201, response.text) + self.assertIn("key_not_found", "\n".join(logs.output)) + finally: + config.FLAGS.require_signatures = False + config.FLAGS.allow_insecure_profile_urls = False + + def test_signed_request_with_a_malformed_profile_url_is_not_a_server_fault( + self, + ) -> None: + """A profile URL that urlsplit or getaddrinfo rejects is a client error. + + verify_signature runs on every request carrying Signature-Input and + Signature headers, with or without enforcement. _assert_profile_url_allowed + let ValueError (an invalid IPv6 literal, a non numeric port) and + UnicodeError (an overlong host label) escape, so such a request answered + a bare 500 instead of invalid_profile_url from the spec when enforcing, or + the logged allow when not. + """ + signed = { + "Signature-Input": 'sig1=("@method" "@authority" "@path");keyid="k1"', + "Signature": "sig1=:AAAA:", + } + malformed = ( + "https://[::1", + "https://platform.example:xts/profile", + "https://" + "a" * 70 + ".example/profile", + ) + body = {"line_items": [{"item": {"id": "rose"}, "quantity": 1}]} + with self.client: + for url in malformed: + headers = self._get_headers() | signed + headers["UCP-Agent"] = f'profile="{url}"' + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 201, f"{url}: {response.text}") + + config.FLAGS.require_signatures = True + try: + for url in malformed: + headers = self._get_headers() | signed + headers["UCP-Agent"] = f'profile="{url}"' + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 400, f"{url}: {response.text}") + self.assertEqual( + response.json()["messages"][0]["code"], "invalid_profile_url" + ) + finally: + config.FLAGS.require_signatures = False + + def test_signed_request_with_a_tab_in_the_profile_url_is_not_a_server_fault( + self, + ) -> None: + """Httpx rejects a control character in a URL with InvalidURL. + + urlsplit strips a horizontal tab, so the URL guard passes, and + httpx.InvalidURL is not an HTTPError, so fetch_signing_keys let it escape. + The insecure carve out is set here only so that no name resolution runs; + httpx raises before any connection is attempted. + """ + signed = { + "Signature-Input": 'sig1=("@method" "@authority" "@path");keyid="k1"', + "Signature": "sig1=:AAAA:", + } + headers = self._get_headers() | signed + headers["UCP-Agent"] = 'profile="https://platform.example/\tprofile"' + body = {"line_items": [{"item": {"id": "rose"}, "quantity": 1}]} + config.FLAGS.allow_insecure_profile_urls = True + try: + with self.client: + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 201, response.text) + config.FLAGS.require_signatures = True + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 400, response.text) + self.assertEqual( + response.json()["messages"][0]["code"], "invalid_profile_url" + ) + finally: + config.FLAGS.require_signatures = False + config.FLAGS.allow_insecure_profile_urls = False + + def test_signed_request_with_an_invalid_a_label_host_is_not_a_server_fault( + self, + ) -> None: + """An ``xn--`` host label that is not valid Punycode is a client error. + + httpx decodes such a label while preparing the request and the idna + package raises IDNAError, a UnicodeError that is neither httpx.InvalidURL + nor httpx.HTTPError, so fetch_signing_keys let it escape. urlsplit and + the host check accept the label, and a wildcard DNS zone resolves it, so + the default mode reached this on the wire. The insecure carve out is set + here only so that no name resolution runs. + """ + signed = { + "Signature-Input": 'sig1=("@method" "@authority" "@path");keyid="k1"', + "Signature": "sig1=:AAAA:", + } + headers = self._get_headers() | signed + headers["UCP-Agent"] = 'profile="https://xn--0/profile"' + body = {"line_items": [{"item": {"id": "rose"}, "quantity": 1}]} + config.FLAGS.allow_insecure_profile_urls = True + try: + with self.client: + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 201, response.text) + config.FLAGS.require_signatures = True + response = self.client.post( + "/checkout-sessions", headers=headers, json=body + ) + self.assertEqual(response.status_code, 400, response.text) + self.assertEqual( + response.json()["messages"][0]["code"], "invalid_profile_url" + ) + finally: + config.FLAGS.require_signatures = False + config.FLAGS.allow_insecure_profile_urls = False + if __name__ == "__main__": absltest.main() diff --git a/rest/python/server/ucp_signing.py b/rest/python/server/ucp_signing.py index 68740fb..371f769 100644 --- a/rest/python/server/ucp_signing.py +++ b/rest/python/server/ucp_signing.py @@ -341,6 +341,9 @@ def public_key_from_jwk(jwk: dict): """ kty = jwk.get("kty") crv = jwk.get("crv") + # The profile is a document the signer publishes, so every member may be + # of any type; a coordinate or curve of the wrong type raises TypeError from + # the decoder or the curve lookup and is as malformed as a missing one. try: if kty == "EC" and crv in _EC_CURVES: x = int.from_bytes(_b64u_decode(jwk["x"]), "big") @@ -348,7 +351,7 @@ def public_key_from_jwk(jwk: dict): return ec.EllipticCurvePublicNumbers(x, y, ec.SECP256R1()).public_key() if kty == "OKP" and crv == "Ed25519": return ed25519.Ed25519PublicKey.from_public_bytes(_b64u_decode(jwk["x"])) - except (KeyError, ValueError) as exc: + except (KeyError, ValueError, TypeError) as exc: raise SignatureError( "signature_invalid", 401, f"Malformed JWK: {exc}" ) from exc @@ -477,7 +480,11 @@ def _sig_capable(jwk: dict) -> bool: if jwk.get("use") == "enc": return False key_ops = jwk.get("key_ops") - return key_ops is None or "verify" in key_ops + if key_ops is None: + return True + # key_ops is an array (RFC 7517 Section 4.3); any other shape is a + # malformed key and is skipped rather than raised on. + return isinstance(key_ops, list) and "verify" in key_ops def verify_request( @@ -529,7 +536,17 @@ def verify_request( # Resolve keyid only among signature-capable keys (ucp#566): a profile's # keys[] is a general JWK Set that may carry non-signature keys, so skip # use:"enc" and any key_ops that omits "verify" before matching on kid. - keys_by_kid = {k.get("kid"): k for k in keys if _sig_capable(k)} + # kid is a required string member of every published key (RFC 7517 + # Section 4.5; profile_schema.json). A key with a kid of any other shape, or + # with none, can never be the key a keyid parameter names and is skipped, + # which also keeps an unhashable value out of the lookup below. Admitting a + # key with no kid would let a Signature-Input that omits keyid, read as + # None, match it as None. + keys_by_kid = { + k["kid"]: k + for k in keys + if isinstance(k, dict) and isinstance(k.get("kid"), str) and _sig_capable(k) + } def resolve(name: str) -> str | None: if name == "@method": @@ -712,7 +729,17 @@ def _assert_profile_url_allowed(url: str, allow_insecure: bool) -> None: HTTPS, or resolves to a special-use address. """ - split = urlsplit(url) + # urlsplit raises ValueError on an invalid IPv6 literal and .port on a + # non numeric port; getaddrinfo raises UnicodeError on an overlong host + # label. Each is a malformed profile URL, invalid_profile_url in the spec, + # not a server fault. + try: + split = urlsplit(url) + port = split.port + except ValueError as exc: + raise SignatureError( + "invalid_profile_url", 400, f"Profile URL is malformed: {url!r}" + ) from exc if split.scheme != "https" and not ( allow_insecure and split.scheme == "http" ): @@ -731,9 +758,11 @@ def _assert_profile_url_allowed(url: str, allow_insecure: bool) -> None: if allow_insecure: return try: - infos = socket.getaddrinfo( - host, split.port or 443, proto=socket.IPPROTO_TCP - ) + infos = socket.getaddrinfo(host, port or 443, proto=socket.IPPROTO_TCP) + except UnicodeError as exc: + raise SignatureError( + "invalid_profile_url", 400, f"Profile URL host is malformed: {host!r}" + ) from exc except socket.gaierror as exc: raise SignatureError( "profile_unreachable", 424, f"Cannot resolve profile host: {host}" @@ -780,9 +809,24 @@ async def fetch_signing_keys( return cached[1] _assert_profile_url_allowed(profile_url, allow_insecure) + # httpx reads the proxy variables of the environment when the client is + # built, so a malformed one raises here. That is a fault of the operator, + # not of the client, and it stays a server fault outside the mapping below. + client = httpx.AsyncClient(follow_redirects=False) try: - async with httpx.AsyncClient(follow_redirects=False) as client: + async with client: response = await client.get(profile_url, timeout=5.0) + except (httpx.InvalidURL, UnicodeError) as exc: + # Neither is an HTTPError. httpx refuses a URL it cannot represent (a + # control character, for one) before any connection, and it decodes an + # ``xn--`` host label while preparing the request, where the idna package + # raises on an A-label that is not valid Punycode. The errors of the idna + # package derive from UnicodeError, which getaddrinfo raises above too. + # urlsplit and the host check accept both shapes, so this is the second + # place a malformed URL surfaces. + raise SignatureError( + "invalid_profile_url", 400, f"Profile URL is malformed: {exc}" + ) from exc except httpx.HTTPError as exc: raise SignatureError( "profile_unreachable", 424, f"Profile fetch failed: {exc}" @@ -793,9 +837,12 @@ async def fetch_signing_keys( 424, f"Profile fetch returned HTTP {response.status_code}", ) + # json.loads raises ValueError on a syntax or encoding fault and + # RecursionError on a document nested past the limit of the interpreter; + # both are a document the server cannot read, published by the signer. try: document = response.json() - except ValueError as exc: + except (ValueError, RecursionError) as exc: raise SignatureError( "profile_malformed", 422, "Profile is not valid JSON" ) from exc @@ -828,4 +875,8 @@ def _extract_keys(document: dict) -> list[dict]: if not isinstance(document, dict): return [] value = document.get("signing_keys") - return value if isinstance(value, list) and value else [] + if not isinstance(value, list): + return [] + # A JWK is an object; anything else in the array is not a key and must not + # reach the verifier, where it would raise instead of being skipped. + return [key for key in value if isinstance(key, dict)] diff --git a/rest/python/server/ucp_signing_test.py b/rest/python/server/ucp_signing_test.py index 2452e8a..66e2d15 100644 --- a/rest/python/server/ucp_signing_test.py +++ b/rest/python/server/ucp_signing_test.py @@ -431,6 +431,26 @@ def test_non_json_is_malformed(self) -> None: self._fetch(lambda req: httpx.Response(200, text="not json")) self.assertEqual(ctx.exception.code, "profile_malformed") + def test_deeply_nested_document_is_malformed(self) -> None: + """A document nested past the limit of the decoder is profile_malformed. + + json.loads recurses once per nesting level and raises RecursionError, + not ValueError, when the limit of the interpreter is reached, so + response.json() let it escape the mapping. The depth that trips it sits + a few levels under 10000 on CPython 3.12.13 and a few dozen under 1000 + on 3.11.15, and moves with the stack the call sits on; a profile is a + document the signer publishes, so the depth is chosen by the signer. + """ + depth = 100_000 + body = ('{"signing_keys": ' + "[" * depth + "]" * depth + "}").encode() + with self.assertRaises(signing.SignatureError) as ctx: + self._fetch( + lambda req: httpx.Response( + 200, content=body, headers={"content-type": "application/json"} + ) + ) + self.assertEqual(ctx.exception.code, "profile_malformed") + def test_keyless_is_malformed(self) -> None: """A profile with no keys yields profile_malformed.""" with self.assertRaises(signing.SignatureError) as ctx: @@ -683,6 +703,11 @@ def test_future_top_level_keys_is_not_read(self) -> None: doc = {"ucp": {"version": "2026-04-08"}, "keys": [{"kid": "future"}]} self.assertEqual(signing._extract_keys(doc), []) + def test_signing_keys_that_is_not_an_array_yields_no_keys(self) -> None: + """A signing_keys of any other type publishes no keys and raises nothing.""" + for value in (2030, True, "xts-2030", {"kid": "k"}): + self.assertEqual(signing._extract_keys({"signing_keys": value}), []) + class SigCapableTest(absltest.TestCase): """Signature-capable key filtering (ucp#566). @@ -759,6 +784,193 @@ def test_second_call_is_cached(self) -> None: signing.clear_key_cache() +class ProfileUrlFaultTest(absltest.TestCase): + """Profile URL faults that the standard library or httpx raise on.""" + + def test_invalid_ipv6_literal_is_invalid_profile_url(self) -> None: + """Urlsplit raises ValueError on an unterminated IPv6 literal.""" + with self.assertRaises(signing.SignatureError) as ctx: + signing._assert_profile_url_allowed("https://[::1", False) + self.assertEqual(ctx.exception.code, "invalid_profile_url") + + def test_non_numeric_port_is_invalid_profile_url(self) -> None: + """SplitResult.port raises ValueError on a port that is not a number.""" + with self.assertRaises(signing.SignatureError) as ctx: + signing._assert_profile_url_allowed("https://example.com:xts/p", False) + self.assertEqual(ctx.exception.code, "invalid_profile_url") + + def test_overlong_host_label_is_invalid_profile_url(self) -> None: + """Getaddrinfo raises UnicodeError before any lookup on a label over 63.""" + with self.assertRaises(signing.SignatureError) as ctx: + signing._assert_profile_url_allowed( + "https://" + "a" * 70 + ".example/p", False + ) + self.assertEqual(ctx.exception.code, "invalid_profile_url") + + def test_control_character_in_url_is_invalid_profile_url(self) -> None: + """httpx.InvalidURL is not an HTTPError and must still map to the code.""" + import asyncio + + signing.clear_key_cache() + with self.assertRaises(signing.SignatureError) as ctx: + asyncio.run( + signing.fetch_signing_keys( + "https://platform.example/\tprofile", allow_insecure=True + ) + ) + self.assertEqual(ctx.exception.code, "invalid_profile_url") + + def test_invalid_a_label_in_host_is_invalid_profile_url(self) -> None: + """idna.IDNAError is a UnicodeError, not an HTTPError, and must map too. + + httpx decodes an ``xn--`` host label while preparing the request, before + any connection, and the idna package raises on an A-label that is not + valid Punycode. urlsplit and the host check both accept the label, so + this is the third place a malformed profile URL surfaces. + """ + import asyncio + + signing.clear_key_cache() + with self.assertRaises(signing.SignatureError) as ctx: + asyncio.run( + signing.fetch_signing_keys("https://xn--0/profile", allow_insecure=True) + ) + self.assertEqual(ctx.exception.code, "invalid_profile_url") + + +class ProxyEnvironmentTest(absltest.TestCase): + """A malformed proxy variable on the server is a fault of the operator.""" + + def test_malformed_proxy_variable_is_not_the_profile_url_fault(self) -> None: + """Httpx parses ALL_PROXY when the client is built, before any request. + + The InvalidURL it raises names the environment of the server, not the + profile URL the client sent, so it must not be reported as + invalid_profile_url and must not be logged and allowed in the default + mode. It stays a server fault, as on main. + + Every other proxy variable is removed for the duration of the call, in + any spelling: urllib reads the variables whose name ends in ``_proxy`` + case insensitively and lets a lowercase spelling override an uppercase + one, and httpx ignores every proxy variable once ``no_proxy`` carries a + ``*``. Either would let the shell of whoever runs the suite turn this + into a connection attempt and a different error. + """ + import asyncio + import os + from unittest import mock + + signing.clear_key_cache() + without_proxies = { + name: value + for name, value in os.environ.items() + if not name.lower().endswith("_proxy") + } + with ( + mock.patch.dict( + os.environ, without_proxies | {"ALL_PROXY": "http://[::1"}, clear=True + ), + self.assertRaises(httpx.InvalidURL), + ): + asyncio.run( + signing.fetch_signing_keys( + "http://127.0.0.1:9/profile", allow_insecure=True + ) + ) + + +class MalformedJwkTest(absltest.TestCase): + """A profile the signer controls may publish keys of any shape. + + Every shape below escaped verify_request as AttributeError or TypeError + before the guards, which a server answers as a bare 500 even when it is not + enforcing signatures. Each must be a SignatureError instead. + """ + + HEADERS = { + "signature-input": 'sig1=("@method" "@authority" "@path");keyid="k1"', + "signature": "sig1=:AAAA:", + } + + def _verify(self, keys: list) -> None: + signing.verify_request( + "POST", "merchant.test", "/checkout-sessions", "", self.HEADERS, b"", keys + ) + + def test_non_object_key_is_skipped(self) -> None: + """A signing_keys[] entry that is not an object is not a key.""" + with self.assertRaises(signing.SignatureError) as ctx: + self._verify(["abc", None, 5]) + self.assertEqual(ctx.exception.code, "key_not_found") + + def test_key_ops_of_the_wrong_type_is_not_signature_capable(self) -> None: + """key_ops is an array (RFC 7517 Section 4.3); anything else is skipped. + + A number raised TypeError before the guard. A string containing verify + and an object with a verify member passed the membership test and were + treated as signature capable; each is skipped now, the behaviour change + this fix discloses. + """ + jwk = {"kid": "k1", "kty": "EC", "crv": "P-256", "x": "AA", "y": "AA"} + for key_ops in (5, "verify", "xverifyx", {"verify": 1}): + with self.assertRaises(signing.SignatureError) as ctx: + self._verify([jwk | {"key_ops": key_ops}]) + self.assertEqual(ctx.exception.code, "key_not_found", repr(key_ops)) + + def test_malformed_coordinates_and_curve_are_signature_invalid(self) -> None: + """Coordinates or a curve of the wrong type are a malformed JWK.""" + for jwk in ( + {"kid": "k1", "kty": "OKP", "crv": "Ed25519", "x": 5}, + {"kid": "k1", "kty": "EC", "crv": ["P-256"], "x": "AA", "y": "AA"}, + {"kid": "k1", "kty": "EC", "crv": "P-256", "x": ["a"], "y": "AA"}, + ): + with self.assertRaises(signing.SignatureError) as ctx: + self._verify([jwk]) + self.assertEqual(ctx.exception.code, "signature_invalid") + + def test_kid_of_the_wrong_type_is_not_matched(self) -> None: + """Kid is a string (RFC 7517 Section 4.5); any other shape never matches.""" + jwk = {"kty": "EC", "crv": "P-256", "x": "AA", "y": "AA"} + with self.assertRaises(signing.SignatureError) as ctx: + self._verify([jwk | {"kid": ["k1"]}, jwk | {"kid": {"k": 1}}]) + self.assertEqual(ctx.exception.code, "key_not_found") + + def test_key_without_kid_is_not_matched_by_a_signature_without_keyid( + self, + ) -> None: + """A published key with no kid is never the key a signature names. + + A Signature-Input without a keyid parameter reads as None, and so did + the kid of a published key that omits the member, so the lookup matched + the two and a signature made with such a key verified. kid is a required + string member of every signing_keys[] entry (profile_schema.json), so a + key without one cannot be found by kid and the candidate skips as + key_not_found, as it does for a keyid that names no key. + """ + key = ec.generate_private_key(ec.SECP256R1()) + jwk = signing.jwk_from_public_key(key.public_key(), "k1") + del jwk["kid"] + components = ["@method", "@authority", "@path"] + params = '("@method" "@authority" "@path");created=1893456000' + resolve = {"@method": "GET", "@authority": "m.example", "@path": "/p"}.get + base = signing.build_signature_base(components, params, resolve) + signature = signing._raw_sign(key, base) + # The signature itself is sound: only the lookup must turn it away. + signing.verify_raw_signature(jwk, base, signature) + headers = { + "signature-input": f"sig1={params}", + "signature": "sig1=:" + base64.b64encode(signature).decode("ascii") + ":", + } + with self.assertRaises(signing.SignatureError) as ctx: + signing.verify_request("GET", "m.example", "/p", "", headers, b"", [jwk]) + self.assertEqual(ctx.exception.code, "key_not_found") + + def test_extract_keys_keeps_only_objects(self) -> None: + """_extract_keys returns the object entries of signing_keys[] only.""" + document = {"signing_keys": [1, {"kid": "k1"}, None, "x"]} + self.assertEqual(signing._extract_keys(document), [{"kid": "k1"}]) + + class SsrfResolveTest(absltest.TestCase): """Profile-URL host vetting: resolve failures, hostless URLs, public pass."""