diff --git a/docs/maintainers/value-contract-prototypes.md b/docs/maintainers/value-contract-prototypes.md new file mode 100644 index 00000000..a147d9ea --- /dev/null +++ b/docs/maintainers/value-contract-prototypes.md @@ -0,0 +1,124 @@ +# Value extension prototypes + +These rules are candidates for #337 and #339. They do not change the 0.11 codec +policy. The prototypes live under `test/prototypes`. Shipped wrappers cannot +import them. + +## Exact integer option + +The generation option is `integerMode: 'bigint'` for one callable. It maps every +Python `int` parameter and result to TypeScript `bigint`. Safe integers use the +same type. Python `bool` remains `boolean`. Calls without the option keep the +safe `number` rule. + +The current value-contract revision represents Python `int` only as a safe JSON +`number`. The compiler and return validator follow that rule. This prototype +does not generate `bigint` wrappers; that needs a deliberate contract extension. + +The proposed generated wrapper must require `exactIntegerDecimalV2` in the +bridge's `meta.valueCapabilities`. It must send +`params.valuePolicy.integer = 'bigint-v2'` on each call. A new bridge must +reject unknown policies. An old bridge lacks the capability, so the wrapper must +reject the call before sending a tag. The prototype tests this policy in an +isolated adapter. The shipped wrapper does not implement it. + +The version 2 integer envelope is +`{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. +`codecVersion` uses the parsed JSON numeric value. The lexemes `2`, `2.0`, and +`2e0` all mean version 2. Booleans and strings do not. The decimal must match +`^(?:0|[1-9][0-9]*|-[1-9][0-9]*)$`. This excludes `-0`, leading zeros, plus +signs, spaces, exponents, and Unicode digits. The value permits at most 4096 +digits, excluding the sign. It uses at most 4097 ASCII bytes. Whole request and +response byte limits still apply. + +The TypeScript encoder tags every `bigint`, including safe values. The Python +request decoder requires the tag at each declared integer node. It rejects raw +JSON numbers there. It uses the trusted resolved value contract to distinguish +integer nodes from float nodes. A generated wrapper must not let a client supply +or change that contract. + +`JSON.stringify(1.0)` emits `1`. The Python decoder converts that token to +`float` only at a declared float node. `JSON.stringify(-0)` emits `0`, so the +TypeScript encoder uses +`{"__tywrap__":"float","codecVersion":2,"encoding":"negative-zero"}` for +negative zero. The Python decoder accepts that tag only at a float node. It +rejects malformed float tags and nonfinite numbers. + +The Python response encoder tags every `int`, including safe values. The +TypeScript response decoder returns `bigint` at declared integer nodes. The +generated return validator must require `bigint`. It must reject an ordinary +number and a raw tag. Arrow int64 keeps its current Arrow encoding and decoded +type. + +Both prototypes walk nested arrays and string-keyed records within depth and +node limits. They reject unsupported values and malformed envelopes. Encoders +reject cycles with paths. Ordinary records cannot use `__tywrap__` as a key +because the decoder reserves it for envelopes. + +The prototype counts UTF-8 bytes of compact JSON. Python emits Unicode +characters directly, as `JSON.stringify` does. Both paths reject unpaired +surrogates. Production must check raw wire bytes before parsing; object-level +checks do not prove equal byte counts for alternate numeric JSON spellings. + +The bounded tests must round trip positive and negative values beyond 64 bits. +They must cover floats, booleans, nested records, malformed tags, digit limits, +byte limits, and old bridges. Production work needs separate review of +capability negotiation, generated types, and migration cost. + +### Exact integer migration cost + +Opting in changes every declared Python `int` at that callable to TypeScript +`bigint`, including values within the safe number range. Callers must update +number arithmetic, comparisons, and stored JSON. `JSON.stringify` rejects a +plain `bigint`, so callers that persist results need an explicit encoding. + +Each tagged integer adds 73 JSON bytes beyond its decimal digits. For example, +`7` uses one byte as a JSON number and 74 bytes in the version 2 envelope. +Nested collections multiply that cost by their integer count. Teams must measure +representative call payloads against the byte limit before enabling the option. + +The client must check the bridge capability before it sends tagged values. This +requires a coordinated client and bridge rollout. Old bridges reject the opted +call. Arrow int64 columns keep their current type and wire format. + +Before production adoption, inventory generated call sites that use `int`, +measure payload growth, and typecheck downstream callers after the opt-in. The +prototype has not measured those application-specific costs. + +## Dataclass output option + +The generation option is `dataclassReturns: 'validated-v2'` for a callable with +an annotated dataclass result. Dataclass inputs remain unsupported. Ordinary +class instances do not qualify. + +The generator compiles `dataclasses.fields` into a structural TypeScript record. +It includes fields with defaults and `init=False`. An instance returns those +fields. `Optional[T]` permits `null`; a default does not make a returned +property optional. + +The proposed generated wrapper must require `dataclassFieldsV2` in +`meta.valueCapabilities`. It must send +`params.valuePolicy.dataclass = 'fields-v2'` on each call. The Python prototype +encoder accepts only the declared dataclass type. It emits a version 2 envelope +with `module.qualname` identity and an ordered `fields` record. It rejects +unsupported fields and cycles with paths. It applies depth, node, and byte +limits. It does not call `dataclasses.asdict` or create object handles. + +The TypeScript prototype decoder checks marker, version, encoding, type +identity, and exact fields before it removes the envelope. It returns a plain +record. A module-private `WeakMap` stores verified type identity. The prototype +origin check reads that entry. A plain record cannot claim dataclass origin. A +nested dataclass gets its own entry. The current generated return validator +checks required field types but does not check origin or reject extra keys. The +adapter must perform those checks before it calls the validator. + +The Python prototype checks the declared root dataclass type. It serializes +nested dataclasses by their observed runtime type and does not check a compiled +field contract. The TypeScript decoder rejects unexpected nested identities or +field values. Server-side field validation remains open. + +The bounded prototype still needs a generated `Promise` wrapper from the +actual compiler model. It must return `{x: 1, y: 2}` and reject missing, extra, +and wrong typed fields. It must test defaults, optional fields, nesting, cycles, +unsupported fields, depth, and byte limits. Production work needs independent +review of identity, generated types, and migration cost. diff --git a/test/prototypes/value_extensions.py b/test/prototypes/value_extensions.py new file mode 100644 index 00000000..135c519b --- /dev/null +++ b/test/prototypes/value_extensions.py @@ -0,0 +1,391 @@ +"""Isolated wire prototypes for #337 and #339. Production never imports this file.""" + +from __future__ import annotations + +import dataclasses +import json +import math +import re +import sys +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + + +MAX_DECIMAL_DIGITS = 4096 +MAX_DEPTH = 64 +MAX_NODES = 100_000 +MAX_PAYLOAD_BYTES = 10 * 1024 * 1024 +SAFE_INTEGER_MAX = 2**53 - 1 +MAX_INTEGER_MAGNITUDE = 10**MAX_DECIMAL_DIGITS +DECIMAL = re.compile(r'(?:0|[1-9][0-9]*|-[1-9][0-9]*)\Z', re.ASCII) +INTEGER_FIELDS = frozenset({'__tywrap__', 'codecVersion', 'encoding', 'value'}) + + +class PrototypeError(ValueError): + """A value cannot satisfy the proposed extension contract.""" + + +@dataclass +class Budget: + nodes: int = 0 + + def visit(self, path: str, depth: int) -> None: + self.nodes += 1 + if depth > MAX_DEPTH: + raise PrototypeError(f'maximum depth {MAX_DEPTH} exceeded at {path}') + if self.nodes > MAX_NODES: + raise PrototypeError(f'maximum nodes {MAX_NODES} exceeded at {path}') + + +def _check_payload(value: object, max_payload_bytes: int) -> None: + try: + encoded = json.dumps( + value, allow_nan=False, ensure_ascii=False, separators=(',', ':') + ) + byte_count = len(encoded.encode('utf-8')) + except (UnicodeEncodeError, ValueError) as exc: + raise PrototypeError('payload is not finite UTF-8 JSON') from exc + if byte_count > max_payload_bytes: + raise PrototypeError(f'payload exceeds {max_payload_bytes} bytes') + + +def _version_two(value: object) -> bool: + return type(value) in (int, float) and value == 2 + + +def _child(path: str, key: str | int) -> str: + return f'{path}[{key}]' if isinstance(key, int) else f'{path}.{key}' + + +def require_capability(meta: Mapping[str, object], feature: str, policy: str) -> None: + capabilities = meta.get('valueCapabilities') + if not isinstance(capabilities, list) or feature not in capabilities: + raise PrototypeError(f'bridge lacks {feature} capability') + expected_policy = { + 'exactIntegerDecimalV2': 'bigint-v2', + 'dataclassFieldsV2': 'fields-v2', + }.get(feature) + if policy != expected_policy: + raise PrototypeError(f'unsupported per-call value policy {policy!r}') + + +def _canonical_decimal(value: object, path: str) -> int: + if not isinstance(value, str) or not DECIMAL.fullmatch(value): + raise PrototypeError(f'noncanonical integer decimal at {path}') + digits = len(value) - (1 if value.startswith('-') else 0) + if digits > MAX_DECIMAL_DIGITS: + raise PrototypeError(f'integer exceeds {MAX_DECIMAL_DIGITS} digits at {path}') + return int(value) + + +def _integer_envelope(value: int, path: str) -> dict[str, object]: + if abs(value) >= MAX_INTEGER_MAGNITUDE: + raise PrototypeError(f'integer exceeds {MAX_DECIMAL_DIGITS} digits at {path}') + try: + decimal = str(value) + except ValueError as exc: + raise PrototypeError(f'integer conversion failed at {path}') from exc + _canonical_decimal(decimal, path) + return { + '__tywrap__': 'integer', + 'codecVersion': 2, + 'encoding': 'decimal', + 'value': decimal, + } + + +def encode_exact_integers( + value: object, + *, + max_payload_bytes: int = MAX_PAYLOAD_BYTES, +) -> object: + """Encode all Python integers as exact tagged values for an opted call.""" + budget = Budget() + active: set[int] = set() + + def visit(current: object, path: str, depth: int) -> object: + budget.visit(path, depth) + if current is None or type(current) in (bool, str): + return current + if type(current) is int: + return _integer_envelope(current, path) + if type(current) is float and math.isfinite(current): + return current + if isinstance(current, (list, tuple, dict)): + marker = id(current) + if marker in active: + raise PrototypeError(f'cycle at {path}') + active.add(marker) + try: + if isinstance(current, dict): + if any(not isinstance(key, str) for key in current): + raise PrototypeError(f'non-string record key at {path}') + if '__tywrap__' in current: + raise PrototypeError(f'reserved record key at {path}.__tywrap__') + return { + key: visit(item, _child(path, key), depth + 1) + for key, item in current.items() + } + return [ + visit(item, _child(path, index), depth + 1) + for index, item in enumerate(current) + ] + finally: + active.remove(marker) + raise PrototypeError(f'unsupported value at {path}: {type(current).__name__}') + + result = visit(value, 'result', 0) + _check_payload(result, max_payload_bytes) + return result + + +def decode_exact_integers( + value: object, + contract: Mapping[str, Any], + *, + max_payload_bytes: int = MAX_PAYLOAD_BYTES, +) -> object: + """Decode an opted request with its trusted resolved value contract.""" + _check_payload(value, max_payload_bytes) + budget = Budget() + active: set[int] = set() + + def visit( + current: object, spec: Mapping[str, Any], path: str, depth: int + ) -> object: + kind = spec.get('kind') + if kind == 'nullable': + if current is None: + return visit(None, {'kind': 'null'}, path, depth) + return visit(current, spec['value'], path, depth) + budget.visit(path, depth) + marker = id(current) if isinstance(current, (list, dict)) else None + if marker is not None: + if marker in active: + raise PrototypeError(f'cycle at {path}') + active.add(marker) + try: + if kind == 'null': + if current is not None: + raise PrototypeError(f'expected null at {path}') + return None + if kind == 'boolean': + if type(current) is not bool: + raise PrototypeError(f'expected boolean at {path}') + return current + if kind == 'string': + if type(current) is not str: + raise PrototypeError(f'expected string at {path}') + return current + if kind == 'integer': + if not isinstance(current, dict): + raise PrototypeError(f'untagged integer at {path}') + if ( + set(current) != INTEGER_FIELDS + or current.get('__tywrap__') != 'integer' + or not _version_two(current.get('codecVersion')) + or current.get('encoding') != 'decimal' + ): + raise PrototypeError(f'invalid integer envelope at {path}') + return _canonical_decimal(current['value'], _child(path, 'value')) + if kind == 'safe-integer': + if ( + type(current) is not int + or current < -SAFE_INTEGER_MAX + or current > SAFE_INTEGER_MAX + ): + raise PrototypeError(f'expected safe integer at {path}') + return current + if kind == 'float': + if isinstance(current, dict): + if current != { + '__tywrap__': 'float', + 'codecVersion': 2, + 'encoding': 'negative-zero', + } or not _version_two(current.get('codecVersion')): + raise PrototypeError(f'invalid float envelope at {path}') + return -0.0 + if type(current) not in (int, float): + raise PrototypeError(f'expected finite float at {path}') + try: + result = float(current) + except OverflowError as exc: + raise PrototypeError(f'expected finite float at {path}') from exc + if not math.isfinite(result): + raise PrototypeError(f'expected finite float at {path}') + return result + if kind == 'array': + if not isinstance(current, list): + raise PrototypeError(f'expected array at {path}') + return [ + visit(item, spec['item'], _child(path, index), depth + 1) + for index, item in enumerate(current) + ] + if kind == 'record': + if not isinstance(current, dict): + raise PrototypeError(f'expected record at {path}') + fields = spec['fields'] + if '__tywrap__' in current: + raise PrototypeError(f'reserved record key at {path}.__tywrap__') + if set(current) != set(fields): + raise PrototypeError(f'fields differ at {path}') + return { + key: visit(current[key], field, _child(path, key), depth + 1) + for key, field in fields.items() + } + raise PrototypeError(f'unsupported contract kind {kind!r} at {path}') + finally: + if marker is not None: + active.remove(marker) + + return visit(value, contract, 'args', 0) + + +def encode_dataclass( + value: object, + expected_type: type, + *, + max_payload_bytes: int = MAX_PAYLOAD_BYTES, +) -> dict[str, object]: + """Encode only an annotated dataclass result as a version 2 envelope.""" + if type(value) is not expected_type or not dataclasses.is_dataclass(value): + raise PrototypeError('result is not the declared dataclass type') + budget = Budget() + active: set[int] = set() + + def visit(current: object, path: str, depth: int) -> object: + budget.visit(path, depth) + if current is None or type(current) in (bool, str): + return current + if type(current) is int: + if current < -SAFE_INTEGER_MAX or current > SAFE_INTEGER_MAX: + raise PrototypeError(f'unsafe integer at {path}') + return current + if type(current) is float and math.isfinite(current): + return current + if dataclasses.is_dataclass(current) and not isinstance(current, type): + marker = id(current) + if marker in active: + raise PrototypeError(f'cycle at {path}') + active.add(marker) + try: + cls = type(current) + fields = { + field.name: visit( + getattr(current, field.name), + _child(path, field.name), + depth + 1, + ) + for field in dataclasses.fields(current) + } + return { + '__tywrap__': 'dataclass', + 'codecVersion': 2, + 'encoding': 'fields', + 'type': f'{cls.__module__}.{cls.__qualname__}', + 'fields': fields, + } + finally: + active.remove(marker) + if isinstance(current, (list, tuple, dict)): + marker = id(current) + if marker in active: + raise PrototypeError(f'cycle at {path}') + active.add(marker) + try: + if isinstance(current, dict): + if any(not isinstance(key, str) for key in current): + raise PrototypeError(f'non-string record key at {path}') + if '__tywrap__' in current: + raise PrototypeError(f'reserved record key at {path}.__tywrap__') + return { + key: visit(item, _child(path, key), depth + 1) + for key, item in current.items() + } + return [ + visit(item, _child(path, index), depth + 1) + for index, item in enumerate(current) + ] + finally: + active.remove(marker) + raise PrototypeError(f'unsupported value at {path}: {type(current).__name__}') + + result = visit(value, 'result', 0) + assert isinstance(result, dict) + _check_payload(result, max_payload_bytes) + return result + + +@dataclass +class Point: + x: int + y: int + + +INTEGER_TEST_CONTRACT: dict[str, Any] = { + 'kind': 'record', + 'fields': { + 'positive': {'kind': 'array', 'item': {'kind': 'integer'}}, + 'negative': { + 'kind': 'record', + 'fields': {'value': {'kind': 'integer'}}, + }, + 'safe': {'kind': 'integer'}, + 'flag': {'kind': 'boolean'}, + 'whole': {'kind': 'float'}, + 'fraction': {'kind': 'float'}, + 'negativeZero': {'kind': 'float'}, + 'mixed': { + 'kind': 'array', + 'item': { + 'kind': 'record', + 'fields': { + 'quantity': {'kind': 'integer'}, + 'ratio': {'kind': 'float'}, + 'flag': {'kind': 'boolean'}, + }, + }, + }, + }, +} + + +def _main() -> None: + raw = sys.stdin.buffer.read(MAX_PAYLOAD_BYTES + 1) + if len(raw) > MAX_PAYLOAD_BYTES: + raise PrototypeError('input payload exceeds byte limit') + payload: Any = json.loads(raw) + action = sys.argv[1] + if action == 'roundtrip-integer': + require_capability( + payload['meta'], 'exactIntegerDecimalV2', payload['policy'] + ) + result = encode_exact_integers( + decode_exact_integers(payload['value'], INTEGER_TEST_CONTRACT) + ) + elif action == 'encode-integer': + result = encode_exact_integers(payload) + elif action == 'encode-integer-limited': + result = encode_exact_integers( + payload['value'], max_payload_bytes=payload['limit'] + ) + elif action == 'roundtrip-single-integer': + require_capability( + payload['meta'], 'exactIntegerDecimalV2', payload['policy'] + ) + result = encode_exact_integers( + decode_exact_integers(payload['value'], {'kind': 'integer'}) + ) + elif action == 'encode-point': + result = encode_dataclass(Point(**payload), Point) + else: + raise PrototypeError(f'unknown action {action!r}') + wire = json.dumps( + result, allow_nan=False, ensure_ascii=False, separators=(',', ':') + ).encode('utf-8') + sys.stdout.buffer.write(wire + b'\n') + + +if __name__ == '__main__': + _main() diff --git a/test/prototypes/value_extensions.ts b/test/prototypes/value_extensions.ts new file mode 100644 index 00000000..67d8d4f4 --- /dev/null +++ b/test/prototypes/value_extensions.ts @@ -0,0 +1,276 @@ +/** Isolated value extension prototype. Shipped codecs never import this file. */ + +export type PrototypeContract = + | { readonly kind: 'integer' } + | { readonly kind: 'safe-integer' } + | { readonly kind: 'float' } + | { readonly kind: 'boolean' } + | { readonly kind: 'string' } + | { readonly kind: 'null' } + | { readonly kind: 'nullable'; readonly value: PrototypeContract } + | { readonly kind: 'array'; readonly item: PrototypeContract } + | { readonly kind: 'record'; readonly fields: Readonly> } + | { + readonly kind: 'dataclass'; + readonly typeId: string; + readonly fields: Readonly>; + }; + +const MAX_DECIMAL_DIGITS = 4096; +const MAX_DEPTH = 64; +const MAX_NODES = 100_000; +const MAX_PAYLOAD_BYTES = 10 * 1024 * 1024; +const DECIMAL = /^(?:0|[1-9][0-9]*|-[1-9][0-9]*)$/; +const provenance = new WeakMap(); + +export class PrototypeError extends Error {} + +function isRecord(value: unknown): value is Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function exactKeys( + value: Record, + expected: readonly string[], + path: string +): void { + const actual = Object.keys(value).sort(); + const required = [...expected].sort(); + if (actual.length !== required.length || actual.some((key, index) => key !== required[index])) { + throw new PrototypeError(`fields differ at ${path}`); + } +} + +function checkPayload(value: unknown, limit: number): void { + const assertUnicodeScalars = (text: string): void => { + for (let index = 0; index < text.length; index += 1) { + const code = text.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = text.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) { + throw new PrototypeError('payload contains an unpaired Unicode surrogate'); + } + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + throw new PrototypeError('payload contains an unpaired Unicode surrogate'); + } + } + }; + const encoded = JSON.stringify(value, (key, item: unknown) => { + assertUnicodeScalars(key); + if (typeof item === 'string') assertUnicodeScalars(item); + return item; + }); + if (encoded === undefined || new TextEncoder().encode(encoded).length > limit) { + throw new PrototypeError(`payload exceeds ${limit} bytes`); + } +} + +function canonicalDecimal(value: unknown, path: string): bigint { + if (typeof value !== 'string' || !DECIMAL.test(value)) { + throw new PrototypeError(`noncanonical integer decimal at ${path}`); + } + const digits = value.startsWith('-') ? value.length - 1 : value.length; + if (digits > MAX_DECIMAL_DIGITS) { + throw new PrototypeError(`integer exceeds ${MAX_DECIMAL_DIGITS} digits at ${path}`); + } + return BigInt(value); +} + +export function requireCapability(meta: unknown, feature: string, policy: string): void { + const capabilities = isRecord(meta) ? meta.valueCapabilities : undefined; + if (!Array.isArray(capabilities) || !capabilities.includes(feature)) { + throw new PrototypeError(`bridge lacks ${feature} capability`); + } + const expected = + feature === 'exactIntegerDecimalV2' + ? 'bigint-v2' + : feature === 'dataclassFieldsV2' + ? 'fields-v2' + : undefined; + if (policy !== expected) { + throw new PrototypeError(`unsupported per-call value policy ${policy}`); + } +} + +interface WalkState { + nodes: number; + active: WeakSet; +} + +function visit(state: WalkState, value: unknown, path: string, depth: number): void { + state.nodes += 1; + if (depth > MAX_DEPTH) { + throw new PrototypeError(`maximum depth ${MAX_DEPTH} exceeded at ${path}`); + } + if (state.nodes > MAX_NODES) { + throw new PrototypeError(`maximum nodes ${MAX_NODES} exceeded at ${path}`); + } + if (typeof value === 'object' && value !== null) { + if (state.active.has(value)) { + throw new PrototypeError(`cycle at ${path}`); + } + state.active.add(value); + } +} + +function leave(state: WalkState, value: unknown): void { + if (typeof value === 'object' && value !== null) { + state.active.delete(value); + } +} + +function walk( + value: unknown, + contract: PrototypeContract, + mode: 'encode' | 'decode', + path: string, + depth: number, + state: WalkState +): unknown { + if (contract.kind === 'nullable') { + return value === null + ? walk(null, { kind: 'null' }, mode, path, depth, state) + : walk(value, contract.value, mode, path, depth, state); + } + visit(state, value, path, depth); + try { + if (contract.kind === 'null') { + if (value !== null) throw new PrototypeError(`expected null at ${path}`); + return null; + } + if (contract.kind === 'boolean' || contract.kind === 'string') { + if (typeof value !== contract.kind) { + throw new PrototypeError(`expected ${contract.kind} at ${path}`); + } + return value; + } + if (contract.kind === 'float') { + if (mode === 'decode' && isRecord(value)) { + exactKeys(value, ['__tywrap__', 'codecVersion', 'encoding'], path); + if ( + value.__tywrap__ !== 'float' || + value.codecVersion !== 2 || + value.encoding !== 'negative-zero' + ) { + throw new PrototypeError(`invalid float envelope at ${path}`); + } + return -0; + } + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new PrototypeError(`expected finite float at ${path}`); + } + if (mode === 'encode' && Object.is(value, -0)) { + return { __tywrap__: 'float', codecVersion: 2, encoding: 'negative-zero' }; + } + return value; + } + if (contract.kind === 'safe-integer') { + if (typeof value !== 'number' || !Number.isSafeInteger(value)) { + throw new PrototypeError(`expected safe integer at ${path}`); + } + return value; + } + if (contract.kind === 'integer') { + if (mode === 'encode') { + if (typeof value !== 'bigint') { + throw new PrototypeError(`expected bigint at ${path}`); + } + const decimal = value.toString(); + canonicalDecimal(decimal, path); + return { __tywrap__: 'integer', codecVersion: 2, encoding: 'decimal', value: decimal }; + } + if (!isRecord(value)) { + throw new PrototypeError(`expected integer envelope at ${path}`); + } + exactKeys(value, ['__tywrap__', 'codecVersion', 'encoding', 'value'], path); + if ( + value.__tywrap__ !== 'integer' || + value.codecVersion !== 2 || + value.encoding !== 'decimal' + ) { + throw new PrototypeError(`invalid integer envelope at ${path}`); + } + return canonicalDecimal(value.value, `${path}.value`); + } + if (contract.kind === 'array') { + if (!Array.isArray(value)) throw new PrototypeError(`expected array at ${path}`); + return value.map((item, index) => + walk(item, contract.item, mode, `${path}[${index}]`, depth + 1, state) + ); + } + if (contract.kind === 'record') { + if (!isRecord(value)) throw new PrototypeError(`expected record at ${path}`); + if (Object.hasOwn(value, '__tywrap__')) { + throw new PrototypeError(`reserved record key at ${path}.__tywrap__`); + } + exactKeys(value, Object.keys(contract.fields), path); + return Object.fromEntries( + Object.entries(contract.fields).map(([key, field]) => [ + key, + walk(value[key], field, mode, `${path}.${key}`, depth + 1, state), + ]) + ); + } + if (mode === 'encode') { + throw new PrototypeError(`dataclass inputs are unsupported at ${path}`); + } + if (!isRecord(value)) throw new PrototypeError(`expected dataclass envelope at ${path}`); + exactKeys(value, ['__tywrap__', 'codecVersion', 'encoding', 'type', 'fields'], path); + if ( + value.__tywrap__ !== 'dataclass' || + value.codecVersion !== 2 || + value.encoding !== 'fields' || + value.type !== contract.typeId || + !isRecord(value.fields) + ) { + throw new PrototypeError(`invalid dataclass identity at ${path}`); + } + exactKeys(value.fields, Object.keys(contract.fields), `${path}.fields`); + const decoded = Object.fromEntries( + Object.entries(contract.fields).map(([key, field]) => [ + key, + walk(value.fields[key], field, 'decode', `${path}.fields.${key}`, depth + 1, state), + ]) + ); + provenance.set(decoded, contract); + return decoded; + } finally { + leave(state, value); + } +} + +export function encodeExactRequest( + value: unknown, + contract: PrototypeContract, + maxPayloadBytes = MAX_PAYLOAD_BYTES +): unknown { + const encoded = walk(value, contract, 'encode', 'args', 0, { + nodes: 0, + active: new WeakSet(), + }); + checkPayload(encoded, maxPayloadBytes); + return encoded; +} + +export function decodeExactResponse( + value: unknown, + contract: PrototypeContract, + maxPayloadBytes = MAX_PAYLOAD_BYTES +): unknown { + checkPayload(value, maxPayloadBytes); + return walk(value, contract, 'decode', 'result', 0, { + nodes: 0, + active: new WeakSet(), + }); +} + +export function validateDataclassOrigin(value: unknown, contract: PrototypeContract): void { + if (contract.kind !== 'dataclass' || !isRecord(value) || provenance.get(value) !== contract) { + throw new PrototypeError('value lacks verified dataclass provenance'); + } +} diff --git a/test/python/test_value_extensions_prototype.py b/test/python/test_value_extensions_prototype.py new file mode 100644 index 00000000..0ae5fef4 --- /dev/null +++ b/test/python/test_value_extensions_prototype.py @@ -0,0 +1,264 @@ +"""Bounded design tests. These codecs are not part of the shipped bridge.""" + +from __future__ import annotations + +import math +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + + +sys.path.insert(0, str(Path(__file__).parent.parent / 'prototypes')) +import value_extensions as prototype # noqa: E402 + +INTEGER = {'kind': 'integer'} + + +def test_nested_exact_integers_round_trip_beyond_int64() -> None: + value = { + 'outer': [2**80 + 1, -(2**130 + 7)], + 'nested': {'negative': -(2**130 + 7)}, + 'safe': 7, + 'flag': True, + } + encoded = prototype.encode_exact_integers(value) + assert encoded['outer'][0]['value'] == str(2**80 + 1) + assert encoded['safe']['value'] == '7' + assert encoded['flag'] is True + contract = { + 'kind': 'record', + 'fields': { + 'outer': {'kind': 'array', 'item': INTEGER}, + 'nested': {'kind': 'record', 'fields': {'negative': INTEGER}}, + 'safe': INTEGER, + 'flag': {'kind': 'boolean'}, + }, + } + assert prototype.decode_exact_integers(encoded, contract) == value + + +@pytest.mark.parametrize('bad', ['-0', '+1', '01', '-01', '1.0', '1e2', ' 1', '١']) +def test_integer_decimal_rejects_noncanonical_values(bad: str) -> None: + envelope = { + '__tywrap__': 'integer', + 'codecVersion': 2, + 'encoding': 'decimal', + 'value': bad, + } + with pytest.raises(prototype.PrototypeError, match='noncanonical integer decimal'): + prototype.decode_exact_integers( + {'item': [envelope]}, + { + 'kind': 'record', + 'fields': {'item': {'kind': 'array', 'item': INTEGER}}, + }, + ) + + +def test_integer_digit_and_payload_caps() -> None: + accepted = int('9' * prototype.MAX_DECIMAL_DIGITS) + assert prototype.decode_exact_integers( + prototype.encode_exact_integers(accepted), INTEGER + ) == accepted + with pytest.raises(prototype.PrototypeError, match='4096 digits'): + prototype.encode_exact_integers(int('9' * (prototype.MAX_DECIMAL_DIGITS + 1))) + with pytest.raises( + prototype.PrototypeError, match=r'4096 digits at result.big\[0\]' + ): + prototype.encode_exact_integers({'big': [10**5000]}) + with pytest.raises(prototype.PrototypeError, match='payload exceeds 80 bytes'): + prototype.encode_exact_integers({'text': 'x' * 80}, max_payload_bytes=80) + + +def test_unicode_payload_cap_counts_compact_utf8_bytes() -> None: + assert prototype.encode_exact_integers('é', max_payload_bytes=4) == 'é' + with pytest.raises(prototype.PrototypeError, match='payload exceeds 3 bytes'): + prototype.encode_exact_integers('é', max_payload_bytes=3) + + nested = {'café': ['🍵', 'é']} + expected_bytes = len('{"café":["🍵","é"]}'.encode('utf-8')) + assert prototype.encode_exact_integers( + nested, max_payload_bytes=expected_bytes + ) == nested + with pytest.raises(prototype.PrototypeError, match='payload exceeds'): + prototype.encode_exact_integers( + nested, max_payload_bytes=expected_bytes - 1 + ) + + +@pytest.mark.parametrize( + ('version_token', 'accepted'), + [('2', True), ('2.0', True), ('2e0', True), ('true', False), ('"2"', False), ('2.5', False), ('null', False)], +) +def test_integer_version_uses_json_numeric_value(version_token: str, accepted: bool) -> None: + envelope = json.loads( + '{"__tywrap__":"integer","codecVersion":' + + version_token + + ',"encoding":"decimal","value":"7"}' + ) + if accepted: + assert prototype.decode_exact_integers(envelope, INTEGER) == 7 + else: + with pytest.raises(prototype.PrototypeError): + prototype.decode_exact_integers(envelope, INTEGER) + + +def test_integer_envelope_and_capability_fail_closed() -> None: + encoded = prototype.encode_exact_integers(2**80) + for change in ( + {'codecVersion': 1}, + {'encoding': 'json'}, + {'unexpected': True}, + ): + with pytest.raises(prototype.PrototypeError, match='invalid integer envelope'): + prototype.decode_exact_integers({**encoded, **change}, INTEGER) + with pytest.raises(prototype.PrototypeError, match='untagged integer at args'): + prototype.decode_exact_integers(42, INTEGER) + with pytest.raises(prototype.PrototypeError, match='reserved record key at result.__tywrap__'): + prototype.encode_exact_integers({'__tywrap__': 'ordinary'}) + with pytest.raises(prototype.PrototypeError, match='bridge lacks'): + prototype.require_capability({}, 'exactIntegerDecimalV2', 'bigint-v2') + with pytest.raises(prototype.PrototypeError, match='unsupported per-call'): + prototype.require_capability( + {'valueCapabilities': ['exactIntegerDecimalV2']}, + 'exactIntegerDecimalV2', + 'fields-v2', + ) + + +def test_float_contract_recovers_integer_json_tokens_without_weakening_integer() -> None: + assert prototype.decode_exact_integers(1, {'kind': 'float'}) == 1.0 + assert type(prototype.decode_exact_integers(1, {'kind': 'float'})) is float + assert prototype.decode_exact_integers(1.5, {'kind': 'float'}) == 1.5 + zero = prototype.decode_exact_integers( + {'__tywrap__': 'float', 'codecVersion': 2, 'encoding': 'negative-zero'}, + {'kind': 'float'}, + ) + assert math.copysign(1, zero) == -1 + with pytest.raises(prototype.PrototypeError, match='untagged integer'): + prototype.decode_exact_integers(1, INTEGER) + with pytest.raises( + prototype.PrototypeError, + match=r'untagged integer at args\.mixed\[0\]\.quantity', + ): + prototype.decode_exact_integers( + {'mixed': [{'quantity': 1, 'ratio': 2, 'flag': True}]}, + { + 'kind': 'record', + 'fields': { + 'mixed': { + 'kind': 'array', + 'item': { + 'kind': 'record', + 'fields': { + 'quantity': INTEGER, + 'ratio': {'kind': 'float'}, + 'flag': {'kind': 'boolean'}, + }, + }, + }, + }, + }, + ) + with pytest.raises(prototype.PrototypeError, match='expected finite float'): + prototype.decode_exact_integers(True, {'kind': 'float'}) + with pytest.raises(prototype.PrototypeError, match='invalid float envelope'): + prototype.decode_exact_integers( + {'__tywrap__': 'float', 'codecVersion': 1, 'encoding': 'negative-zero'}, + {'kind': 'float'}, + ) + + +def test_integer_cycles_and_depth_reject_with_path() -> None: + cycle: list[object] = [] + cycle.append(cycle) + with pytest.raises(prototype.PrototypeError, match=r'cycle at result\[0\]'): + prototype.encode_exact_integers(cycle) + nested: object = 1 + for _ in range(prototype.MAX_DEPTH + 1): + nested = [nested] + with pytest.raises(prototype.PrototypeError, match='maximum depth'): + prototype.encode_exact_integers(nested) + + +def test_dataclass_point_fields_and_identity() -> None: + encoded = prototype.encode_dataclass(prototype.Point(1, 2), prototype.Point) + assert encoded == { + '__tywrap__': 'dataclass', + 'codecVersion': 2, + 'encoding': 'fields', + 'type': 'value_extensions.Point', + 'fields': {'x': 1, 'y': 2}, + } + + @dataclass + class Record: + name: str + optional: int | None = None + output: int = field(init=False, default=3) + + record = prototype.encode_dataclass(Record('a'), Record) + assert record['fields'] == {'name': 'a', 'optional': None, 'output': 3} + + +def test_dataclass_nested_values_and_failures() -> None: + @dataclass + class Container: + points: list[prototype.Point] + + encoded = prototype.encode_dataclass(Container([prototype.Point(1, 2)]), Container) + assert encoded['fields']['points'][0]['fields'] == {'x': 1, 'y': 2} + + with pytest.raises(prototype.PrototypeError, match='declared dataclass type'): + prototype.encode_dataclass(prototype.Point(1, 2), Container) + + class OrdinaryPoint: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + with pytest.raises(prototype.PrototypeError, match='declared dataclass type'): + prototype.encode_dataclass(OrdinaryPoint(1, 2), prototype.Point) + + @dataclass + class UnsupportedAnnotation: + x: object + y: int + + assert prototype.encode_dataclass( + UnsupportedAnnotation(1, 2), UnsupportedAnnotation + )['fields'] == {'x': 1, 'y': 2} + + @dataclass + class Unsupported: + value: object + + with pytest.raises(prototype.PrototypeError, match='unsupported value at result.value'): + prototype.encode_dataclass(Unsupported(object()), Unsupported) + + @dataclass + class Cyclic: + value: object = None + + cyclic = Cyclic() + cyclic.value = cyclic + with pytest.raises(prototype.PrototypeError, match='cycle at result.value'): + prototype.encode_dataclass(cyclic, Cyclic) + + with pytest.raises(prototype.PrototypeError, match='payload exceeds 60 bytes'): + prototype.encode_dataclass( + prototype.Point(1, 2), prototype.Point, max_payload_bytes=60 + ) + + @dataclass + class Nested: + value: object + + deep: object = 1 + for _ in range(prototype.MAX_DEPTH + 1): + deep = Nested(deep) + with pytest.raises(prototype.PrototypeError, match='maximum depth'): + prototype.encode_dataclass(deep, Nested) diff --git a/test/value_extensions_prototype.test.ts b/test/value_extensions_prototype.test.ts new file mode 100644 index 00000000..fb55abb7 --- /dev/null +++ b/test/value_extensions_prototype.test.ts @@ -0,0 +1,270 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { PYTHON, PYTHON_AVAILABLE } from './helpers/python-probe.js'; +import { + decodeExactResponse, + encodeExactRequest, + requireCapability, + validateDataclassOrigin, + type PrototypeContract, +} from './prototypes/value_extensions.js'; + +const pythonScript = join(process.cwd(), 'test', 'prototypes', 'value_extensions.py'); +const pythonPath = PYTHON ?? 'python3'; + +function pythonAction(action: string, input: string): unknown { + return JSON.parse( + execFileSync(pythonPath, [pythonScript, action], { + input, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + }) + ) as unknown; +} + +const integerContract: PrototypeContract = { + kind: 'record', + fields: { + positive: { kind: 'array', item: { kind: 'integer' } }, + negative: { kind: 'record', fields: { value: { kind: 'integer' } } }, + safe: { kind: 'integer' }, + flag: { kind: 'boolean' }, + whole: { kind: 'float' }, + fraction: { kind: 'float' }, + negativeZero: { kind: 'float' }, + mixed: { + kind: 'array', + item: { + kind: 'record', + fields: { + quantity: { kind: 'integer' }, + ratio: { kind: 'float' }, + flag: { kind: 'boolean' }, + }, + }, + }, + }, +}; + +describe.skipIf(!PYTHON_AVAILABLE || !existsSync(pythonScript))( + 'exact integer cross-language prototype', + () => { + it('round-trips nested positive and negative values beyond int64 in both directions', () => { + const positive = 2n ** 80n + 1n; + const negative = -(2n ** 130n + 7n); + const value = { + positive: [positive], + negative: { value: negative }, + safe: 7n, + flag: true, + whole: 1, + fraction: 1.5, + negativeZero: -0, + mixed: [{ quantity: -9n, ratio: 2, flag: false }], + }; + + const pythonWire = pythonAction( + 'encode-integer', + `{"positive":[${positive}],"negative":{"value":${negative}},"safe":7,"flag":true,"whole":1.0,"fraction":1.5,"negativeZero":-0.0,"mixed":[{"quantity":-9,"ratio":2.0,"flag":false}]}` + ); + expect(decodeExactResponse(pythonWire, integerContract)).toEqual(value); + + requireCapability( + { valueCapabilities: ['exactIntegerDecimalV2'] }, + 'exactIntegerDecimalV2', + 'bigint-v2' + ); + const request = encodeExactRequest(value, integerContract); + expect((request as { negativeZero: unknown }).negativeZero).toEqual({ + __tywrap__: 'float', + codecVersion: 2, + encoding: 'negative-zero', + }); + const response = pythonAction( + 'roundtrip-integer', + JSON.stringify({ + meta: { valueCapabilities: ['exactIntegerDecimalV2'] }, + policy: 'bigint-v2', + value: request, + }) + ); + expect(decodeExactResponse(response, integerContract)).toEqual(value); + expect(Object.is((response as { negativeZero: number }).negativeZero, -0)).toBe(true); + }); + } +); + +describe('exact integer rejection prototype', () => { + const integer: PrototypeContract = { kind: 'integer' }; + + it('rejects an old bridge, a number under bigint typing, and a version 1 tag', () => { + expect(() => requireCapability({}, 'exactIntegerDecimalV2', 'bigint-v2')).toThrow( + /bridge lacks/ + ); + expect(() => encodeExactRequest(42, integer)).toThrow(/expected bigint at args/); + expect(() => + decodeExactResponse( + { __tywrap__: 'integer', codecVersion: 1, encoding: 'decimal', value: '42' }, + integer + ) + ).toThrow(/invalid integer envelope/); + }); + + it.each(['-0', '+1', '01', '-01', '1.0', '1e2', ' 1', '١'])( + 'rejects noncanonical decimal %s', + decimal => { + expect(() => + decodeExactResponse( + { __tywrap__: 'integer', codecVersion: 2, encoding: 'decimal', value: decimal }, + integer + ) + ).toThrow(/noncanonical integer decimal/); + } + ); + + it('enforces digit and payload limits', () => { + const max = BigInt('9'.repeat(4096)); + expect(decodeExactResponse(encodeExactRequest(max, integer), integer)).toBe(max); + expect(() => encodeExactRequest(BigInt('9'.repeat(4097)), integer)).toThrow(/4096 digits/); + expect(() => encodeExactRequest(1n, integer, 20)).toThrow(/payload exceeds 20 bytes/); + }); + + it('counts compact UTF-8 JSON bytes for Unicode strings and nested records', () => { + const text: PrototypeContract = { kind: 'string' }; + expect(encodeExactRequest('é', text, 4)).toBe('é'); + expect(() => encodeExactRequest('é', text, 3)).toThrow(/payload exceeds 3 bytes/); + + const nested = { café: ['🍵', 'é'] }; + const nestedContract: PrototypeContract = { + kind: 'record', + fields: { café: { kind: 'array', item: text } }, + }; + const limit = new TextEncoder().encode('{"café":["🍵","é"]}').length; + expect(encodeExactRequest(nested, nestedContract, limit)).toEqual(nested); + expect(() => encodeExactRequest(nested, nestedContract, limit - 1)).toThrow(/payload exceeds/); + expect(() => encodeExactRequest('\ud800', text)).toThrow(/unpaired Unicode surrogate/); + }); + + it('rejects a record key reserved for envelopes', () => { + const record: PrototypeContract = { + kind: 'record', + fields: { __tywrap__: { kind: 'string' } }, + }; + expect(() => encodeExactRequest({ __tywrap__: 'ordinary' }, record)).toThrow( + /reserved record key/ + ); + }); + + it('rejects malformed negative-zero envelopes', () => { + const float: PrototypeContract = { kind: 'float' }; + expect(() => + decodeExactResponse( + { __tywrap__: 'float', codecVersion: 1, encoding: 'negative-zero' }, + float + ) + ).toThrow(/invalid float envelope/); + }); +}); + +describe.skipIf(!PYTHON_AVAILABLE || !existsSync(pythonScript))( + 'cross-language version and Unicode policy', + () => { + it.each([ + ['2', true], + ['2.0', true], + ['2e0', true], + ['true', false], + ['"2"', false], + ['2.5', false], + ['null', false], + ] as const)('agrees on integer envelope version %s', (version, accepted) => { + const raw = `{"meta":{"valueCapabilities":["exactIntegerDecimalV2"]},"policy":"bigint-v2","value":{"__tywrap__":"integer","codecVersion":${version},"encoding":"decimal","value":"7"}}`; + const request = JSON.parse(raw) as { value: unknown }; + if (accepted) { + expect(decodeExactResponse(request.value, { kind: 'integer' })).toBe(7n); + } else { + expect(() => decodeExactResponse(request.value, { kind: 'integer' })).toThrow(); + } + const python = spawnSync(pythonPath, [pythonScript, 'roundtrip-single-integer'], { + input: raw, + encoding: 'utf8', + }); + expect(python.status === 0).toBe(accepted); + if (accepted) { + expect(JSON.parse(python.stdout)).toEqual({ + __tywrap__: 'integer', + codecVersion: 2, + encoding: 'decimal', + value: '7', + }); + } + }); + + it('agrees on nested Unicode and rejects unpaired surrogates', () => { + const nested = { café: ['🍵', 'é'] }; + expect(pythonAction('encode-integer', JSON.stringify(nested))).toEqual(nested); + const encoded = spawnSync(pythonPath, [pythonScript, 'encode-integer'], { + input: JSON.stringify(nested), + encoding: 'utf8', + }); + expect(encoded.status).toBe(0); + expect(encoded.stdout.trim()).toBe(JSON.stringify(nested)); + const wire = Buffer.from(JSON.stringify(nested), 'utf8'); + const exact = spawnSync(pythonPath, [pythonScript, 'encode-integer-limited'], { + input: JSON.stringify({ value: nested, limit: wire.length }), + }); + expect(exact.status).toBe(0); + expect(exact.stdout).toEqual(Buffer.concat([wire, Buffer.from('\n')])); + expect(exact.stdout.subarray(0, -1).length).toBe(wire.length); + const tooSmall = spawnSync(pythonPath, [pythonScript, 'encode-integer-limited'], { + input: JSON.stringify({ value: nested, limit: wire.length - 1 }), + }); + expect(tooSmall.status).not.toBe(0); + + const short = spawnSync(pythonPath, [pythonScript, 'encode-integer-limited'], { + input: JSON.stringify({ value: 'é', limit: 4 }), + }); + expect(short.stdout).toEqual(Buffer.from('"é"\n', 'utf8')); + expect(short.stdout.subarray(0, -1).length).toBe(4); + const bad = spawnSync(pythonPath, [pythonScript, 'encode-integer'], { + input: JSON.stringify('\ud800'), + encoding: 'utf8', + }); + expect(bad.status).not.toBe(0); + expect(bad.stderr).toContain('PrototypeError'); + }); + } +); + +describe.skipIf(!PYTHON_AVAILABLE || !existsSync(pythonScript))( + 'dataclass output prototype', + () => { + const point: PrototypeContract = { + kind: 'dataclass', + typeId: '__main__.Point', + fields: { x: { kind: 'safe-integer' }, y: { kind: 'safe-integer' } }, + }; + + it('decodes Point and preserves verified origin', () => { + expect(() => encodeExactRequest({ x: 1, y: 2 }, point)).toThrow( + /dataclass inputs are unsupported at args/ + ); + const wire = pythonAction('encode-point', JSON.stringify({ x: 1, y: 2 })); + const decoded = decodeExactResponse(wire, point); + expect(decoded).toEqual({ x: 1, y: 2 }); + expect(() => validateDataclassOrigin(decoded, point)).not.toThrow(); + expect(() => validateDataclassOrigin({ x: 1, y: 2 }, point)).toThrow(/lacks verified/); + expect(() => + decodeExactResponse({ ...(wire as object), type: 'other.Point' }, point) + ).toThrow(/invalid dataclass identity/); + expect(() => decodeExactResponse({ ...(wire as object), fields: { x: 1 } }, point)).toThrow( + /fields differ/ + ); + expect(() => + decodeExactResponse({ ...(wire as object), fields: { x: '1', y: 2 } }, point) + ).toThrow(/expected safe integer/); + }); + } +);