From eabfe66b42e69914c4b1ee5679073b5ceef05667 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sat, 19 Sep 2026 23:55:41 -0700 Subject: [PATCH 01/50] docs(codec): freeze internal value contract prototype --- docs/maintainers/value-contracts.md | 12 +++++ docs/maintainers/value-contracts.v1.json | 53 +++++++++++++++++++ src/contracts/value-contract.ts | 66 ++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 docs/maintainers/value-contracts.md create mode 100644 docs/maintainers/value-contracts.v1.json create mode 100644 src/contracts/value-contract.ts diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md new file mode 100644 index 00000000..26309ec7 --- /dev/null +++ b/docs/maintainers/value-contracts.md @@ -0,0 +1,12 @@ +# Value contracts + +The frozen internal prototype is [value-contracts.v1.json](value-contracts.v1.json). +The TypeScript interface is `src/contracts/value-contract.ts`. Its revision is 1. + +Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. A sequence or record holds child contracts. An ndarray holds one element contract. A Torch tensor uses the ndarray contract for its value. Unsupported conversions carry a reason and guidance. + +The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. + +Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow tables. JSON fallback for ndarray and pandas values still rejects unsafe integers. This is a behavior change from silent rounding to a call error. + +The prototype covers safe integers, float16 Arrow ndarrays, Torch reuse, and nested records. Bigint and dataclass designs remain separate. Runtime changes and proof fixtures will follow this interface commit. diff --git a/docs/maintainers/value-contracts.v1.json b/docs/maintainers/value-contracts.v1.json new file mode 100644 index 00000000..d1776b5a --- /dev/null +++ b/docs/maintainers/value-contracts.v1.json @@ -0,0 +1,53 @@ +{ + "revision": 1, + "status": "frozen-internal-prototype", + "transport": "tywrap/1 value RPC", + "codecVersion": 1, + "rules": { + "integer": { + "logicalKind": "Python int, excluding bool", + "constraint": "-9007199254740991 <= value <= 9007199254740991", + "wire": "JSON integer number", + "decodedAs": "JavaScript number", + "failure": "reject before JSON encoding with result path and explicit conversion guidance" + }, + "float": { + "logicalKind": "finite Python float", + "constraint": "finite", + "wire": "JSON number", + "decodedAs": "JavaScript number", + "failure": "reject NaN and Infinity" + }, + "sequence": { + "logicalKind": "list or tuple of contracted values", + "wire": "JSON array", + "decodedAs": "JavaScript array", + "composition": "apply each item rule at its result path" + }, + "record": { + "logicalKind": "string-keyed record of contracted values", + "wire": "JSON object", + "decodedAs": "JavaScript object", + "composition": "apply each field rule at its result path" + }, + "ndarray-float16": { + "logicalKind": "NumPy float16 ndarray", + "wire": "codecVersion 1 ndarray Arrow envelope; JSON fallback stays explicit", + "decodedAs": "JavaScript numeric scalar or nested array", + "conversion": "decode each Arrow Uint16 storage word as IEEE 754 binary16 before reshape", + "failure": "reject non-finite converted values" + }, + "torch-float16": { + "logicalKind": "Torch float16 tensor", + "wire": "codecVersion 1 torch.tensor envelope containing ndarray envelope", + "decodedAs": "tensor record with numeric data", + "conversion": "reuse ndarray-float16 rule for nested value" + } + }, + "compatibility": { + "wireChange": false, + "unknownVersion": "reject", + "unsafeInteger": "previous clients could round; new producers reject", + "generatedWrappers": "regenerate when callable-contract code consumes revision 1" + } +} diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts new file mode 100644 index 00000000..be631727 --- /dev/null +++ b/src/contracts/value-contract.ts @@ -0,0 +1,66 @@ +/** Internal value policy consumed by codecs and callable-contract generation. */ +export const VALUE_CONTRACT_REVISION = 1 as const; + +/** JSON numbers can carry Python integers exactly only within this range. */ +export const MAX_SAFE_JSON_INTEGER = 2 ** 53 - 1; + +export type ValueContract = + | { readonly kind: 'null'; readonly wire: 'json'; readonly decodedAs: 'null' } + | { readonly kind: 'boolean'; readonly wire: 'json'; readonly decodedAs: 'boolean' } + | { + readonly kind: 'integer'; + readonly wire: 'json'; + readonly decodedAs: 'number'; + readonly constraint: 'safe-integer'; + } + | { + readonly kind: 'float'; + readonly wire: 'json'; + readonly decodedAs: 'number'; + readonly constraint: 'finite'; + } + | { readonly kind: 'string'; readonly wire: 'json'; readonly decodedAs: 'string' } + | { + readonly kind: 'sequence'; + readonly wire: 'json'; + readonly decodedAs: 'array'; + readonly item: ValueContract; + } + | { + readonly kind: 'record'; + readonly wire: 'json'; + readonly decodedAs: 'object'; + readonly fields: readonly ValueContractField[]; + readonly additionalValues?: ValueContract; + } + | { + readonly kind: 'ndarray'; + readonly wire: 'arrow' | 'json-fallback'; + readonly decodedAs: 'nested-array-or-scalar'; + readonly element: ValueContract; + readonly dtype?: string; + readonly rank?: number; + } + | { + readonly kind: 'torch-tensor'; + readonly wire: 'ndarray-envelope'; + readonly decodedAs: 'tensor-record'; + readonly value: Extract; + } + | { + readonly kind: 'unsupported'; + readonly wire: 'none'; + readonly decodedAs: 'never'; + readonly reason: string; + readonly guidance: string; + }; + +export interface ValueContractField { + readonly name: string; + readonly value: ValueContract; + readonly required: boolean; +} + +export function isSafeJsonInteger(value: number): boolean { + return Number.isInteger(value) && Math.abs(value) <= MAX_SAFE_JSON_INTEGER; +} From 103ccd6ca064802931e042fb9de23b38d5180885 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:03:55 -0700 Subject: [PATCH 02/50] docs(codec): narrow prototype to float16 scientific values --- docs/maintainers/value-contracts.md | 4 ++-- docs/maintainers/value-contracts.v1.json | 4 +++- src/contracts/value-contract.ts | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index 26309ec7..da4ea5f4 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -1,9 +1,9 @@ # Value contracts The frozen internal prototype is [value-contracts.v1.json](value-contracts.v1.json). -The TypeScript interface is `src/contracts/value-contract.ts`. Its revision is 1. +That JSON file owns the policy. The TypeScript interface mirrors it and has revision 1. -Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. A sequence or record holds child contracts. An ndarray holds one element contract. A Torch tensor uses the ndarray contract for its value. Unsupported conversions carry a reason and guidance. +Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. A sequence or record holds child contracts. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. diff --git a/docs/maintainers/value-contracts.v1.json b/docs/maintainers/value-contracts.v1.json index d1776b5a..1fb06e44 100644 --- a/docs/maintainers/value-contracts.v1.json +++ b/docs/maintainers/value-contracts.v1.json @@ -6,7 +6,8 @@ "rules": { "integer": { "logicalKind": "Python int, excluding bool", - "constraint": "-9007199254740991 <= value <= 9007199254740991", + "minimum": -9007199254740991, + "maximum": 9007199254740991, "wire": "JSON integer number", "decodedAs": "JavaScript number", "failure": "reject before JSON encoding with result path and explicit conversion guidance" @@ -44,6 +45,7 @@ "conversion": "reuse ndarray-float16 rule for nested value" } }, + "prototypeBoundary": "Only float16 scientific values use the new ndarray contract. Existing Arrow int64 values may decode as bigint and keep their current rules.", "compatibility": { "wireChange": false, "unknownVersion": "reject", diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts index be631727..524a0dce 100644 --- a/src/contracts/value-contract.ts +++ b/src/contracts/value-contract.ts @@ -34,18 +34,18 @@ export type ValueContract = readonly additionalValues?: ValueContract; } | { - readonly kind: 'ndarray'; + readonly kind: 'ndarray-float16'; readonly wire: 'arrow' | 'json-fallback'; readonly decodedAs: 'nested-array-or-scalar'; - readonly element: ValueContract; - readonly dtype?: string; + readonly element: Extract; + readonly dtype: 'float16'; readonly rank?: number; } | { - readonly kind: 'torch-tensor'; + readonly kind: 'torch-float16'; readonly wire: 'ndarray-envelope'; readonly decodedAs: 'tensor-record'; - readonly value: Extract; + readonly value: Extract; } | { readonly kind: 'unsupported'; From dfef732b38845f31e06dc22fff0da6d64254e78b Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:12:08 -0700 Subject: [PATCH 03/50] feat(codegen): validate and preserve callable contracts --- docs/maintainers/callable-contracts.md | 42 ++ src/core/callable-compiler.ts | 645 +++++++++++++++++++++++++ src/core/generator.ts | 113 ++++- src/core/ir-contract.ts | 422 ++++++++++++++++ src/types/index.ts | 13 + src/tywrap.ts | 84 ++-- test/callable-compiler.test.ts | 287 +++++++++++ test/generator.test.ts | 94 ++++ test/integration.test.ts | 65 +++ test/ir-contract-validation.test.ts | 88 ++++ 10 files changed, 1801 insertions(+), 52 deletions(-) create mode 100644 docs/maintainers/callable-contracts.md create mode 100644 src/core/callable-compiler.ts create mode 100644 src/core/ir-contract.ts create mode 100644 test/callable-compiler.test.ts create mode 100644 test/ir-contract-validation.test.ts diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md new file mode 100644 index 00000000..93047c74 --- /dev/null +++ b/docs/maintainers/callable-contracts.md @@ -0,0 +1,42 @@ +# Callable compilation contracts + +`tywrap_ir` emits IR 0.4. `validateIrContract()` accepts unknown JSON and checks every required IR field. Each diagnostic has a JSON path. + +`compileContract()` is an internal pure step. It accepts validated IR, a parser-mapped module, a value conversion description, capability descriptions, and a generator. It returns the resolved module, callable records, local diagnostics, and TypeScript and declaration file content. + +The compiler does not read files. It does not start Python. It does not read a cache. It does not write files. It does not access a runtime registry. + +`generate()` owns Python discovery, cache reads, output comparison, and writes. It validates IR before it maps annotations. It also preserves the current generated-file behavior while the value-contract prototype lacks union and reference nodes. + +## Resolved callable data + +The compiler records each parameter, result, and overload result with its logical Python type, direction, source path, and conversion result. It uses this record to create one resolved module model. The generator then uses that model for declarations, call wrappers, and return validators. + +Each value conversion uses revision 1 of [value-contracts.v1.json](value-contracts.v1.json). Do not infer a wire rule from the TypeScript type. + +| Case | Result | +| --- | --- | +| `int` | JSON number, JavaScript number, safe-integer constraint | +| nested `dict[str, T]` | JSON record with the child conversion at every value path | +| float16 ndarray | Existing Arrow or JSON-fallback envelope and finite number result | +| float16 Torch tensor | Existing Torch envelope containing the float16 ndarray conversion | + +The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. + +## Diagnostics and fallback + +Known unsupported cases create error diagnostics. The compiler changes their generated result to `unknown`. + +- Coroutines require the capability owned by #338. +- Dataclasses require an adapter owned by #339. +- Non-string record keys require an explicit conversion. + +Unresolved annotations create warning diagnostics. They retain the generator's current explicit fallback. The compiler does not silently claim a supported transport. + +IR overload signatures stay separate from the implementation signature. Generated declarations retain each input-to-result relation. The wrapper keeps the implementation signature for Python argument binding and runtime calls. + +## Deferred work + +Revision 1 does not describe union, reference, or heterogeneous tuple values. Do not route the compiler projection through default generation until that policy is explicit. + +The client-binding work may consume the resolved callable model later. It must remain opt-in. It must not add files to default generation or mutate the global runtime registry. diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts new file mode 100644 index 00000000..6c72519a --- /dev/null +++ b/src/core/callable-compiler.ts @@ -0,0 +1,645 @@ +/** + * Pure callable compilation. + * + * `generate()` owns discovery, caching, and writes. This module owns the + * deterministic step between validated IR and generated wrapper files. + */ + +import { + VALUE_CONTRACT_REVISION, + type ValueContract, +} from '../contracts/value-contract.js'; +import type { + GeneratedCode, + Parameter, + PythonFunction, + PythonModule, + PythonType, +} from '../types/index.js'; +import type { CodeGenerator } from './generator.js'; +import type { ValidatedIrContract } from './ir-contract.js'; + +export type CallableDirection = 'input' | 'output'; + +export type CallableCapability = + | 'value-rpc' + | 'return-validation' + | 'scientific-ndarray' + | 'scientific-torch' + | 'coroutine-execution' + | 'dataclass-adapter'; + +export interface CapabilityDescription { + name: CallableCapability; + available: boolean; + guidance: string; +} + +export interface ValueConversionRequest { + direction: CallableDirection; + logicalType: PythonType; + path: string; +} + +export interface SupportedValueResolution { + status: 'supported'; + value: ValueContract; +} + +export interface UnsupportedValueResolution { + status: 'unsupported'; + reason: string; + guidance: string; +} + +export interface UnresolvedValueResolution { + status: 'unresolved'; + annotation: string; +} + +export type ValueResolution = + | SupportedValueResolution + | UnsupportedValueResolution + | UnresolvedValueResolution; + +/** + * Maps an internal Python type to the frozen #343 value policy. + * + * The compiler does not infer wire behavior from a TypeScript spelling. + */ +export interface ValueConversionDescription { + revision: typeof VALUE_CONTRACT_REVISION; + resolve(request: ValueConversionRequest): ValueResolution; +} + +export interface CallableCompilationDiagnostic { + severity: 'warning' | 'error'; + code: + | 'conversion-unresolved' + | 'conversion-unsupported' + | 'capability-unavailable' + | 'coroutine-unsupported' + | 'dataclass-unsupported'; + path: string; + message: string; +} + +export interface ResolvedCallableValue { + direction: CallableDirection; + path: string; + logicalType: PythonType; + resolution: ValueResolution; +} + +export interface ResolvedCallable { + name: string; + path: string; + parameters: readonly ResolvedCallableValue[]; + result: ResolvedCallableValue; + overloadResults: readonly ResolvedCallableValue[]; + requiredCapabilities: readonly CallableCapability[]; +} + +export interface CompiledFile { + kind: 'typescript' | 'declaration'; + content: string; +} + +export interface CompiledContract { + /** Validated IR identity. It is retained for diagnostics and reproducibility. */ + ir: Pick; + /** Resolved model consumed by declarations, wrappers, and return validators. */ + module: PythonModule; + callables: readonly ResolvedCallable[]; + diagnostics: readonly CallableCompilationDiagnostic[]; + files: readonly CompiledFile[]; + generated: GeneratedCode; +} + +export interface CompileContractOptions { + /** Parser-mapped model derived from the supplied validated IR. */ + module: PythonModule; + /** The generator is synchronous and performs no filesystem or runtime work. */ + generator: Pick; + conversion: ValueConversionDescription; + capabilities: readonly CapabilityDescription[]; + annotatedJSDoc?: boolean; +} + +const UNKNOWN_TYPE: PythonType = { kind: 'custom', name: 'Any', module: 'typing' }; + +export const DEFAULT_CALLABLE_CAPABILITIES: readonly CapabilityDescription[] = [ + { + name: 'value-rpc', + available: true, + guidance: 'Use the existing tywrap/1 value RPC transport.', + }, + { + name: 'return-validation', + available: true, + guidance: 'Generate a return validator for supported decoded values.', + }, + { + name: 'scientific-ndarray', + available: true, + guidance: 'Keep the existing ndarray Arrow envelope and JSON fallback.', + }, + { + name: 'scientific-torch', + available: true, + guidance: 'Reuse the ndarray conversion inside the existing Torch envelope.', + }, + { + name: 'coroutine-execution', + available: false, + guidance: 'Coroutine execution belongs to #338. Expose a value-returning adapter until then.', + }, + { + name: 'dataclass-adapter', + available: false, + guidance: 'Use a TypedDict or an explicit record adapter until #339 defines dataclass conversion.', + }, +]; + +function leafName(type: PythonType): string | null { + if (type.kind !== 'custom' && type.kind !== 'generic') { + return null; + } + return type.name.split('.').at(-1) ?? null; +} + +function annotationName(type: PythonType): string { + switch (type.kind) { + case 'primitive': + return type.name; + case 'custom': + case 'generic': + return type.module ? `${type.module}.${type.name}` : type.name; + default: + return type.kind; + } +} + +function firstUnresolvedChild( + resolutions: readonly ValueResolution[] +): UnsupportedValueResolution | UnresolvedValueResolution | null { + return resolutions.find(resolution => resolution.status !== 'supported') ?? null; +} + +function resolveSequence( + item: PythonType, + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + const child = conversion.resolve({ ...request, logicalType: item, path: `${request.path}[]` }); + if (child.status !== 'supported') { + return child; + } + return { + status: 'supported', + value: { kind: 'sequence', wire: 'json', decodedAs: 'array', item: child.value }, + }; +} + +function resolveRecord( + valueType: PythonType, + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + const child = conversion.resolve({ ...request, logicalType: valueType, path: `${request.path}{value}` }); + if (child.status !== 'supported') { + return child; + } + return { + status: 'supported', + value: { + kind: 'record', + wire: 'json', + decodedAs: 'object', + fields: [], + additionalValues: child.value, + }, + }; +} + +function resolveTuple( + items: readonly PythonType[], + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + if (items.length === 0) { + return { + status: 'supported', + value: { + kind: 'sequence', + wire: 'json', + decodedAs: 'array', + item: { kind: 'null', wire: 'json', decodedAs: 'null' }, + }, + }; + } + const entries = items.map((item, index) => + conversion.resolve({ ...request, logicalType: item, path: `${request.path}[${index}]` }) + ); + const childFailure = firstUnresolvedChild(entries); + if (childFailure) { + return childFailure; + } + const values = entries as SupportedValueResolution[]; + const first = values[0]?.value; + if (!first || values.some(entry => JSON.stringify(entry.value) !== JSON.stringify(first))) { + return { + status: 'unsupported', + reason: 'The frozen value contract does not represent a heterogeneous tuple.', + guidance: 'Return a TypedDict or a homogeneous list until tuple conversion is specified.', + }; + } + return { + status: 'supported', + value: { kind: 'sequence', wire: 'json', decodedAs: 'array', item: first }, + }; +} + +function resolveNdarray(type: PythonType): ValueResolution { + const typeArgument = type.kind === 'generic' ? type.typeArgs[0] : undefined; + const dtype = typeArgument ? leafName(typeArgument) : undefined; + const normalizedDtype = dtype?.toLowerCase(); + if (normalizedDtype && normalizedDtype !== 'float16') { + return { + status: 'unresolved', + annotation: `${annotationName(type)}[${dtype}]`, + }; + } + return { + status: 'supported', + value: { + kind: 'ndarray-float16', + wire: 'arrow', + decodedAs: 'nested-array-or-scalar', + element: { kind: 'float', wire: 'json', decodedAs: 'number', constraint: 'finite' }, + dtype: 'float16', + }, + }; +} + +/** The conversion set that #335 may use with the frozen value-contract revision. */ +export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { + revision: VALUE_CONTRACT_REVISION, + resolve(request): ValueResolution { + const { logicalType: type } = request; + switch (type.kind) { + case 'primitive': + if (type.name === 'None') { + return { status: 'supported', value: { kind: 'null', wire: 'json', decodedAs: 'null' } }; + } + if (type.name === 'bool') { + return { + status: 'supported', + value: { kind: 'boolean', wire: 'json', decodedAs: 'boolean' }, + }; + } + if (type.name === 'int') { + return { + status: 'supported', + value: { + kind: 'integer', + wire: 'json', + decodedAs: 'number', + constraint: 'safe-integer', + }, + }; + } + if (type.name === 'float') { + return { + status: 'supported', + value: { kind: 'float', wire: 'json', decodedAs: 'number', constraint: 'finite' }, + }; + } + if (type.name === 'str') { + return { status: 'supported', value: { kind: 'string', wire: 'json', decodedAs: 'string' } }; + } + return { + status: 'unsupported', + reason: 'The frozen value contract does not specify bytes conversion.', + guidance: 'Use an explicit string or ndarray adapter for byte data.', + }; + case 'annotated': + return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.base }); + case 'final': + case 'classvar': + return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.type }); + case 'collection': + if (type.name === 'dict') { + const key = type.itemTypes[0]; + if (key?.kind !== 'primitive' || key.name !== 'str') { + return { + status: 'unsupported', + reason: 'The frozen record contract requires string keys.', + guidance: 'Convert keys to strings before crossing the value RPC boundary.', + }; + } + return resolveRecord( + type.itemTypes[1] ?? UNKNOWN_TYPE, + request, + DEFAULT_VALUE_CONVERSION + ); + } + if (type.name === 'tuple') { + return resolveTuple(type.itemTypes, request, DEFAULT_VALUE_CONVERSION); + } + return resolveSequence(type.itemTypes[0] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + case 'generic': { + const leaf = leafName(type); + if (leaf === 'NDArray' || leaf === 'ndarray') { + return resolveNdarray(type); + } + if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { + return resolveSequence(type.typeArgs[0] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + } + if (['dict', 'Dict', 'Mapping', 'MutableMapping'].includes(leaf ?? '')) { + const key = type.typeArgs[0]; + if (key?.kind !== 'primitive' || key.name !== 'str') { + return { + status: 'unsupported', + reason: 'The frozen record contract requires string keys.', + guidance: 'Convert keys to strings before crossing the value RPC boundary.', + }; + } + return resolveRecord(type.typeArgs[1] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + } + return { status: 'unresolved', annotation: annotationName(type) }; + } + case 'custom': { + const leaf = leafName(type); + if (leaf === 'ndarray' || leaf === 'NDArray') { + return resolveNdarray(type); + } + if (leaf === 'Tensor' && type.module?.startsWith('torch')) { + const ndarray = resolveNdarray({ kind: 'custom', name: 'ndarray', module: 'numpy' }); + if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { + return ndarray; + } + return { + status: 'supported', + value: { + kind: 'torch-float16', + wire: 'ndarray-envelope', + decodedAs: 'tensor-record', + value: ndarray.value, + }, + }; + } + return { status: 'unresolved', annotation: annotationName(type) }; + } + default: + return { status: 'unresolved', annotation: annotationName(type) }; + } + }, +}; + +function capabilityNamesFor(value: ValueContract): CallableCapability[] { + const names: CallableCapability[] = ['value-rpc', 'return-validation']; + if (value.kind === 'ndarray-float16') { + names.push('scientific-ndarray'); + } + if (value.kind === 'torch-float16') { + names.push('scientific-torch', 'scientific-ndarray'); + } + return names; +} + +function containsNamedType(type: PythonType, names: ReadonlySet): boolean { + if ((type.kind === 'custom' || type.kind === 'generic') && names.has(leafName(type) ?? '')) { + return true; + } + switch (type.kind) { + case 'collection': + return type.itemTypes.some(item => containsNamedType(item, names)); + case 'union': + return type.types.some(item => containsNamedType(item, names)); + case 'optional': + case 'final': + case 'classvar': + case 'unpack': + return containsNamedType(type.type, names); + case 'annotated': + return containsNamedType(type.base, names); + case 'generic': + return type.typeArgs.some(item => containsNamedType(item, names)); + case 'callable': + return ( + type.parameters.some(item => containsNamedType(item, names)) || + containsNamedType(type.returnType, names) + ); + default: + return false; + } +} + +function diagnosticForResolution( + resolution: ValueResolution, + path: string +): CallableCompilationDiagnostic | null { + if (resolution.status === 'supported') { + return null; + } + if (resolution.status === 'unresolved') { + return { + severity: 'warning', + code: 'conversion-unresolved', + path, + message: `${path}: no value conversion is defined for ${resolution.annotation}; the generated type keeps its existing explicit fallback.`, + }; + } + return { + severity: 'error', + code: 'conversion-unsupported', + path, + message: `${path}: ${resolution.reason} ${resolution.guidance}`, + }; +} + +function resolveCallable( + func: PythonFunction, + path: string, + conversion: ValueConversionDescription, + capabilities: ReadonlyMap, + dataclassNames: ReadonlySet, + diagnostics: CallableCompilationDiagnostic[] +): { function: PythonFunction; callable: ResolvedCallable } { + const parameters = func.parameters.map((parameter, index) => ({ + direction: 'input' as const, + path: `${path}.parameters[${index}]`, + logicalType: parameter.type, + resolution: conversion.resolve({ + direction: 'input', + logicalType: parameter.type, + path: `${path}.parameters[${index}]`, + }), + })); + const result: ResolvedCallableValue = { + direction: 'output', + path: `${path}.returns`, + logicalType: func.returnType, + resolution: conversion.resolve({ direction: 'output', logicalType: func.returnType, path: `${path}.returns` }), + }; + const overloadResults = (func.overloads ?? []).map((overload, index) => ({ + direction: 'output' as const, + path: `${path}.overloads[${index}].returns`, + logicalType: overload.returnType, + resolution: conversion.resolve({ + direction: 'output', + logicalType: overload.returnType, + path: `${path}.overloads[${index}].returns`, + }), + })); + const requiredCapabilities = new Set(); + for (const value of [...parameters, result, ...overloadResults]) { + const diagnostic = diagnosticForResolution(value.resolution, value.path); + if (diagnostic) { + diagnostics.push(diagnostic); + } + if (value.resolution.status === 'supported') { + capabilityNamesFor(value.resolution.value).forEach(capability => requiredCapabilities.add(capability)); + } + } + + const forceUnsupportedOutput = ( + resolution: ValueResolution, + logicalType: PythonType, + outputPath: string + ): ValueResolution => { + if (containsNamedType(logicalType, dataclassNames)) { + diagnostics.push({ + severity: 'error', + code: 'dataclass-unsupported', + path: outputPath, + message: `${outputPath}: a dataclass annotation has no value adapter. Use a TypedDict or an explicit record adapter until #339.`, + }); + return { + status: 'unsupported', + reason: 'No dataclass value adapter is available.', + guidance: 'Use a TypedDict or an explicit record adapter.', + }; + } + if (func.isAsync && capabilities.get('coroutine-execution')?.available !== true) { + diagnostics.push({ + severity: 'error', + code: 'coroutine-unsupported', + path: outputPath, + message: `${outputPath}: this callable requires coroutine execution, which #338 has not implemented.`, + }); + return { + status: 'unsupported', + reason: 'Coroutine execution is unavailable.', + guidance: 'Expose a synchronous value-returning adapter until #338 lands.', + }; + } + return resolution; + }; + + result.resolution = forceUnsupportedOutput(result.resolution, result.logicalType, result.path); + overloadResults.forEach(overload => { + overload.resolution = forceUnsupportedOutput( + overload.resolution, + overload.logicalType, + overload.path + ); + }); + for (const capability of requiredCapabilities) { + const description = capabilities.get(capability); + if (description?.available !== true) { + diagnostics.push({ + severity: 'error', + code: 'capability-unavailable', + path, + message: `${path}: ${capability} is required. ${description?.guidance ?? 'Add a declared capability before generating this callable.'}`, + }); + } + } + + const resolvedParameters: Parameter[] = parameters.map((parameter, index) => ({ + ...func.parameters[index]!, + type: + parameter.resolution.status === 'unsupported' ? UNKNOWN_TYPE : parameter.logicalType, + })); + const resolvedOverloads = (func.overloads ?? []).map((overload, index) => ({ + ...overload, + returnType: + overloadResults[index]?.resolution.status === 'unsupported' ? UNKNOWN_TYPE : overload.returnType, + })); + const resolvedFunction: PythonFunction = { + ...func, + parameters: resolvedParameters, + signature: { + ...func.signature, + parameters: resolvedParameters, + returnType: result.resolution.status === 'unsupported' ? UNKNOWN_TYPE : func.returnType, + }, + returnType: result.resolution.status === 'unsupported' ? UNKNOWN_TYPE : func.returnType, + overloads: resolvedOverloads, + }; + return { + function: resolvedFunction, + callable: { + name: func.name, + path, + parameters, + result, + overloadResults, + requiredCapabilities: [...requiredCapabilities].sort(), + }, + }; +} + +/** + * Compile already-validated IR without filesystem, subprocess, cache, or + * registry access. `generate()` supplies the model and writes these files. + */ +export function compileContract( + ir: ValidatedIrContract, + options: CompileContractOptions +): CompiledContract { + const capabilities = new Map(options.capabilities.map(capability => [capability.name, capability])); + const dataclassNames = new Set( + options.module.classes.filter(cls => cls.kind === 'dataclass').map(cls => cls.name) + ); + const diagnostics: CallableCompilationDiagnostic[] = []; + const callables: ResolvedCallable[] = []; + const compileFunction = (func: PythonFunction, path: string): PythonFunction => { + const result = resolveCallable( + func, + path, + options.conversion, + capabilities, + dataclassNames, + diagnostics + ); + callables.push(result.callable); + return result.function; + }; + const module: PythonModule = { + ...options.module, + functions: options.module.functions.map((func, index) => + compileFunction(func, `$.functions[${index}]`) + ), + classes: options.module.classes.map((cls, classIndex) => ({ + ...cls, + methods: cls.methods.map((method, methodIndex) => + compileFunction(method, `$.classes[${classIndex}].methods[${methodIndex}]`) + ), + })), + }; + const generated = options.generator.generateModuleDefinition(module, options.annotatedJSDoc); + return { + ir: { ir_version: ir.ir_version, module: ir.module }, + module, + callables, + diagnostics, + files: [ + { kind: 'typescript', content: generated.typescript }, + { kind: 'declaration', content: generated.declaration }, + ], + generated, + }; +} diff --git a/src/core/generator.ts b/src/core/generator.ts index 899fa6a1..412be8d1 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -9,6 +9,7 @@ import type { PythonModule, PythonType, PythonTypeAlias, + Parameter, GeneratedCode, TypescriptType, } from '../types/index.js'; @@ -592,13 +593,20 @@ export class CodeGenerator { const positionalParams = filteredParams.filter(p => !p.keywordOnly && !p.varArgs && !p.kwArgs); const genericContext = this.buildGenericRenderContext( this.getTypeParameters(func.typeParameters), - [func.returnType, ...filteredParams.map(param => param.type)], + [ + func.returnType, + ...filteredParams.map(param => param.type), + ...(func.overloads ?? []).flatMap(overload => [ + overload.returnType, + ...overload.parameters.map(parameter => parameter.type), + ]), + ], moduleName, localDeclaredNames ); const typeParamDecl = genericContext.declaration; - const tsTypeForValue = (p: (typeof filteredParams)[number]): string => + const tsTypeForValue = (p: Parameter): string => this.typeToTsFromPython(p.type, genericContext, 'value'); const kwargsType = (() => { @@ -665,7 +673,104 @@ export class CodeGenerator { const validatorName = `__validate${this.escapeIdentifier(func.name, { preserveCase: true })}Result`; const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.returnSchema(func.returnType, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; - // Overloads: generate trailing optional parameter drop variants (exclude *args/**kwargs). + const renderDeclaredOverload = (overload: NonNullable[number]): string[] => { + const overloadParams = overload.parameters.filter( + parameter => parameter.name !== 'self' && parameter.name !== 'cls' + ); + const overloadPositional = overloadParams.filter( + parameter => !parameter.keywordOnly && !parameter.varArgs && !parameter.kwArgs + ); + const overloadKeywordOnly = overloadParams.filter(parameter => parameter.keywordOnly); + const overloadVarArgs = overloadParams.find(parameter => parameter.varArgs); + const overloadHasVarKwArgs = overloadParams.some(parameter => parameter.kwArgs); + const overloadNeedsKwargs = overloadKeywordOnly.length > 0 || overloadHasVarKwArgs; + const overloadNeedsVarArgsArray = Boolean(overloadVarArgs) && overloadNeedsKwargs; + const overloadKwargsType = (() => { + if (!overloadNeedsKwargs) { + return ''; + } + if (overloadKeywordOnly.length === 0 && overloadHasVarKwArgs) { + return 'Record'; + } + const properties = overloadKeywordOnly + .map( + parameter => + `${JSON.stringify(parameter.name)}${parameter.optional ? '?' : ''}: ${tsTypeForValue(parameter)};` + ) + .join(' '); + const object = `{ ${properties} }`; + return overloadHasVarKwArgs ? `(${object} & Record)` : object; + })(); + const renderPositional = (parameter: Parameter, forceRequired = false): string => + `${this.escapeIdentifier(parameter.name)}${!forceRequired && parameter.optional ? '?' : ''}: ${tsTypeForValue(parameter)}`; + const renderVarArgs = (forceRequired = false): string | null => { + if (!overloadVarArgs) { + return null; + } + if (!overloadNeedsVarArgsArray) { + return `...${this.escapeIdentifier(overloadVarArgs.name)}: unknown[]`; + } + return `${this.escapeIdentifier(overloadVarArgs.name)}${forceRequired ? '' : '?'}: unknown[]`; + }; + const renderKwargs = (forceRequired = false): string | null => + overloadNeedsKwargs + ? `kwargs${forceRequired ? '' : '?'}: ${overloadKwargsType}` + : null; + const overloadReturnType = this.typeToTsFromPython( + overload.returnType, + genericContext, + 'return' + ); + const signatures: string[] = []; + const addSignature = (parameters: string[]): void => { + signatures.push( + `export function ${fname}${typeParamDecl}(${parameters.join(', ')}): Promise<${overloadReturnType}>;` + ); + }; + const firstOptional = overloadPositional.findIndex(parameter => parameter.optional); + const requiredPositionalCount = + firstOptional >= 0 ? firstOptional : overloadPositional.length; + const requiredKwOnly = overloadKeywordOnly.some(parameter => !parameter.optional); + + if (requiredKwOnly) { + for (let count = requiredPositionalCount; count <= overloadPositional.length; count += 1) { + const head = overloadPositional + .slice(0, count) + .map(parameter => renderPositional(parameter, true)); + const rest: string[] = []; + if (overloadVarArgs) { + if (overloadNeedsVarArgsArray) { + rest.push(`${this.escapeIdentifier(overloadVarArgs.name)}: unknown[] | undefined`); + } else { + rest.push(`...${this.escapeIdentifier(overloadVarArgs.name)}: unknown[]`); + } + } + const kwargs = renderKwargs(true); + if (kwargs) { + rest.push(kwargs); + } + addSignature([...head, ...rest]); + if (overloadVarArgs && overloadNeedsVarArgsArray && kwargs) { + addSignature([...head, kwargs]); + } + } + return signatures; + } + + const parameters = overloadPositional.map(parameter => renderPositional(parameter)); + const varArgs = renderVarArgs(); + if (varArgs) { + parameters.push(varArgs); + } + const kwargs = renderKwargs(); + if (kwargs) { + parameters.push(kwargs); + } + addSignature(parameters); + return signatures; + }; + + // Optional parameter overloads make Python's trailing defaults available to TypeScript callers. // Why: Python APIs frequently have many optional tail params. TypeScript callers expect // `fn(a)`, `fn(a, b)`, ... all to typecheck. We emit a family of overloads that progressively // "drop" optional tail args, but also include the full positional signature (<= length) so a @@ -673,7 +778,7 @@ export class CodeGenerator { const firstOptionalIndex = positionalParams.findIndex(p => p.optional); const requiredKwOnlyNames = keywordOnlyParams.filter(p => !p.optional).map(p => p.name); const keywordOnlyNames = keywordOnlyParams.map(p => p.name); - const overloads: string[] = []; + const overloads: string[] = (func.overloads ?? []).flatMap(renderDeclaredOverload); if (requiredKwOnlyNames.length > 0) { // Required keyword-only params must be represented with a required `kwargs` parameter. // Avoid "required after optional" by emitting overloads where all preceding parameters are required. diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts new file mode 100644 index 00000000..d914aac2 --- /dev/null +++ b/src/core/ir-contract.ts @@ -0,0 +1,422 @@ +/** + * Validate the JSON contract emitted by tywrap_ir 0.4 before compilation. + * + * This module does not read files, start Python, or write generated output. + * It only accepts unknown JSON-compatible input and returns located diagnostics. + */ + +import type { IrContract } from '../types/index.js'; + +export const TYWRAP_IR_VERSION = '0.4.0' as const; + +const PARAMETER_KINDS = new Set([ + 'POSITIONAL_ONLY', + 'POSITIONAL_OR_KEYWORD', + 'VAR_POSITIONAL', + 'KEYWORD_ONLY', + 'VAR_KEYWORD', +]); +const FIELD_KINDS = new Set([...PARAMETER_KINDS, 'FIELD']); + +const METHOD_KINDS = new Set(['instance', 'class', 'static']); +const TYPE_PARAMETER_KINDS = new Set(['typevar', 'paramspec', 'typevartuple']); +const VARIANCES = new Set(['covariant', 'contravariant', 'invariant']); + +export interface IrParameter { + name: string; + kind: string; + annotation: string | null; + default: boolean; +} + +export interface IrTypeParameter { + name: string; + kind: string; + bound: string | null; + constraints: string[] | null; + variance: string | null; +} + +export interface IrOverload { + parameters: IrParameter[]; + returns: string | null; +} + +export interface IrFunction { + name: string; + qualname: string; + docstring: string | null; + parameters: IrParameter[]; + returns: string | null; + is_async: boolean; + is_generator: boolean; + type_params: IrTypeParameter[]; + method_kind: 'instance' | 'class' | 'static'; + overloads: IrOverload[]; +} + +export interface IrAccessor { + name: string; + returns: string | null; + docstring: string | null; + read_only: boolean | null; + is_cached: boolean; +} + +export interface IrClass { + name: string; + qualname: string; + docstring: string | null; + bases: string[]; + methods: IrFunction[]; + typed_dict: boolean; + total: boolean | null; + fields: IrParameter[]; + is_protocol: boolean; + is_namedtuple: boolean; + is_dataclass: boolean; + is_pydantic: boolean; + type_params: IrTypeParameter[]; + accessors: IrAccessor[]; +} + +export interface IrConstant { + name: string; + annotation: string | null; + value_repr: string | null; + is_final: boolean; +} + +export interface IrTypeAlias { + name: string; + definition: string; + is_generic: boolean; + type_params: IrTypeParameter[]; +} + +export interface ValidatedIrContract extends IrContract { + ir_version: typeof TYWRAP_IR_VERSION; + module: string; + functions: IrFunction[]; + classes: IrClass[]; + constants: IrConstant[]; + type_aliases: IrTypeAlias[]; + metadata: Record; + warnings: string[]; +} + +export interface IrDiagnostic { + code: 'ir-version-mismatch' | 'contract-invalid'; + path: string; + message: string; +} + +export type IrValidationResult = + | { ok: true; contract: ValidatedIrContract; diagnostics: readonly [] } + | { ok: false; contract: null; diagnostics: readonly IrDiagnostic[] }; + +class ValidationState { + readonly diagnostics: IrDiagnostic[] = []; + + constructor(private readonly source: string) {} + + invalid(path: string, message: string): void { + this.diagnostics.push({ code: 'contract-invalid', path, message: `${this.source} ${message}` }); + } + + versionMismatch(path: string, found: string): void { + this.diagnostics.push({ + code: 'ir-version-mismatch', + path, + message: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${this.source} declares ${found}. Regenerate the contract with a matching tywrap_ir.`, + }); + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function recordAt(value: unknown, path: string, state: ValidationState): Record | null { + if (!isRecord(value)) { + state.invalid(path, `${path} must be an object.`); + return null; + } + return value; +} + +function requiredValue( + object: Record, + name: string, + path: string, + state: ValidationState +): unknown { + if (!(name in object)) { + state.invalid(`${path}.${name}`, `${path} is missing required field ${name}.`); + return undefined; + } + return object[name]; +} + +function requiredString( + object: Record, + name: string, + path: string, + state: ValidationState, + allowEmpty = false +): string | null { + const value = requiredValue(object, name, path, state); + if (typeof value !== 'string') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a string.`); + return null; + } + if (!allowEmpty && value.length === 0) { + state.invalid(`${path}.${name}`, `${path}.${name} must not be empty.`); + return null; + } + return value; +} + +function requiredNullableString( + object: Record, + name: string, + path: string, + state: ValidationState +): string | null { + const value = requiredValue(object, name, path, state); + if (value === null) { + return null; + } + if (typeof value !== 'string') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a string or null.`); + return null; + } + return value; +} + +function requiredBoolean( + object: Record, + name: string, + path: string, + state: ValidationState +): boolean | null { + const value = requiredValue(object, name, path, state); + if (typeof value !== 'boolean') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a boolean.`); + return null; + } + return value; +} + +function requiredArray( + object: Record, + name: string, + path: string, + state: ValidationState +): unknown[] | null { + const value = requiredValue(object, name, path, state); + if (!Array.isArray(value)) { + const noun = path === '$' ? `required array field ${name}` : `${path}.${name}`; + state.invalid(`${path}.${name}`, `${path} is missing ${noun}.`); + return null; + } + return value; +} + +function validateStringArray(value: unknown, path: string, state: ValidationState): void { + if (!Array.isArray(value)) { + state.invalid(path, `${path} must be an array.`); + return; + } + value.forEach((entry, index) => { + if (typeof entry !== 'string') { + state.invalid(`${path}[${index}]`, `${path}[${index}] must be a string.`); + } + }); +} + +function validateParameter( + value: unknown, + path: string, + state: ValidationState, + allowedKinds: ReadonlySet = PARAMETER_KINDS +): void { + const parameter = recordAt(value, path, state); + if (!parameter) { + return; + } + requiredString(parameter, 'name', path, state); + const kind = requiredString(parameter, 'kind', path, state); + if (kind !== null && !allowedKinds.has(kind)) { + state.invalid(`${path}.kind`, `${path}.kind must be a supported Python parameter kind.`); + } + requiredNullableString(parameter, 'annotation', path, state); + requiredBoolean(parameter, 'default', path, state); +} + +function validateTypeParameter(value: unknown, path: string, state: ValidationState): void { + const parameter = recordAt(value, path, state); + if (!parameter) { + return; + } + requiredString(parameter, 'name', path, state); + const kind = requiredString(parameter, 'kind', path, state); + if (kind !== null && !TYPE_PARAMETER_KINDS.has(kind)) { + state.invalid(`${path}.kind`, `${path}.kind must be typevar, paramspec, or typevartuple.`); + } + requiredNullableString(parameter, 'bound', path, state); + const constraints = requiredValue(parameter, 'constraints', path, state); + if (constraints !== null) { + validateStringArray(constraints, `${path}.constraints`, state); + } + const variance = requiredValue(parameter, 'variance', path, state); + if (variance !== null && (typeof variance !== 'string' || !VARIANCES.has(variance))) { + state.invalid(`${path}.variance`, `${path}.variance must be a supported variance or null.`); + } +} + +function validateFunction(value: unknown, path: string, state: ValidationState): void { + const callable = recordAt(value, path, state); + if (!callable) { + return; + } + requiredString(callable, 'name', path, state); + requiredString(callable, 'qualname', path, state); + requiredNullableString(callable, 'docstring', path, state); + const parameters = requiredArray(callable, 'parameters', path, state); + parameters?.forEach((parameter, index) => validateParameter(parameter, `${path}.parameters[${index}]`, state)); + requiredNullableString(callable, 'returns', path, state); + requiredBoolean(callable, 'is_async', path, state); + requiredBoolean(callable, 'is_generator', path, state); + const typeParameters = requiredArray(callable, 'type_params', path, state); + typeParameters?.forEach((parameter, index) => + validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) + ); + const methodKind = requiredString(callable, 'method_kind', path, state); + if (methodKind !== null && !METHOD_KINDS.has(methodKind)) { + state.invalid(`${path}.method_kind`, `${path}.method_kind must be instance, class, or static.`); + } + const overloads = requiredArray(callable, 'overloads', path, state); + overloads?.forEach((overload, index) => { + const overloadPath = `${path}.overloads[${index}]`; + const signature = recordAt(overload, overloadPath, state); + if (!signature) { + return; + } + const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); + overloadParameters?.forEach((parameter, parameterIndex) => + validateParameter(parameter, `${overloadPath}.parameters[${parameterIndex}]`, state) + ); + requiredNullableString(signature, 'returns', overloadPath, state); + }); +} + +function validateAccessor(value: unknown, path: string, state: ValidationState): void { + const accessor = recordAt(value, path, state); + if (!accessor) { + return; + } + requiredString(accessor, 'name', path, state); + requiredNullableString(accessor, 'returns', path, state); + requiredNullableString(accessor, 'docstring', path, state); + const readOnly = requiredValue(accessor, 'read_only', path, state); + if (readOnly !== null && typeof readOnly !== 'boolean') { + state.invalid(`${path}.read_only`, `${path}.read_only must be a boolean or null.`); + } + requiredBoolean(accessor, 'is_cached', path, state); +} + +function validateClass(value: unknown, path: string, state: ValidationState): void { + const cls = recordAt(value, path, state); + if (!cls) { + return; + } + requiredString(cls, 'name', path, state); + requiredString(cls, 'qualname', path, state); + requiredNullableString(cls, 'docstring', path, state); + validateStringArray(requiredValue(cls, 'bases', path, state), `${path}.bases`, state); + const methods = requiredArray(cls, 'methods', path, state); + methods?.forEach((method, index) => validateFunction(method, `${path}.methods[${index}]`, state)); + requiredBoolean(cls, 'typed_dict', path, state); + const total = requiredValue(cls, 'total', path, state); + if (total !== null && typeof total !== 'boolean') { + state.invalid(`${path}.total`, `${path}.total must be a boolean or null.`); + } + const fields = requiredArray(cls, 'fields', path, state); + fields?.forEach((field, index) => + validateParameter(field, `${path}.fields[${index}]`, state, FIELD_KINDS) + ); + requiredBoolean(cls, 'is_protocol', path, state); + requiredBoolean(cls, 'is_namedtuple', path, state); + requiredBoolean(cls, 'is_dataclass', path, state); + requiredBoolean(cls, 'is_pydantic', path, state); + const typeParameters = requiredArray(cls, 'type_params', path, state); + typeParameters?.forEach((parameter, index) => + validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) + ); + const accessors = requiredArray(cls, 'accessors', path, state); + accessors?.forEach((accessor, index) => validateAccessor(accessor, `${path}.accessors[${index}]`, state)); +} + +function validateConstant(value: unknown, path: string, state: ValidationState): void { + const constant = recordAt(value, path, state); + if (!constant) { + return; + } + requiredString(constant, 'name', path, state); + requiredNullableString(constant, 'annotation', path, state); + requiredNullableString(constant, 'value_repr', path, state); + requiredBoolean(constant, 'is_final', path, state); +} + +function validateTypeAlias(value: unknown, path: string, state: ValidationState): void { + const alias = recordAt(value, path, state); + if (!alias) { + return; + } + requiredString(alias, 'name', path, state); + requiredString(alias, 'definition', path, state, true); + requiredBoolean(alias, 'is_generic', path, state); + const typeParameters = requiredArray(alias, 'type_params', path, state); + typeParameters?.forEach((parameter, index) => + validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) + ); +} + +/** + * Validate every field emitted by `tywrap_ir` 0.4. + * + * The return value is safe to compile without further structural checks. The + * caller owns all filesystem, cache, interpreter, and output operations. + */ +export function validateIrContract(input: unknown, source = 'IR'): IrValidationResult { + const state = new ValidationState(source); + const contract = recordAt(input, '$', state); + if (!contract) { + return { ok: false, contract: null, diagnostics: state.diagnostics }; + } + + const version = requiredString(contract, 'ir_version', '$', state); + if (version !== null && version !== TYWRAP_IR_VERSION) { + state.versionMismatch('$.ir_version', version); + } + requiredString(contract, 'module', '$', state); + const functions = requiredArray(contract, 'functions', '$', state); + functions?.forEach((functionValue, index) => validateFunction(functionValue, `$.functions[${index}]`, state)); + const classes = requiredArray(contract, 'classes', '$', state); + classes?.forEach((classValue, index) => validateClass(classValue, `$.classes[${index}]`, state)); + const constants = requiredArray(contract, 'constants', '$', state); + constants?.forEach((constant, index) => validateConstant(constant, `$.constants[${index}]`, state)); + const aliases = requiredArray(contract, 'type_aliases', '$', state); + aliases?.forEach((alias, index) => validateTypeAlias(alias, `$.type_aliases[${index}]`, state)); + const metadata = requiredValue(contract, 'metadata', '$', state); + if (!isRecord(metadata)) { + state.invalid('$.metadata', '$.metadata must be an object.'); + } + validateStringArray(requiredValue(contract, 'warnings', '$', state), '$.warnings', state); + + if (state.diagnostics.length > 0) { + return { ok: false, contract: null, diagnostics: state.diagnostics }; + } + return { ok: true, contract: input as ValidatedIrContract, diagnostics: [] }; +} diff --git a/src/types/index.ts b/src/types/index.ts index c5b0d408..04b23bb5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -40,6 +40,14 @@ export interface PythonFunction { typeParameters?: PythonGenericParameter[]; returnType: PythonType; parameters: Parameter[]; + /** + * Distinct callable signatures extracted from `@typing.overload`. + * + * The implementation signature stays in `parameters` and `returnType` for + * runtime binding. Each overload retains its own input-to-result relation for + * generated declarations. + */ + overloads?: PythonFunctionOverload[]; /** * Binding of this callable on its owning class. Defaults to `'instance'`. * @see PythonMethodKind @@ -47,6 +55,11 @@ export interface PythonFunction { methodKind?: PythonMethodKind; } +export interface PythonFunctionOverload { + parameters: Parameter[]; + returnType: PythonType; +} + /** * A `@property` or `functools.cached_property` exposed on a class. * diff --git a/src/tywrap.ts b/src/tywrap.ts index 4e079c4f..7a2a8b5b 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -3,6 +3,11 @@ */ import { CodeGenerator } from './core/generator.js'; +import { + TYWRAP_IR_VERSION, + validateIrContract, + type ValidatedIrContract, +} from './core/ir-contract.js'; import { TypeMapper } from './core/mapper.js'; import { parseAnnotationToPythonType } from './core/annotation-parser.js'; import { createConfig } from './config/index.js'; @@ -16,7 +21,6 @@ import type { Parameter, PythonType, PythonModuleConfig, - IrContract, } from './types/index.js'; import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js'; import { globalCache } from './utils/cache.js'; @@ -24,8 +28,6 @@ import { resolvePythonExecutable } from './utils/python.js'; import { computeIrCacheFilename } from './utils/ir-cache.js'; import { logger } from './utils/logger.js'; -const TYWRAP_IR_VERSION = '0.4.0'; - // Collect unknown typing constructs encountered during annotation parsing (per-generate run) let unknownTypeNamesCollector: Map = new Map(); function recordUnknown(name: string): void { @@ -101,7 +103,7 @@ function stableJson(value: unknown): unknown { return value; } -function serializeContract(ir: IrContract): string { +function serializeContract(ir: ValidatedIrContract): string { const contract = { ...ir }; delete contract.metadata; return `${JSON.stringify(stableJson(contract), null, 2)}\n`; @@ -110,43 +112,17 @@ function serializeContract(ir: IrContract): string { function validateIrVersion( ir: unknown, source: string -): { ir: IrContract | null; error?: string; code?: GenerateFailure['code'] } { - if (ir === null || typeof ir !== 'object' || Array.isArray(ir)) { - return { ir: null, code: 'contract-invalid', error: `${source} is not a JSON IR object.` }; - } - const contract = ir as IrContract; - if (typeof contract.ir_version !== 'string') { - return { - ir: null, - code: 'contract-invalid', - error: `${source} is missing its string ir_version.`, - }; +): { ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] } { + const result = validateIrContract(ir, source); + if (result.ok) { + return { ir: result.contract }; } - if (contract.ir_version !== TYWRAP_IR_VERSION) { - return { - ir: null, - code: 'ir-version-mismatch', - error: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${source} declares ${contract.ir_version}. Regenerate the contract with a matching tywrap_ir.`, - }; - } - if (typeof contract.module !== 'string' || contract.module.length === 0) { - return { ir: null, code: 'contract-invalid', error: `${source} is missing its module name.` }; - } - const requiredArrays: ReadonlyArray = [ - ['functions', contract.functions], - ['classes', contract.classes], - ['type_aliases', contract.type_aliases], - ]; - for (const [field, value] of requiredArrays) { - if (!Array.isArray(value)) { - return { - ir: null, - code: 'contract-invalid', - error: `${source} is missing required array field ${field}.`, - }; - } - } - return { ir: contract }; + const first = result.diagnostics[0]; + return { + ir: null, + code: first?.code ?? 'contract-invalid', + error: first?.message ?? `${source} is not a JSON IR object.`, + }; } async function safeReadFile(path: string): Promise { @@ -172,7 +148,7 @@ async function fetchAndCacheIr( cacheKey: string, caching: boolean, checkMode: boolean -): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> { +): Promise<{ ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] }> { let ir: unknown | null = null; let irError: string | undefined; if (caching && fsUtils.isAvailable() && !checkMode) { @@ -217,7 +193,7 @@ async function fetchAndCacheIr( async function readContractInput( moduleKey: string, contractInput: TywrapOptions['contractInput'] -): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> { +): Promise<{ ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] }> { const inputPath = typeof contractInput === 'string' ? contractInput : contractInput?.[moduleKey]; if (!inputPath) { return { @@ -656,12 +632,11 @@ function collectModuleTypeVarNames(obj: Record): Set { return names; } -function transformIrToTsModel(ir: unknown): TSPythonModule { - const obj: Record = - typeof ir === 'object' && ir !== null ? (ir as Record) : {}; - const functions = (obj.functions as unknown[]) ?? []; - const classes = (obj.classes as unknown[]) ?? []; - const aliases = (obj.type_aliases as unknown[]) ?? []; +function transformIrToTsModel(ir: ValidatedIrContract): TSPythonModule { + const obj: Record = ir; + const functions = ir.functions; + const classes = ir.classes; + const aliases = ir.type_aliases; const moduleTypeVarNames = collectModuleTypeVarNames(obj); const parseType = ( annotation: unknown, @@ -736,6 +711,19 @@ function transformIrToTsModel(ir: unknown): TSPythonModule { ? (f.parameters as unknown[]).map(v => mapParam((v ?? {}) as Record, annotationTypeParameters) ) + : [], + overloads: Array.isArray(f.overloads) + ? (f.overloads as unknown[]).map(overload => { + const signature = (overload ?? {}) as Record; + return { + parameters: Array.isArray(signature.parameters) + ? (signature.parameters as unknown[]).map(value => + mapParam((value ?? {}) as Record, annotationTypeParameters) + ) + : [], + returnType: parseType(signature.returns, annotationTypeParameters), + }; + }) : [], methodKind: mapMethodKind(f.method_kind), }; diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts new file mode 100644 index 00000000..40d4cdb1 --- /dev/null +++ b/test/callable-compiler.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, it } from 'vitest'; +import { + compileContract, + DEFAULT_CALLABLE_CAPABILITIES, + DEFAULT_VALUE_CONVERSION, +} from '../src/core/callable-compiler.js'; +import { CodeGenerator } from '../src/core/generator.js'; +import { validateIrContract } from '../src/core/ir-contract.js'; +import type { PythonModule } from '../src/types/index.js'; + +const rawIr = { + ir_version: '0.4.0', + module: 'fixture', + functions: [ + { + name: 'select', + qualname: 'fixture.select', + docstring: null, + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str | int', default: false }, + ], + returns: 'str | int', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [ + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: false }, + ], + returns: 'int', + }, + ], + }, + { + name: 'nested', + qualname: 'fixture.nested', + docstring: null, + parameters: [], + returns: 'dict[str, dict[str, int]]', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + { + name: 'async_value', + qualname: 'fixture.async_value', + docstring: null, + parameters: [], + returns: 'str', + is_async: true, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + { + name: 'point', + qualname: 'fixture.point', + docstring: null, + parameters: [], + returns: 'fixture.Point', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + ], + classes: [ + { + name: 'Point', + qualname: 'fixture.Point', + docstring: null, + bases: ['object'], + methods: [], + typed_dict: false, + total: null, + fields: [ + { name: 'x', kind: 'FIELD', annotation: 'int', default: false }, + { name: 'y', kind: 'FIELD', annotation: 'int', default: false }, + ], + is_protocol: false, + is_namedtuple: false, + is_dataclass: true, + is_pydantic: false, + type_params: [], + accessors: [], + }, + ], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [], +}; + +const moduleModel: PythonModule = { + name: 'fixture', + functions: [ + { + name: 'select', + signature: { parameters: [], returnType: { kind: 'custom', name: 'Any' }, isAsync: false, isGenerator: false }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + overloads: [ + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'int' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'int' }, + }, + ], + }, + { + name: 'nested', + signature: { parameters: [], returnType: { kind: 'custom', name: 'Any' }, isAsync: false, isGenerator: false }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [], + returnType: { + kind: 'collection', + name: 'dict', + itemTypes: [ + { kind: 'primitive', name: 'str' }, + { + kind: 'collection', + name: 'dict', + itemTypes: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + ], + }, + }, + { + name: 'async_value', + signature: { parameters: [], returnType: { kind: 'primitive', name: 'str' }, isAsync: true, isGenerator: false }, + decorators: [], + isAsync: true, + isGenerator: false, + parameters: [], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + name: 'point', + signature: { parameters: [], returnType: { kind: 'custom', name: 'Point' }, isAsync: false, isGenerator: false }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [], + returnType: { kind: 'custom', name: 'Point' }, + }, + ], + classes: [ + { + name: 'Point', + bases: [], + methods: [], + properties: [], + decorators: [], + kind: 'dataclass', + }, + ], + typeAliases: [], + imports: [], + exports: [], +}; + +describe('compileContract', () => { + it('resolves overloads and nested records once before emitting both wrapper files', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + + expect(compiled.files.map(file => file.kind)).toEqual(['typescript', 'declaration']); + expect(compiled.generated.typescript).toContain( + 'export function select(key: string): Promise;' + ); + expect(compiled.generated.declaration).toContain( + 'export function select(key: number): Promise;' + ); + const nested = compiled.callables.find(callable => callable.name === 'nested'); + expect(nested?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'record', + additionalValues: { + kind: 'record', + additionalValues: { kind: 'integer', constraint: 'safe-integer' }, + }, + }, + }); + }); + + it('degrades unimplemented dataclass and coroutine outputs with local diagnostics', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'coroutine-unsupported', path: '$.functions[2].returns' }), + expect.objectContaining({ code: 'dataclass-unsupported', path: '$.functions[3].returns' }), + ]) + ); + expect(compiled.module.functions[2]?.returnType).toEqual({ + kind: 'custom', + name: 'Any', + module: 'typing', + }); + expect(compiled.module.functions[3]?.returnType).toEqual({ + kind: 'custom', + name: 'Any', + module: 'typing', + }); + }); +}); diff --git a/test/generator.test.ts b/test/generator.test.ts index e23e4dfd..e0d8a0dd 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -561,6 +561,100 @@ describe('CodeGenerator', () => { ); }); + it('preserves declared overload input-to-return relationships', () => { + const code = gen.generateFunctionWrapper( + { + name: 'get_value', + signature: { + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + overloads: [ + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'int' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'int' }, + }, + ], + } as any, + 'advanced_types' + ); + + expect(code.typescript).toContain('export function getValue(key: string): Promise;'); + expect(code.typescript).toContain('export function getValue(key: number): Promise;'); + expect(code.typescript).toContain( + 'export async function getValue(key: string | number): Promise' + ); + expect(code.declaration).toContain('export function getValue(key: string): Promise;'); + expect(code.declaration).toContain('export function getValue(key: number): Promise;'); + }); + it('models *args as an array parameter when kwargs are present', () => { const code = gen.generateFunctionWrapper( { diff --git a/test/integration.test.ts b/test/integration.test.ts index 659f67c2..db537357 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -231,6 +231,71 @@ describe('IR-only integration', () => { } }, 30_000); + it('preserves Python overload input-to-return relationships in generated declarations', async () => { + const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-overload-contract-')); + try { + const outDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { advanced_types: { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: ['test/fixtures/python'], + output: { dir: outDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as any); + + expect(result.failures).toEqual([]); + const generated = await fsUtils.readFile(join(outDir, 'advanced_types.generated.d.ts')); + expect(generated).toContain('export function getValue(key: string): Promise;'); + expect(generated).toContain('export function getValue(key: number): Promise;'); + + const consumerPath = join(tempDir, 'consumer.ts'); + await writeFile( + consumerPath, + `import { getValue } from './generated/advanced_types.generated.js'; + +const text: Promise = getValue('key'); +const integer: Promise = getValue(1); +// @ts-expect-error boolean has no declared Python overload +void getValue(true); +// @ts-expect-error string overload cannot return Promise +const wrong: Promise = getValue('key'); + +void text; +void integer; +void wrong; +`, + 'utf8' + ); + const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); + const compile = await processUtils.exec( + process.execPath, + [ + tscPath, + '--ignoreConfig', + '--noEmit', + '--pretty', + 'false', + '--target', + 'ES2022', + '--lib', + 'ES2022,DOM,DOM.Iterable', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck', + consumerPath, + ], + { cwd: process.cwd(), timeoutMs: 30_000 } + ); + + expect(compile.code).toBe(0); + expect(compile.stderr).toBe(''); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); + it('generates safe generic wrappers and declaration files that typecheck', async () => { if (!(await supportsVariadicTypingFeatures())) { return; diff --git a/test/ir-contract-validation.test.ts b/test/ir-contract-validation.test.ts new file mode 100644 index 00000000..984e4667 --- /dev/null +++ b/test/ir-contract-validation.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; +import { TYWRAP_IR_VERSION, validateIrContract } from '../src/core/ir-contract.js'; + +const validContract = { + ir_version: TYWRAP_IR_VERSION, + module: 'fixture', + functions: [ + { + name: 'get_value', + qualname: 'fixture.get_value', + docstring: null, + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str | int', default: false }, + ], + returns: 'str | int', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [ + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: false }, + ], + returns: 'int', + }, + ], + }, + ], + classes: [], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [], +}; + +describe('validateIrContract', () => { + it('accepts the complete IR 0.4 callable shape, including overloads', () => { + const result = validateIrContract(validContract, 'fixture contract'); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.contract.functions[0]?.overloads).toHaveLength(2); + } + }); + + it('reports each malformed nested overload field at its JSON path', () => { + const invalid = structuredClone(validContract); + invalid.functions[0]!.overloads[1]!.parameters[0]!.kind = 'MALFORMED'; + invalid.functions[0]!.overloads[1]!.returns = 42 as unknown as string; + + const result = validateIrContract(invalid, 'fixture contract'); + + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'contract-invalid', + path: '$.functions[0].overloads[1].parameters[0].kind', + }), + expect.objectContaining({ + code: 'contract-invalid', + path: '$.functions[0].overloads[1].returns', + }), + ]) + ); + } + }); + + it('keeps IR version mismatch separate from structural diagnostics', () => { + const invalid = { ...validContract, ir_version: '0.3.0' }; + const result = validateIrContract(invalid, 'fixture contract'); + + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics).toContainEqual( + expect.objectContaining({ code: 'ir-version-mismatch', path: '$.ir_version' }) + ); + } + }); +}); From 56cfa617b7f3286111d2755d61c8f4a1d232fbfd Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:13:49 -0700 Subject: [PATCH 04/50] docs(codec): compose unions and fixed tuples in value contract --- .../value-contract-fixtures.v2.json | 49 +++++++++++++++++++ docs/maintainers/value-contracts.md | 34 +++++++++++-- ...tracts.v1.json => value-contracts.v2.json} | 25 +++++++++- src/contracts/value-contract.ts | 14 +++++- 4 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 docs/maintainers/value-contract-fixtures.v2.json rename docs/maintainers/{value-contracts.v1.json => value-contracts.v2.json} (61%) diff --git a/docs/maintainers/value-contract-fixtures.v2.json b/docs/maintainers/value-contract-fixtures.v2.json new file mode 100644 index 00000000..abe3a5d8 --- /dev/null +++ b/docs/maintainers/value-contract-fixtures.v2.json @@ -0,0 +1,49 @@ +{ + "revision": 2, + "integerCases": [ + { "decimal": "-9007199254740991", "outcome": "number" }, + { "decimal": "9007199254740991", "outcome": "number" }, + { "decimal": "-9007199254740992", "outcome": "reject" }, + { "decimal": "9007199254740992", "outcome": "reject" }, + { "decimal": "9007199254740993", "outcome": "reject" }, + { "decimal": "-9223372036854775808", "outcome": "reject" }, + { "decimal": "9223372036854775807", "outcome": "reject" }, + { "decimal": "265252859812191058636308480000000", "outcome": "reject" } + ], + "binary16Cases": [ + { "wordHex": "0000", "outcome": "0" }, + { "wordHex": "8000", "outcome": "-0" }, + { "wordHex": "0001", "outcome": "5.960464477539063e-8" }, + { "wordHex": "03ff", "outcome": "0.00006097555160522461" }, + { "wordHex": "0400", "outcome": "0.00006103515625" }, + { "wordHex": "3e00", "outcome": "1.5" }, + { "wordHex": "c080", "outcome": "-2.25" }, + { "wordHex": "7bff", "outcome": "65504" }, + { "wordHex": "fbff", "outcome": "-65504" }, + { "wordHex": "7c00", "outcome": "reject" }, + { "wordHex": "fc00", "outcome": "reject" }, + { "wordHex": "7e00", "outcome": "reject" } + ], + "compositionCases": [ + { + "path": "result.outer[0].value", + "logicalValue": "9007199254740993", + "outcome": "reject" + }, + { + "path": "result.outer[0].tensor.value", + "logicalValue": "float16[1.5,-2.25]", + "outcome": "number[1.5,-2.25]" + } + ], + "unionCases": [ + { "contract": "str|int", "value": "label", "outcome": "string" }, + { "contract": "str|int", "value": 42, "outcome": "safe-integer number" }, + { "contract": "str|int", "value": 9007199254740992, "outcome": "reject" } + ], + "tupleCases": [ + { "contract": "tuple[str,int]", "value": ["label", 42], "outcome": "accept" }, + { "contract": "tuple[str,int]", "value": ["label"], "outcome": "reject-length" }, + { "contract": "tuple[str,int]", "value": ["label", 9007199254740992], "outcome": "reject-integer" } + ] +} diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index da4ea5f4..dc1ac691 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -1,12 +1,38 @@ # Value contracts -The frozen internal prototype is [value-contracts.v1.json](value-contracts.v1.json). -That JSON file owns the policy. The TypeScript interface mirrors it and has revision 1. +The frozen internal prototype is [value-contracts.v2.json](value-contracts.v2.json). +That JSON file owns the policy. The TypeScript interface mirrors it and has revision 2. -Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. A sequence or record holds child contracts. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. +Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. Sequences, fixed tuples, records, and unions compose child contracts. A union has two to 32 alternatives. The compiler rejects a union if one wire value could decode two ways. Contract depth stops at 64. Named records can inline within that limit. Recursive aliases stay unsupported until a reviewed reference graph exists. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow tables. JSON fallback for ndarray and pandas values still rejects unsafe integers. This is a behavior change from silent rounding to a call error. -The prototype covers safe integers, float16 Arrow ndarrays, Torch reuse, and nested records. Bigint and dataclass designs remain separate. Runtime changes and proof fixtures will follow this interface commit. +The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused tests also compare every finite binary16 storage word with an independent float32 bit reference. They test signed zero with `Object.is`. + +The Python serializer checks plain integers, NumPy scalar integers, and model dumps during its existing value walk. A JSON ndarray or sparse matrix checks integer data before it creates an envelope. The TypeScript decoder converts float16 Arrow storage during extraction, before it reshapes the ndarray. Torch uses that same ndarray path. These checks do not walk the full decoded payload again for schema validation. + +## Exact bigint extension for #337 + +This extension is a design sketch. The 0.11 policy still rejects unsafe plain Python integers. + +Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map Python `int` to TypeScript `bigint` for both parameters and returns. The return validator must require `typeof value === 'bigint'`, including every record and sequence member. Keep `bool` separate. Calls without the option keep `number` and the safe range. + +Add a version 2 integer value envelope with a canonical signed decimal string. A sample value is `{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. The request encoder and Python request decoder must validate the same grammar and digit limit. The response encoder and TypeScript decoder must do the same. Use `BigInt` only after validation. Keep Arrow int64 columns on their current Arrow path. + +The bridge must advertise exact integer support during capability negotiation. A generated bigint wrapper must reject an older bridge before its first call. The new bridge must emit version 2 integer envelopes only when the call opts in. An older client therefore continues to receive the 0.11 safe integer policy. Unknown integer envelopes, mixed versions, oversized strings, and malformed decimal values must fail with a path. Do not guess a conversion. + +Acceptance evidence still needed: a bounded prototype must round trip positive and negative values beyond 64 bits through nested records and arrays. It must test bigint inputs, malformed tags, payload limits, and both mixed-version directions. An independent review must approve the wire grammar, capability rule, generated types, and migration cost before #337 can close. + +## Dataclass return extension for #339 + +This extension is a design sketch. Dataclass inputs remain unsupported and need a separate input contract. Users must pass a plain record through an explicitly typed function if they need that behavior now. + +For an annotated dataclass return, compile a structural TypeScript record from `dataclasses.fields`. Include each instance field in the output, including fields with defaults or `init=False`. A default does not make a returned field optional. A field annotated `Optional[T]` permits `null`. Compose nested dataclasses, records, and sequences only when every field has a supported value contract. Reject ordinary class instances and unsupported fields with their result paths. + +Use a version 2 dataclass envelope with a declared type identity and a `fields` record. For example, `Point(1, 2)` would carry `{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must check the type identity, exact field names, field values, and required fields. It must reject extra fields unless the compiled contract allows them. The Python encoder must use the current bounded value walk. It must detect cycles and enforce depth, node, and payload limits without calling `dataclasses.asdict` as a shortcut. + +The bridge must advertise dataclass value support. A generated dataclass wrapper must reject an older bridge before calling it. A new bridge must send the envelope only when the wrapper requests it. This leaves existing value RPC and ordinary class behavior unchanged. + +Acceptance evidence still needed: a generated `Promise` wrapper must return the exact plain record and reject missing, extra, or wrong typed fields. Tests must cover optional and defaulted fields, nested values, cycles, unsupported fields, depth, and payload limits. An independent review must approve the type identity, version rule, and migration cost before #339 can close. diff --git a/docs/maintainers/value-contracts.v1.json b/docs/maintainers/value-contracts.v2.json similarity index 61% rename from docs/maintainers/value-contracts.v1.json rename to docs/maintainers/value-contracts.v2.json index 1fb06e44..db1a9e35 100644 --- a/docs/maintainers/value-contracts.v1.json +++ b/docs/maintainers/value-contracts.v2.json @@ -1,5 +1,5 @@ { - "revision": 1, + "revision": 2, "status": "frozen-internal-prototype", "transport": "tywrap/1 value RPC", "codecVersion": 1, @@ -20,11 +20,26 @@ "failure": "reject NaN and Infinity" }, "sequence": { - "logicalKind": "list or tuple of contracted values", + "logicalKind": "list or variadic tuple of one contracted item kind", "wire": "JSON array", "decodedAs": "JavaScript array", "composition": "apply each item rule at its result path" }, + "tuple": { + "logicalKind": "fixed-length heterogeneous tuple of contracted values", + "wire": "JSON array", + "decodedAs": "JavaScript array with one contracted value per position", + "composition": "require exact length and apply each item rule at its result index; an empty tuple is valid" + }, + "union": { + "logicalKind": "one value matching a nonempty set of contracted alternatives", + "wire": "the selected alternative's existing wire encoding", + "decodedAs": "the selected alternative's JavaScript value", + "minimumAlternatives": 2, + "maximumAlternatives": 32, + "composition": "require at least one matching alternative; reject a contract if the same wire value can match alternatives with different decoded representations; order has no semantic effect", + "failure": "reject an empty or one-option union during contract construction; reject a value that matches no alternative with its path" + }, "record": { "logicalKind": "string-keyed record of contracted values", "wire": "JSON object", @@ -46,6 +61,12 @@ } }, "prototypeBoundary": "Only float16 scientific values use the new ndarray contract. Existing Arrow int64 values may decode as bigint and keep their current rules.", + "contractLimits": { + "maximumDepth": 64, + "maximumUnionAlternatives": 32, + "namedRecordAliases": "inline nonrecursive aliases within the depth limit", + "recursiveAliases": "unsupported until a reviewed reference-graph contract exists" + }, "compatibility": { "wireChange": false, "unknownVersion": "reject", diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts index 524a0dce..52c9351c 100644 --- a/src/contracts/value-contract.ts +++ b/src/contracts/value-contract.ts @@ -1,5 +1,5 @@ /** Internal value policy consumed by codecs and callable-contract generation. */ -export const VALUE_CONTRACT_REVISION = 1 as const; +export const VALUE_CONTRACT_REVISION = 2 as const; /** JSON numbers can carry Python integers exactly only within this range. */ export const MAX_SAFE_JSON_INTEGER = 2 ** 53 - 1; @@ -26,6 +26,18 @@ export type ValueContract = readonly decodedAs: 'array'; readonly item: ValueContract; } + | { + readonly kind: 'tuple'; + readonly wire: 'json'; + readonly decodedAs: 'array'; + readonly items: readonly ValueContract[]; + } + | { + readonly kind: 'union'; + readonly wire: 'selected-option'; + readonly decodedAs: 'selected-option'; + readonly options: readonly [ValueContract, ValueContract, ...ValueContract[]]; + } | { readonly kind: 'record'; readonly wire: 'json'; From f300b71afc5ee95f0eb87a3b40c39f4551b59458 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:40:47 -0700 Subject: [PATCH 05/50] feat(codegen): route callable contracts through generation --- docs/maintainers/callable-contracts.md | 18 +- src/core/callable-compiler.ts | 417 ++++++++++++++++++------- src/core/generator.ts | 144 ++++++++- src/core/ir-contract.ts | 62 +++- src/runtime/index.ts | 3 + src/runtime/validators.ts | 101 +++++- src/types/index.ts | 20 ++ src/tywrap.ts | 200 ++++++------ test/callable-compiler.test.ts | 125 +++++++- test/generator.test.ts | 55 ++++ test/ir-contract-validation.test.ts | 29 ++ test/runtime_return_validation.test.ts | 15 + 12 files changed, 949 insertions(+), 240 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 93047c74..15cc309d 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -6,37 +6,41 @@ The compiler does not read files. It does not start Python. It does not read a cache. It does not write files. It does not access a runtime registry. -`generate()` owns Python discovery, cache reads, output comparison, and writes. It validates IR before it maps annotations. It also preserves the current generated-file behavior while the value-contract prototype lacks union and reference nodes. +`generate()` owns Python discovery, cache reads, output comparison, and writes. It validates IR and maps annotations, then calls `compileContract()` before emission. Offline contract files omit extractor metadata for stable bytes. The reader restores an empty metadata object. ## Resolved callable data -The compiler records each parameter, result, and overload result with its logical Python type, direction, source path, and conversion result. It uses this record to create one resolved module model. The generator then uses that model for declarations, call wrappers, and return validators. +The compiler records each parameter, result, overload parameter, and overload result with its logical Python type, direction, source path, and conversion result. It creates one resolved module model for declarations, call wrappers, and return validators. -Each value conversion uses revision 1 of [value-contracts.v1.json](value-contracts.v1.json). Do not infer a wire rule from the TypeScript type. +Each value conversion uses revision 2 of [value-contracts.v2.json](value-contracts.v2.json). Do not infer a wire rule from the TypeScript type. | Case | Result | | --- | --- | | `int` | JSON number, JavaScript number, safe-integer constraint | | nested `dict[str, T]` | JSON record with the child conversion at every value path | +| `str | int` | Selected union alternative with one child contract per option | +| `tuple[int, str]` | Exact JSON array with a contract for each index | | float16 ndarray | Existing Arrow or JSON-fallback envelope and finite number result | | float16 Torch tensor | Existing Torch envelope containing the float16 ndarray conversion | +The compiler requires an explicit float16 dtype. An untyped ndarray or Torch tensor stays unresolved. + The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. ## Diagnostics and fallback -Known unsupported cases create error diagnostics. The compiler changes their generated result to `unknown`. +Known unsupported cases create error diagnostics. The compiler changes the affected generated type to `unknown`. `generate()` returns each diagnostic as a warning, so `--fail-on-warn` rejects strict builds. - Coroutines require the capability owned by #338. - Dataclasses require an adapter owned by #339. - Non-string record keys require an explicit conversion. -Unresolved annotations create warning diagnostics. They retain the generator's current explicit fallback. The compiler does not silently claim a supported transport. +Unresolved annotations create warning diagnostics. They retain the generator's current fallback and do not claim a supported conversion. -IR overload signatures stay separate from the implementation signature. Generated declarations retain each input-to-result relation. The wrapper keeps the implementation signature for Python argument binding and runtime calls. +IR overload signatures stay separate from the implementation signature. Generated declarations retain each input-to-result relation. At runtime, the wrapper selects a result validator when exactly one supported overload matches the bound arguments. Ambiguous calls use the implementation validator. ## Deferred work -Revision 1 does not describe union, reference, or heterogeneous tuple values. Do not route the compiler projection through default generation until that policy is explicit. +Revision 2 does not describe references or recursive aliases. The compiler reports these cases as unresolved. A later value-contract revision can give them a supported conversion. The client-binding work may consume the resolved callable model later. It must remain opt-in. It must not add files to default generation or mutate the global runtime registry. diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 6c72519a..5b8b3bd5 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -17,7 +17,7 @@ import type { PythonType, } from '../types/index.js'; import type { CodeGenerator } from './generator.js'; -import type { ValidatedIrContract } from './ir-contract.js'; +import type { IrFunction, ValidatedIrContract } from './ir-contract.js'; export type CallableDirection = 'input' | 'output'; @@ -39,6 +39,8 @@ export interface ValueConversionRequest { direction: CallableDirection; logicalType: PythonType; path: string; + /** Internal recursion counter. Revision 2 rejects contracts deeper than 64 nodes. */ + depth?: number; } export interface SupportedValueResolution { @@ -79,7 +81,8 @@ export interface CallableCompilationDiagnostic { | 'conversion-unsupported' | 'capability-unavailable' | 'coroutine-unsupported' - | 'dataclass-unsupported'; + | 'dataclass-unsupported' + | 'overload-ambiguous'; path: string; message: string; } @@ -96,6 +99,7 @@ export interface ResolvedCallable { path: string; parameters: readonly ResolvedCallableValue[]; result: ResolvedCallableValue; + overloadParameters: readonly (readonly ResolvedCallableValue[])[]; overloadResults: readonly ResolvedCallableValue[]; requiredCapabilities: readonly CallableCapability[]; } @@ -191,7 +195,12 @@ function resolveSequence( request: ValueConversionRequest, conversion: ValueConversionDescription ): ValueResolution { - const child = conversion.resolve({ ...request, logicalType: item, path: `${request.path}[]` }); + const child = conversion.resolve({ + ...request, + logicalType: item, + path: `${request.path}[]`, + depth: (request.depth ?? 0) + 1, + }); if (child.status !== 'supported') { return child; } @@ -206,7 +215,12 @@ function resolveRecord( request: ValueConversionRequest, conversion: ValueConversionDescription ): ValueResolution { - const child = conversion.resolve({ ...request, logicalType: valueType, path: `${request.path}{value}` }); + const child = conversion.resolve({ + ...request, + logicalType: valueType, + path: `${request.path}{value}`, + depth: (request.depth ?? 0) + 1, + }); if (child.status !== 'supported') { return child; } @@ -231,15 +245,54 @@ function resolveTuple( return { status: 'supported', value: { - kind: 'sequence', + kind: 'tuple', wire: 'json', decodedAs: 'array', - item: { kind: 'null', wire: 'json', decodedAs: 'null' }, + items: [], }, }; } + if (items.length === 2 && leafName(items[1]!) === '...') { + return resolveSequence(items[0]!, request, conversion); + } const entries = items.map((item, index) => - conversion.resolve({ ...request, logicalType: item, path: `${request.path}[${index}]` }) + conversion.resolve({ + ...request, + logicalType: item, + path: `${request.path}[${index}]`, + depth: (request.depth ?? 0) + 1, + }) + ); + const childFailure = firstUnresolvedChild(entries); + if (childFailure) { + return childFailure; + } + const values = entries as SupportedValueResolution[]; + return { + status: 'supported', + value: { kind: 'tuple', wire: 'json', decodedAs: 'array', items: values.map(entry => entry.value) }, + }; +} + +function resolveUnion( + options: readonly PythonType[], + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + if (options.length < 2 || options.length > 32) { + return { + status: 'unsupported', + reason: 'Revision 2 accepts unions with two to 32 alternatives.', + guidance: 'Split a larger union or provide an explicit tagged record adapter.', + }; + } + const entries = options.map((option, index) => + conversion.resolve({ + ...request, + logicalType: option, + path: `${request.path}.options[${index}]`, + depth: (request.depth ?? 0) + 1, + }) ); const childFailure = firstUnresolvedChild(entries); if (childFailure) { @@ -247,16 +300,22 @@ function resolveTuple( } const values = entries as SupportedValueResolution[]; const first = values[0]?.value; - if (!first || values.some(entry => JSON.stringify(entry.value) !== JSON.stringify(first))) { + const second = values[1]?.value; + if (!first || !second) { return { status: 'unsupported', - reason: 'The frozen value contract does not represent a heterogeneous tuple.', - guidance: 'Return a TypedDict or a homogeneous list until tuple conversion is specified.', + reason: 'Revision 2 requires at least two supported union alternatives.', + guidance: 'Provide two explicit value alternatives.', }; } return { status: 'supported', - value: { kind: 'sequence', wire: 'json', decodedAs: 'array', item: first }, + value: { + kind: 'union', + wire: 'selected-option', + decodedAs: 'selected-option', + options: [first, second, ...values.slice(2).map(entry => entry.value)], + }, }; } @@ -264,10 +323,10 @@ function resolveNdarray(type: PythonType): ValueResolution { const typeArgument = type.kind === 'generic' ? type.typeArgs[0] : undefined; const dtype = typeArgument ? leafName(typeArgument) : undefined; const normalizedDtype = dtype?.toLowerCase(); - if (normalizedDtype && normalizedDtype !== 'float16') { + if (normalizedDtype !== 'float16') { return { status: 'unresolved', - annotation: `${annotationName(type)}[${dtype}]`, + annotation: annotationName(type), }; } return { @@ -287,6 +346,13 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { revision: VALUE_CONTRACT_REVISION, resolve(request): ValueResolution { const { logicalType: type } = request; + if ((request.depth ?? 0) > 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } switch (type.kind) { case 'primitive': if (type.name === 'None') { @@ -328,6 +394,14 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { case 'final': case 'classvar': return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.type }); + case 'optional': + return resolveUnion( + [type.type, { kind: 'primitive', name: 'None' }], + request, + DEFAULT_VALUE_CONVERSION + ); + case 'union': + return resolveUnion(type.types, request, DEFAULT_VALUE_CONVERSION); case 'collection': if (type.name === 'dict') { const key = type.itemTypes[0]; @@ -353,6 +427,26 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { if (leaf === 'NDArray' || leaf === 'ndarray') { return resolveNdarray(type); } + if (leaf === 'Tensor' && type.module?.startsWith('torch')) { + const ndarray = resolveNdarray({ + kind: 'generic', + name: 'ndarray', + module: 'numpy', + typeArgs: type.typeArgs, + }); + if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { + return ndarray; + } + return { + status: 'supported', + value: { + kind: 'torch-float16', + wire: 'ndarray-envelope', + decodedAs: 'tensor-record', + value: ndarray.value, + }, + }; + } if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { return resolveSequence(type.typeArgs[0] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); } @@ -374,21 +468,6 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { if (leaf === 'ndarray' || leaf === 'NDArray') { return resolveNdarray(type); } - if (leaf === 'Tensor' && type.module?.startsWith('torch')) { - const ndarray = resolveNdarray({ kind: 'custom', name: 'ndarray', module: 'numpy' }); - if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { - return ndarray; - } - return { - status: 'supported', - value: { - kind: 'torch-float16', - wire: 'ndarray-envelope', - decodedAs: 'tensor-record', - value: ndarray.value, - }, - }; - } return { status: 'unresolved', annotation: annotationName(type) }; } default: @@ -398,14 +477,35 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { }; function capabilityNamesFor(value: ValueContract): CallableCapability[] { - const names: CallableCapability[] = ['value-rpc', 'return-validation']; - if (value.kind === 'ndarray-float16') { - names.push('scientific-ndarray'); - } - if (value.kind === 'torch-float16') { - names.push('scientific-torch', 'scientific-ndarray'); - } - return names; + const names = new Set(['value-rpc', 'return-validation']); + const collect = (current: ValueContract): void => { + switch (current.kind) { + case 'sequence': + collect(current.item); + break; + case 'tuple': + current.items.forEach(collect); + break; + case 'union': + current.options.forEach(collect); + break; + case 'record': + current.fields.forEach(field => collect(field.value)); + if (current.additionalValues) { + collect(current.additionalValues); + } + break; + case 'ndarray-float16': + names.add('scientific-ndarray'); + break; + case 'torch-float16': + names.add('scientific-torch'); + names.add('scientific-ndarray'); + break; + } + }; + collect(value); + return [...names]; } function containsNamedType(type: PythonType, names: ReadonlySet): boolean { @@ -459,6 +559,47 @@ function diagnosticForResolution( }; } +function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { + if (left.kind === 'union') { + return left.options.some(option => valuesMayOverlap(option, right)); + } + if (right.kind === 'union') { + return right.options.some(option => valuesMayOverlap(left, option)); + } + if ( + (left.kind === 'integer' && right.kind === 'float') || + (left.kind === 'float' && right.kind === 'integer') + ) { + return true; + } + if (left.kind !== right.kind) { + return false; + } + if (left.kind === 'tuple' && right.kind === 'tuple') { + return left.items.length === right.items.length && + left.items.every((item, index) => valuesMayOverlap(item, right.items[index]!)); + } + if (left.kind === 'sequence' && right.kind === 'sequence') { + return valuesMayOverlap(left.item, right.item); + } + return true; +} + +function overloadsMayOverlap( + left: readonly ResolvedCallableValue[], + right: readonly ResolvedCallableValue[] +): boolean { + if (left.length !== right.length) { + return false; + } + return left.every((value, index) => { + const other = right[index]; + return value.resolution.status !== 'supported' || + other?.resolution.status !== 'supported' || + valuesMayOverlap(value.resolution.value, other.resolution.value); + }); +} + function resolveCallable( func: PythonFunction, path: string, @@ -467,94 +608,103 @@ function resolveCallable( dataclassNames: ReadonlySet, diagnostics: CallableCompilationDiagnostic[] ): { function: PythonFunction; callable: ResolvedCallable } { - const parameters = func.parameters.map((parameter, index) => ({ - direction: 'input' as const, - path: `${path}.parameters[${index}]`, - logicalType: parameter.type, - resolution: conversion.resolve({ - direction: 'input', - logicalType: parameter.type, - path: `${path}.parameters[${index}]`, - }), - })); - const result: ResolvedCallableValue = { - direction: 'output', - path: `${path}.returns`, - logicalType: func.returnType, - resolution: conversion.resolve({ direction: 'output', logicalType: func.returnType, path: `${path}.returns` }), - }; - const overloadResults = (func.overloads ?? []).map((overload, index) => ({ - direction: 'output' as const, - path: `${path}.overloads[${index}].returns`, - logicalType: overload.returnType, - resolution: conversion.resolve({ - direction: 'output', - logicalType: overload.returnType, - path: `${path}.overloads[${index}].returns`, - }), - })); + const resolveValue = ( + direction: CallableDirection, + logicalType: PythonType, + valuePath: string + ): ResolvedCallableValue => ({ + direction, + path: valuePath, + logicalType, + resolution: conversion.resolve({ direction, logicalType, path: valuePath }), + }); + const parameters = func.parameters.map((parameter, index) => + resolveValue('input', parameter.type, `${path}.parameters[${index}]`) + ); + const result = resolveValue('output', func.returnType, `${path}.returns`); + const overloadParameters = (func.overloads ?? []).map((overload, overloadIndex) => + overload.parameters.map((parameter, parameterIndex) => + resolveValue( + 'input', + parameter.type, + `${path}.overloads[${overloadIndex}].parameters[${parameterIndex}]` + ) + ) + ); + const overloadResults = (func.overloads ?? []).map((overload, index) => + resolveValue('output', overload.returnType, `${path}.overloads[${index}].returns`) + ); + const values = [...parameters, result, ...overloadParameters.flat(), ...overloadResults]; const requiredCapabilities = new Set(); - for (const value of [...parameters, result, ...overloadResults]) { + for (const value of values) { const diagnostic = diagnosticForResolution(value.resolution, value.path); if (diagnostic) { diagnostics.push(diagnostic); } - if (value.resolution.status === 'supported') { - capabilityNamesFor(value.resolution.value).forEach(capability => requiredCapabilities.add(capability)); + } + for (let index = 0; index < overloadParameters.length; index += 1) { + for (let earlier = 0; earlier < index; earlier += 1) { + if (overloadsMayOverlap(overloadParameters[earlier]!, overloadParameters[index]!)) { + diagnostics.push({ + severity: 'warning', + code: 'overload-ambiguous', + path: `${path}.overloads[${index}]`, + message: `${path}.overloads[${index}]: input values may also match overload ${earlier}. Runtime return validation uses the implementation result for ambiguous calls.`, + }); + } } } - - const forceUnsupportedOutput = ( - resolution: ValueResolution, - logicalType: PythonType, - outputPath: string - ): ValueResolution => { - if (containsNamedType(logicalType, dataclassNames)) { + for (const value of values) { + if (containsNamedType(value.logicalType, dataclassNames)) { diagnostics.push({ severity: 'error', code: 'dataclass-unsupported', - path: outputPath, - message: `${outputPath}: a dataclass annotation has no value adapter. Use a TypedDict or an explicit record adapter until #339.`, + path: value.path, + message: `${value.path}: a dataclass annotation has no value adapter. Use a TypedDict or an explicit record adapter until #339.`, }); - return { + value.resolution = { status: 'unsupported', reason: 'No dataclass value adapter is available.', guidance: 'Use a TypedDict or an explicit record adapter.', }; } - if (func.isAsync && capabilities.get('coroutine-execution')?.available !== true) { + if ( + value.direction === 'output' && + func.isAsync && + capabilities.get('coroutine-execution')?.available !== true + ) { diagnostics.push({ severity: 'error', code: 'coroutine-unsupported', - path: outputPath, - message: `${outputPath}: this callable requires coroutine execution, which #338 has not implemented.`, + path: value.path, + message: `${value.path}: this callable requires coroutine execution, which #338 has not implemented.`, }); - return { + value.resolution = { status: 'unsupported', reason: 'Coroutine execution is unavailable.', guidance: 'Expose a synchronous value-returning adapter until #338 lands.', }; } - return resolution; - }; - - result.resolution = forceUnsupportedOutput(result.resolution, result.logicalType, result.path); - overloadResults.forEach(overload => { - overload.resolution = forceUnsupportedOutput( - overload.resolution, - overload.logicalType, - overload.path - ); - }); - for (const capability of requiredCapabilities) { - const description = capabilities.get(capability); - if (description?.available !== true) { - diagnostics.push({ - severity: 'error', - code: 'capability-unavailable', - path, - message: `${path}: ${capability} is required. ${description?.guidance ?? 'Add a declared capability before generating this callable.'}`, - }); + if (value.resolution.status === 'supported') { + const needed = capabilityNamesFor(value.resolution.value); + needed.forEach(capability => requiredCapabilities.add(capability)); + const missing = needed.filter(capability => capabilities.get(capability)?.available !== true); + if (missing.length > 0) { + missing.forEach(capability => { + const description = capabilities.get(capability); + diagnostics.push({ + severity: 'error', + code: 'capability-unavailable', + path: value.path, + message: `${value.path}: ${capability} is required. ${description?.guidance ?? 'Add a declared capability before generating this callable.'}`, + }); + }); + value.resolution = { + status: 'unsupported', + reason: `Required capability is unavailable: ${missing.join(', ')}.`, + guidance: 'Provide the capability or use a supported value adapter.', + }; + } } } @@ -565,9 +715,18 @@ function resolveCallable( })); const resolvedOverloads = (func.overloads ?? []).map((overload, index) => ({ ...overload, + parameters: overload.parameters.map((parameter, parameterIndex) => ({ + ...parameter, + type: + overloadParameters[index]?.[parameterIndex]?.resolution.status === 'unsupported' + ? UNKNOWN_TYPE + : parameter.type, + })), returnType: overloadResults[index]?.resolution.status === 'unsupported' ? UNKNOWN_TYPE : overload.returnType, })); + const supportedValue = (value: ResolvedCallableValue): ValueContract | undefined => + value.resolution.status === 'supported' ? value.resolution.value : undefined; const resolvedFunction: PythonFunction = { ...func, parameters: resolvedParameters, @@ -578,6 +737,14 @@ function resolveCallable( }, returnType: result.resolution.status === 'unsupported' ? UNKNOWN_TYPE : func.returnType, overloads: resolvedOverloads, + callableContract: { + parameterValues: parameters.map(supportedValue), + returnValue: supportedValue(result), + overloads: overloadResults.map((overload, index) => ({ + parameterValues: overloadParameters[index]!.map(supportedValue), + returnValue: supportedValue(overload), + })), + }, }; return { function: resolvedFunction, @@ -586,6 +753,7 @@ function resolveCallable( path, parameters, result, + overloadParameters, overloadResults, requiredCapabilities: [...requiredCapabilities].sort(), }, @@ -600,9 +768,39 @@ export function compileContract( ir: ValidatedIrContract, options: CompileContractOptions ): CompiledContract { + const matchesFunction = (source: IrFunction, mapped: PythonFunction): boolean => + source.name === mapped.name && + source.parameters.length === mapped.parameters.length && + source.parameters.every((parameter, index) => parameter.name === mapped.parameters[index]?.name) && + source.overloads.length === (mapped.overloads?.length ?? 0) && + source.overloads.every((overload, index) => + overload.parameters.length === mapped.overloads?.[index]?.parameters.length && + overload.parameters.every((parameter, parameterIndex) => + parameter.name === mapped.overloads?.[index]?.parameters[parameterIndex]?.name + ) + ); + if (ir.module !== options.module.name) { + throw new Error(`Compiled module ${options.module.name} does not match IR module ${ir.module}.`); + } + for (const func of options.module.functions) { + const source = ir.functions.find(entry => entry.name === func.name); + if (!source || !matchesFunction(source, func)) { + throw new Error(`Compiled function ${func.name} does not match validated IR.`); + } + } + for (const cls of options.module.classes) { + const source = ir.classes.find(entry => entry.name === cls.name); + if (!source || cls.methods.some(method => + !source.methods.some(entry => + matchesFunction(entry, method) + ) + )) { + throw new Error(`Compiled class ${cls.name} does not match validated IR.`); + } + } const capabilities = new Map(options.capabilities.map(capability => [capability.name, capability])); const dataclassNames = new Set( - options.module.classes.filter(cls => cls.kind === 'dataclass').map(cls => cls.name) + ir.classes.filter(cls => cls.is_dataclass).map(cls => cls.name) ); const diagnostics: CallableCompilationDiagnostic[] = []; const callables: ResolvedCallable[] = []; @@ -620,14 +818,21 @@ export function compileContract( }; const module: PythonModule = { ...options.module, - functions: options.module.functions.map((func, index) => - compileFunction(func, `$.functions[${index}]`) + functions: options.module.functions.map(func => + compileFunction(func, `$.functions[${ir.functions.findIndex(entry => entry.name === func.name)}]`) ), - classes: options.module.classes.map((cls, classIndex) => ({ + classes: options.module.classes.map(cls => ({ ...cls, - methods: cls.methods.map((method, methodIndex) => - compileFunction(method, `$.classes[${classIndex}].methods[${methodIndex}]`) - ), + methods: cls.methods.map(method => { + const sourceClassIndex = ir.classes.findIndex(entry => entry.name === cls.name); + const sourceMethodIndex = ir.classes[sourceClassIndex]!.methods.findIndex( + entry => entry.name === method.name + ); + return compileFunction( + method, + `$.classes[${sourceClassIndex}].methods[${sourceMethodIndex}]` + ); + }), })), }; const generated = options.generator.generateModuleDefinition(module, options.annotatedJSDoc); diff --git a/src/core/generator.ts b/src/core/generator.ts index 412be8d1..36de767d 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -13,6 +13,8 @@ import type { GeneratedCode, TypescriptType, } from '../types/index.js'; +import type { ValueContract } from '../contracts/value-contract.js'; +import type { OverloadReturnSchema, ReturnSchema } from '../runtime/validators.js'; import { globalCache } from '../utils/cache.js'; import { @@ -39,6 +41,46 @@ interface GenericRenderContext { type ReturnDefinitionNames = ReadonlySet; +function valueContractToReturnSchema(value: ValueContract): ReturnSchema { + switch (value.kind) { + case 'null': + return { kind: 'primitive', type: 'null' }; + case 'boolean': + return { kind: 'primitive', type: 'boolean' }; + case 'integer': + return { kind: 'primitive', type: 'number', constraint: value.constraint }; + case 'float': + return { kind: 'primitive', type: 'number', constraint: value.constraint }; + case 'string': + return { kind: 'primitive', type: 'string' }; + case 'sequence': + return { kind: 'array', element: valueContractToReturnSchema(value.item) }; + case 'tuple': + return { kind: 'tuple', elements: value.items.map(valueContractToReturnSchema) }; + case 'union': + return { kind: 'union', options: value.options.map(valueContractToReturnSchema) }; + case 'record': + return { + kind: 'record', + fields: Object.fromEntries( + value.fields.map(field => [ + field.name, + { schema: valueContractToReturnSchema(field.value), optional: !field.required }, + ]) + ), + values: value.additionalValues + ? valueContractToReturnSchema(value.additionalValues) + : undefined, + }; + case 'ndarray-float16': + return { kind: 'marker', marker: 'ndarray', dtype: value.dtype, dims: value.rank }; + case 'torch-float16': + return { kind: 'marker', marker: 'torch.tensor', dtype: value.value.dtype }; + case 'unsupported': + return { kind: 'any' }; + } +} + export interface CodeGeneratorOptions { /** Reports a generated annotation that cannot be represented by emitted declarations. */ onTypeDegrade?: (typeName: string) => void; @@ -543,6 +585,53 @@ export class CodeGenerator { return schema(type); } + private resolvedReturnSchema( + func: PythonFunction, + definitions: ReturnDefinitionNames + ): ReturnSchema { + const value = func.callableContract?.returnValue; + return value + ? valueContractToReturnSchema(value) + : (this.returnSchema(func.returnType, definitions) as ReturnSchema); + } + + private overloadReturnSchemas( + func: PythonFunction, + definitions: ReturnDefinitionNames + ): OverloadReturnSchema[] { + return (func.overloads ?? []).map((overload, overloadIndex) => { + const contract = func.callableContract?.overloads[overloadIndex]; + return { + parameters: overload.parameters + .map((parameter, parameterIndex) => ({ parameter, parameterIndex })) + .filter(({ parameter }) => parameter.name !== 'self' && parameter.name !== 'cls') + .map(({ parameter, parameterIndex }) => ({ + name: parameter.name, + kind: parameter.varArgs + ? 'var-positional' as const + : parameter.kwArgs + ? 'var-keyword' as const + : parameter.keywordOnly + ? 'keyword-only' as const + : parameter.positionalOnly + ? 'positional-only' as const + : 'positional-or-keyword' as const, + optional: parameter.optional, + value: contract?.parameterValues[parameterIndex] + ? valueContractToReturnSchema(contract.parameterValues[parameterIndex]!) + : (this.returnSchema(parameter.type, definitions) as ReturnSchema), + })), + result: contract?.returnValue + ? valueContractToReturnSchema(contract.returnValue) + : (this.returnSchema(overload.returnType, definitions) as ReturnSchema), + selectable: + contract !== undefined && + contract.returnValue !== undefined && + contract.parameterValues.every(value => value !== undefined), + }; + }); + } + private returnDefinitions(module: PythonModule): ReturnDefinitionNames { return new Set( module.classes @@ -671,7 +760,7 @@ export class CodeGenerator { const fname = this.escapeIdentifier(func.name); const moduleId = moduleName ?? '__main__'; const validatorName = `__validate${this.escapeIdentifier(func.name, { preserveCase: true })}Result`; - const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.returnSchema(func.returnType, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; + const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(func, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; const renderDeclaredOverload = (overload: NonNullable[number]): string[] => { const overloadParams = overload.parameters.filter( @@ -779,7 +868,7 @@ export class CodeGenerator { const requiredKwOnlyNames = keywordOnlyParams.filter(p => !p.optional).map(p => p.name); const keywordOnlyNames = keywordOnlyParams.map(p => p.name); const overloads: string[] = (func.overloads ?? []).flatMap(renderDeclaredOverload); - if (requiredKwOnlyNames.length > 0) { + if ((func.overloads?.length ?? 0) === 0 && requiredKwOnlyNames.length > 0) { // Required keyword-only params must be represented with a required `kwargs` parameter. // Avoid "required after optional" by emitting overloads where all preceding parameters are required. const requiredPosCount = @@ -816,7 +905,12 @@ export class CodeGenerator { ); } } - } else if (firstOptionalIndex >= 0 && !varArgsParam && !needsKwargsParam) { + } else if ( + (func.overloads?.length ?? 0) === 0 && + firstOptionalIndex >= 0 && + !varArgsParam && + !needsKwargsParam + ) { for (let i = firstOptionalIndex; i <= positionalParams.length; i++) { const head = positionalParams.slice(0, i).map(p => renderPositionalParam(p)); const rest: string[] = []; @@ -856,9 +950,13 @@ export class CodeGenerator { const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; const callPreludeLines = emitCallPrelude(callDescriptor, this.callEmitHelpers()); const callPrelude = callPreludeLines.length > 0 ? `${callPreludeLines.join('\n')}\n` : ''; + const overloadValidator = (func.overloads?.length ?? 0) > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(func, returnDefinitions))}, __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n` + : ''; + const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${typeParamDecl}(${paramDecl}): Promise<${returnType}> { -${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}); +${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${returnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); } `; @@ -1062,10 +1160,28 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module ); const mname = this.escapeIdentifier(method.name); const validatorName = `__validate${this.escapeIdentifier(cls.name, { preserveCase: true })}${this.escapeIdentifier(method.name, { preserveCase: true })}Result`; - const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.returnSchema(method.returnType, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; - - const overloads: string[] = []; - if (needsKwargsParam && requiredKwOnlyNames.length > 0) { + const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(method, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; + + const declaredOverloads = (method.overloads?.length ?? 0) > 0 + ? this.generateFunctionWrapper( + { + ...method, + parameters: fparams, + overloads: method.overloads?.map(overload => ({ + ...overload, + parameters: overload.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'), + })), + }, + moduleName, + false, + moduleDeclaredNames, + returnDefinitions + ).declaration.match(/^export function .*;$/gm)?.map(line => + ` static ${line.slice('export function '.length)}` + ) ?? [] + : []; + const overloads: string[] = [...declaredOverloads]; + if (declaredOverloads.length === 0 && needsKwargsParam && requiredKwOnlyNames.length > 0) { const firstOptionalIndex = positionalParams.findIndex(p => p.optional); const requiredPosCount = firstOptionalIndex >= 0 ? firstOptionalIndex : positionalParams.length; @@ -1109,10 +1225,14 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module const guardLines = emitArgGuards(callDescriptor); const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; + const overloadValidator = declaredOverloads.length > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(method, returnDefinitions))}, __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n` + : ''; + const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; - const callExpr = `getRuntimeBridge().call<${returnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName})`; + const callExpr = `getRuntimeBridge().call<${returnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${methodTypeParamDecl}(${paramsDecl}): Promise<${returnType}> { - ${returnValidator}${callPrelude}${guards} return ${callExpr}; + ${returnValidator}${callPrelude}${guards}${overloadValidator} return ${callExpr}; }`); methodDeclarations.push( `${overloadDecl}${overloads.length > 0 ? '' : ` ${staticPrefix}${mname}${methodTypeParamDecl}(${paramsDecl}): Promise<${returnType}>;\n`}` @@ -1228,7 +1348,7 @@ ${migrationNote}${declarationMethodsSection} }); const needsRuntime = module.functions.length > 0 || hasRuntimeClasses; const bridgeDecl = needsRuntime - ? `import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` + ? `import { createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` : ''; const ts = `${`${header}${bridgeDecl}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; @@ -1263,7 +1383,7 @@ ${migrationNote}${declarationMethodsSection} declaration, sourceMap: undefined, metadata: { - generatedAt: new Date(), + generatedAt: new Date(0), sourceFiles: [], runtime: 'auto', optimizations: [], diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts index d914aac2..9b48af36 100644 --- a/src/core/ir-contract.ts +++ b/src/core/ir-contract.ts @@ -31,10 +31,10 @@ export interface IrParameter { export interface IrTypeParameter { name: string; - kind: string; + kind: 'typevar' | 'paramspec' | 'typevartuple'; bound: string | null; constraints: string[] | null; - variance: string | null; + variance: 'covariant' | 'contravariant' | 'invariant' | null; } export interface IrOverload { @@ -111,26 +111,59 @@ export interface IrDiagnostic { message: string; } +export interface ValidateIrContractOptions { + /** + * Generated offline contracts omit extractor metadata so their bytes stay + * stable. The reader restores an empty metadata object for that format. + */ + allowOmittedMetadata?: boolean; +} + export type IrValidationResult = | { ok: true; contract: ValidatedIrContract; diagnostics: readonly [] } | { ok: false; contract: null; diagnostics: readonly IrDiagnostic[] }; class ValidationState { readonly diagnostics: IrDiagnostic[] = []; + private overflowed = false; + + private static readonly maxDiagnostics = 100; constructor(private readonly source: string) {} invalid(path: string, message: string): void { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return; + } this.diagnostics.push({ code: 'contract-invalid', path, message: `${this.source} ${message}` }); } versionMismatch(path: string, found: string): void { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return; + } this.diagnostics.push({ code: 'ir-version-mismatch', path, message: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${this.source} declares ${found}. Regenerate the contract with a matching tywrap_ir.`, }); } + + finish(): readonly IrDiagnostic[] { + if (!this.overflowed) { + return this.diagnostics; + } + return [ + ...this.diagnostics, + { + code: 'contract-invalid', + path: '$', + message: `${this.source} has more than ${ValidationState.maxDiagnostics} contract errors.`, + }, + ]; + } } function isRecord(value: unknown): value is Record { @@ -389,11 +422,15 @@ function validateTypeAlias(value: unknown, path: string, state: ValidationState) * The return value is safe to compile without further structural checks. The * caller owns all filesystem, cache, interpreter, and output operations. */ -export function validateIrContract(input: unknown, source = 'IR'): IrValidationResult { +export function validateIrContract( + input: unknown, + source = 'IR', + options: ValidateIrContractOptions = {} +): IrValidationResult { const state = new ValidationState(source); const contract = recordAt(input, '$', state); if (!contract) { - return { ok: false, contract: null, diagnostics: state.diagnostics }; + return { ok: false, contract: null, diagnostics: state.finish() }; } const version = requiredString(contract, 'ir_version', '$', state); @@ -409,14 +446,23 @@ export function validateIrContract(input: unknown, source = 'IR'): IrValidationR constants?.forEach((constant, index) => validateConstant(constant, `$.constants[${index}]`, state)); const aliases = requiredArray(contract, 'type_aliases', '$', state); aliases?.forEach((alias, index) => validateTypeAlias(alias, `$.type_aliases[${index}]`, state)); - const metadata = requiredValue(contract, 'metadata', '$', state); - if (!isRecord(metadata)) { + const metadata = contract.metadata; + if (metadata === undefined && !options.allowOmittedMetadata) { + state.invalid('$.metadata', '$ is missing required field metadata.'); + } else if (metadata !== undefined && !isRecord(metadata)) { state.invalid('$.metadata', '$.metadata must be an object.'); } validateStringArray(requiredValue(contract, 'warnings', '$', state), '$.warnings', state); if (state.diagnostics.length > 0) { - return { ok: false, contract: null, diagnostics: state.diagnostics }; + return { ok: false, contract: null, diagnostics: state.finish() }; } - return { ok: true, contract: input as ValidatedIrContract, diagnostics: [] }; + return { + ok: true, + contract: { + ...contract, + metadata: isRecord(metadata) ? metadata : {}, + } as ValidatedIrContract, + diagnostics: [], + }; } diff --git a/src/runtime/index.ts b/src/runtime/index.ts index c7f335ad..e19a87a7 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -13,7 +13,10 @@ import type { RuntimeExecution } from '../types/index.js'; export { BridgeCodec, type CodecOptions } from './bridge-codec.js'; export { createReturnValidator, + selectOverloadReturnValidator, describeReceivedShape, + type OverloadParameterSchema, + type OverloadReturnSchema, type ReturnSchema, type ReturnValidator, } from './validators.js'; diff --git a/src/runtime/validators.ts b/src/runtime/validators.ts index c3ade3e0..d2db870e 100644 --- a/src/runtime/validators.ts +++ b/src/runtime/validators.ts @@ -24,6 +24,7 @@ export type ReturnSchema = | { kind: 'primitive'; type: 'number' | 'string' | 'boolean' | 'null' | 'undefined' | 'Uint8Array' | 'object'; + constraint?: 'safe-integer' | 'finite'; } | { kind: 'literal'; value: string | number | boolean | null } | { kind: 'array'; element: ReturnSchema } @@ -44,6 +45,21 @@ export type ReturnSchema = export type ReturnValidator = (result: T) => T; +/** One Python parameter used to select a declared overload after call binding. */ +export interface OverloadParameterSchema { + name: string; + kind: 'positional-only' | 'positional-or-keyword' | 'var-positional' | 'keyword-only' | 'var-keyword'; + optional: boolean; + value: ReturnSchema; +} + +/** An overload is selectable only when all of its value rules are supported. */ +export interface OverloadReturnSchema { + parameters: readonly OverloadParameterSchema[]; + result: ReturnSchema; + selectable: boolean; +} + export interface DecodedShapeMetadata { marker: ScientificMarker; dims?: number; @@ -99,7 +115,7 @@ function renderSchema(schema: ReturnSchema): string { case 'any': return 'unknown'; case 'primitive': - return schema.type; + return schema.constraint ?? schema.type; case 'literal': return JSON.stringify(schema.value); case 'array': @@ -157,6 +173,12 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean if (schema.type === 'object') { return isPlainObject(value); } + if (schema.type === 'number' && schema.constraint === 'safe-integer') { + return typeof value === 'number' && Number.isSafeInteger(value); + } + if (schema.type === 'number' && schema.constraint === 'finite') { + return typeof value === 'number' && Number.isFinite(value); + } return typeof value === schema.type; case 'literal': return Object.is(value, schema.value); @@ -227,6 +249,83 @@ export function createReturnValidator( }; } +function matchesOverload( + signature: OverloadReturnSchema, + args: readonly unknown[], + kwargs: Readonly> | undefined +): boolean { + if (!signature.selectable) { + return false; + } + const keywords = kwargs ?? {}; + const usedKeywords = new Set(); + let position = 0; + let acceptsOtherKeywords = false; + const matches = (schema: ReturnSchema, value: unknown): boolean => + check(schema, value, { definitions: {}, pairs: new WeakMap>() }); + + for (const parameter of signature.parameters) { + if (parameter.kind === 'var-positional') { + while (position < args.length) { + if (!matches(parameter.value, args[position])) { + return false; + } + position += 1; + } + continue; + } + if (parameter.kind === 'var-keyword') { + acceptsOtherKeywords = true; + for (const [name, value] of Object.entries(keywords)) { + if (!usedKeywords.has(name) && !matches(parameter.value, value)) { + return false; + } + } + continue; + } + const hasPosition = parameter.kind !== 'keyword-only' && position < args.length; + const hasKeyword = Object.prototype.hasOwnProperty.call(keywords, parameter.name); + if (hasKeyword && parameter.kind === 'positional-only') { + return false; + } + if (hasPosition && hasKeyword) { + return false; + } + if (!hasPosition && !hasKeyword) { + if (!parameter.optional) { + return false; + } + continue; + } + const value = hasPosition ? args[position++] : keywords[parameter.name]; + if (hasKeyword) { + usedKeywords.add(parameter.name); + } + if (!matches(parameter.value, value)) { + return false; + } + } + if (position < args.length) { + return false; + } + return acceptsOtherKeywords || Object.keys(keywords).every(name => usedKeywords.has(name)); +} + +/** Select one supported overload; ambiguous calls use the implementation validator. */ +export function selectOverloadReturnValidator( + overloads: readonly OverloadReturnSchema[], + args: readonly unknown[], + kwargs: Readonly> | undefined, + fallback: ReturnValidator, + callSite: string, + definitions: Readonly> = {} +): ReturnValidator { + const matches = overloads.filter(overload => matchesOverload(overload, args, kwargs)); + return matches.length === 1 + ? createReturnValidator(matches[0]!.result, callSite, definitions) + : fallback; +} + // ═══════════════════════════════════════════════════════════════════════════ // TYPE GUARDS // ═══════════════════════════════════════════════════════════════════════════ diff --git a/src/types/index.ts b/src/types/index.ts index 04b23bb5..0be8b53b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -2,6 +2,8 @@ * Core type definitions for tywrap */ +import type { ValueContract } from '../contracts/value-contract.js'; + export interface PythonModule { name: string; path?: string; @@ -48,6 +50,13 @@ export interface PythonFunction { * generated declarations. */ overloads?: PythonFunctionOverload[]; + /** + * Internal value contracts resolved before code emission. + * + * The compiler sets this field. Parser-created models leave it absent, and + * the generator then uses its legacy schema derivation. + */ + callableContract?: ResolvedCallableContract; /** * Binding of this callable on its owning class. Defaults to `'instance'`. * @see PythonMethodKind @@ -60,6 +69,17 @@ export interface PythonFunctionOverload { returnType: PythonType; } +/** A resolved value contract for a callable signature. */ +export interface ResolvedCallableSignatureContract { + parameterValues: readonly (ValueContract | undefined)[]; + returnValue?: ValueContract; +} + +/** Internal compiler data consumed by generated validators. */ +export interface ResolvedCallableContract extends ResolvedCallableSignatureContract { + overloads: readonly ResolvedCallableSignatureContract[]; +} + /** * A `@property` or `functools.cached_property` exposed on a class. * diff --git a/src/tywrap.ts b/src/tywrap.ts index 7a2a8b5b..d29d3103 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -3,9 +3,19 @@ */ import { CodeGenerator } from './core/generator.js'; +import { + compileContract, + DEFAULT_CALLABLE_CAPABILITIES, + DEFAULT_VALUE_CONVERSION, +} from './core/callable-compiler.js'; import { TYWRAP_IR_VERSION, validateIrContract, + type IrClass, + type IrFunction, + type IrParameter, + type IrTypeAlias, + type IrTypeParameter, type ValidatedIrContract, } from './core/ir-contract.js'; import { TypeMapper } from './core/mapper.js'; @@ -104,8 +114,7 @@ function stableJson(value: unknown): unknown { } function serializeContract(ir: ValidatedIrContract): string { - const contract = { ...ir }; - delete contract.metadata; + const contract = Object.fromEntries(Object.entries(ir).filter(([key]) => key !== 'metadata')); return `${JSON.stringify(stableJson(contract), null, 2)}\n`; } @@ -204,7 +213,17 @@ async function readContractInput( } try { const parsed = JSON.parse(await fsUtils.readFile(inputPath)) as unknown; - const validated = validateIrVersion(parsed, `Contract ${inputPath}`); + const validation = validateIrContract(parsed, `Contract ${inputPath}`, { + allowOmittedMetadata: true, + }); + const validated = validation.ok + ? { ir: validation.contract } + : { + ir: null, + code: validation.diagnostics[0]?.code ?? 'contract-invalid', + error: + validation.diagnostics[0]?.message ?? `Contract ${inputPath} is not a JSON IR object.`, + }; if (!validated.ir) { return { ir: null, error: validated.error, code: validated.code }; } @@ -451,9 +470,19 @@ export async function generate( // Apply module-level export filtering (functions/classes + excludes). filterModuleExports(moduleModel, moduleConfig, moduleKey, warnings); - // Generate module code + // Resolve callable values once before emitting declarations and wrappers. const annotatedJSDoc = Boolean(resolvedOptions.output?.annotatedJSDoc); - const gen = instance.generator.generateModuleDefinition(moduleModel, annotatedJSDoc); + const compiled = compileContract(ir, { + module: moduleModel, + generator: instance.generator, + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + annotatedJSDoc, + }); + const gen = compiled.generated; + for (const diagnostic of compiled.diagnostics) { + warnings.push(diagnostic.message); + } const baseName = moduleModel.name || 'module'; const filesToEmit: Array<{ path: string; content: string }> = [ @@ -633,7 +662,7 @@ function collectModuleTypeVarNames(obj: Record): Set { } function transformIrToTsModel(ir: ValidatedIrContract): TSPythonModule { - const obj: Record = ir; + const obj = ir; const functions = ir.functions; const classes = ir.classes; const aliases = ir.type_aliases; @@ -647,33 +676,23 @@ function transformIrToTsModel(ir: ValidatedIrContract): TSPythonModule { knownTypeVarNames: moduleTypeVarNames, typeParameters, }); - const mapTypeParameters = (value: Record): PythonGenericParameter[] => - Array.isArray(value.type_params) - ? (value.type_params as unknown[]).map(v => { - const param = (v ?? {}) as Record; - return { - name: String(param.name ?? ''), - kind: String(param.kind ?? 'typevar') as PythonGenericParameter['kind'], - bound: param.bound ? parseType(param.bound) : undefined, - constraints: Array.isArray(param.constraints) - ? (param.constraints as unknown[]).map(item => parseType(item)) - : undefined, - variance: - param.variance === 'covariant' || - param.variance === 'contravariant' || - param.variance === 'invariant' - ? param.variance - : undefined, - } satisfies PythonGenericParameter; - }) - : []; + const mapTypeParameters = ( + typeParameters: readonly IrTypeParameter[] + ): PythonGenericParameter[] => + typeParameters.map(param => ({ + name: param.name, + kind: param.kind, + bound: param.bound ? parseType(param.bound) : undefined, + constraints: param.constraints ? param.constraints.map(item => parseType(item)) : undefined, + variance: param.variance ?? undefined, + })); const mapParam = ( - p: Record, + p: IrParameter, typeParameters: readonly PythonGenericParameter[] = [] ): Parameter => ({ - name: String(p.name ?? ''), + name: p.name, type: parseType(p.annotation, typeParameters), - optional: Boolean(p.default), + optional: p.default, varArgs: p.kind === 'VAR_POSITIONAL', kwArgs: p.kind === 'VAR_KEYWORD', positionalOnly: p.kind === 'POSITIONAL_ONLY', @@ -684,122 +703,93 @@ function transformIrToTsModel(ir: ValidatedIrContract): TSPythonModule { value === 'class' || value === 'static' ? value : 'instance'; const mapFunc = ( - f: Record, + f: IrFunction, inheritedTypeParameters: readonly PythonGenericParameter[] = [] ): PythonFunction => { - const localTypeParameters = mapTypeParameters(f); + const localTypeParameters = mapTypeParameters(f.type_params); const annotationTypeParameters = [...inheritedTypeParameters, ...localTypeParameters]; return { name: String(f.name ?? ''), signature: { - parameters: Array.isArray(f.parameters) - ? (f.parameters as unknown[]).map(v => - mapParam((v ?? {}) as Record, annotationTypeParameters) - ) - : [], + parameters: f.parameters.map(parameter => mapParam(parameter, annotationTypeParameters)), returnType: parseType(f.returns, annotationTypeParameters), isAsync: Boolean(f.is_async), isGenerator: Boolean(f.is_generator), }, - docstring: (f.docstring as string | undefined) ?? undefined, + docstring: f.docstring ?? undefined, decorators: [], isAsync: Boolean(f.is_async), isGenerator: Boolean(f.is_generator), typeParameters: [...localTypeParameters], returnType: parseType(f.returns, annotationTypeParameters), - parameters: Array.isArray(f.parameters) - ? (f.parameters as unknown[]).map(v => - mapParam((v ?? {}) as Record, annotationTypeParameters) - ) - : [], - overloads: Array.isArray(f.overloads) - ? (f.overloads as unknown[]).map(overload => { - const signature = (overload ?? {}) as Record; - return { - parameters: Array.isArray(signature.parameters) - ? (signature.parameters as unknown[]).map(value => - mapParam((value ?? {}) as Record, annotationTypeParameters) - ) - : [], - returnType: parseType(signature.returns, annotationTypeParameters), - }; - }) - : [], + parameters: f.parameters.map(parameter => mapParam(parameter, annotationTypeParameters)), + overloads: f.overloads.map(overload => ({ + parameters: overload.parameters.map(parameter => + mapParam(parameter, annotationTypeParameters) + ), + returnType: parseType(overload.returns, annotationTypeParameters), + })), methodKind: mapMethodKind(f.method_kind), }; }; - const mapClass = (c: Record): PythonClass => { - const classTypeParameters = mapTypeParameters(c); + const mapClass = (c: IrClass): PythonClass => { + const classTypeParameters = mapTypeParameters(c.type_params); return { - name: String(c.name ?? ''), - bases: Array.isArray(c.bases) ? (c.bases as string[]) : [], - methods: Array.isArray(c.methods) - ? (c.methods as unknown[]).map(v => - mapFunc((v ?? {}) as Record, classTypeParameters) - ) - : [], - properties: Array.isArray(c.fields) - ? (c.fields as unknown[]).map(v => { - const p = (v ?? {}) as Record; - return { - name: String(p.name ?? ''), - type: parseType(p.annotation, classTypeParameters), - readonly: false, - setter: false, - getter: true, - optional: Boolean(p.default), - }; - }) - : [], - accessors: Array.isArray(c.accessors) - ? (c.accessors as unknown[]).map(v => { - const a = (v ?? {}) as Record; - return { - name: String(a.name ?? ''), - type: parseType(a.returns, classTypeParameters), - docstring: (a.docstring as string | undefined) ?? undefined, - readOnly: typeof a.read_only === 'boolean' ? a.read_only : undefined, - isCached: Boolean(a.is_cached), - }; - }) - : [], - docstring: (c.docstring as string | undefined) ?? undefined, - decorators: (c.typed_dict as boolean) ? ['__typed_dict__'] : [], - kind: (c.typed_dict as boolean) + name: c.name, + bases: c.bases, + methods: c.methods.map(method => mapFunc(method, classTypeParameters)), + properties: c.fields.map(p => ({ + name: p.name, + type: parseType(p.annotation, classTypeParameters), + readonly: false, + setter: false, + getter: true, + optional: p.default, + })), + accessors: c.accessors.map(a => ({ + name: a.name, + type: parseType(a.returns, classTypeParameters), + docstring: a.docstring ?? undefined, + readOnly: a.read_only ?? undefined, + isCached: a.is_cached, + })), + docstring: c.docstring ?? undefined, + decorators: c.typed_dict ? ['__typed_dict__'] : [], + kind: c.typed_dict ? 'typed_dict' - : (c.is_protocol as boolean) + : c.is_protocol ? 'protocol' - : (c.is_namedtuple as boolean) + : c.is_namedtuple ? 'namedtuple' - : (c.is_dataclass as boolean) + : c.is_dataclass ? 'dataclass' - : (c.is_pydantic as boolean) + : c.is_pydantic ? 'pydantic' : 'class', typeParameters: classTypeParameters, }; }; - const mapTypeAlias = (alias: Record): PythonTypeAlias => { - const typeParameters = mapTypeParameters(alias); + const mapTypeAlias = (alias: IrTypeAlias): PythonTypeAlias => { + const typeParameters = mapTypeParameters(alias.type_params); return { - name: String(alias.name ?? ''), + name: alias.name, type: parseType(alias.definition, typeParameters), typeParameters, }; }; const moduleModel: TSPythonModule = { - name: (obj.module as string) ?? 'module', + name: obj.module, path: undefined, version: - typeof (obj.metadata as Record | undefined)?.package_version === 'string' - ? ((obj.metadata as Record | undefined)?.package_version as string) + typeof obj.metadata.package_version === 'string' + ? obj.metadata.package_version : undefined, - functions: functions.map(v => mapFunc((v ?? {}) as Record)), - classes: classes.map(v => mapClass((v ?? {}) as Record)), - typeAliases: aliases.map(v => mapTypeAlias((v ?? {}) as Record)), + functions: functions.map(functionValue => mapFunc(functionValue)), + classes: classes.map(mapClass), + typeAliases: aliases.map(mapTypeAlias), imports: [], exports: [], }; diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 40d4cdb1..13f3cad4 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1,4 +1,8 @@ import { describe, expect, it } from 'vitest'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import ts from 'typescript'; import { compileContract, DEFAULT_CALLABLE_CAPABILITIES, @@ -6,7 +10,9 @@ import { } from '../src/core/callable-compiler.js'; import { CodeGenerator } from '../src/core/generator.js'; import { validateIrContract } from '../src/core/ir-contract.js'; -import type { PythonModule } from '../src/types/index.js'; +import { BridgeValidationError } from '../src/runtime/errors.js'; +import { clearRuntimeBridge, setRuntimeBridge } from 'tywrap/runtime'; +import type { PythonModule, PythonType } from '../src/types/index.js'; const rawIr = { ir_version: '0.4.0', @@ -251,6 +257,123 @@ describe('compileContract', () => { }, }, }); + const selected = compiled.callables.find(callable => callable.name === 'select'); + expect(selected?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'union', + options: [ + { kind: 'string', wire: 'json', decodedAs: 'string' }, + { kind: 'integer', wire: 'json', decodedAs: 'number', constraint: 'safe-integer' }, + ], + }, + }); + expect(compiled.generated.typescript).toContain('selectOverloadReturnValidator('); + expect(compiled.generated.typescript).toContain('"constraint":"safe-integer"'); + }); + + it('rejects a wrong return from the generated wrapper for a selected overload', async () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-overload-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + _functionName: string, + args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = (typeof args[0] === 'string' ? 99 : 4) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + select: (key: string | number) => Promise; + }; + await expect(generated.select('key')).rejects.toThrow(BridgeValidationError); + await expect(generated.select(2)).resolves.toBe(4); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('uses revision 2 for selected unions and fixed heterogeneous tuples', () => { + const union = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + }); + const tuple = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[1].returns', + logicalType: { + kind: 'collection', + name: 'tuple', + itemTypes: [ + { kind: 'primitive', name: 'int' }, + { kind: 'primitive', name: 'str' }, + ], + }, + }); + + expect(union).toMatchObject({ + status: 'supported', + value: { kind: 'union', options: [{ kind: 'string' }, { kind: 'integer' }] }, + }); + expect(tuple).toMatchObject({ + status: 'supported', + value: { kind: 'tuple', items: [{ kind: 'integer' }, { kind: 'string' }] }, + }); + }); + + it('requires dtype evidence before selecting the float16 scientific contract', () => { + const resolve = (logicalType: PythonType) => + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType, + }); + + expect(resolve({ kind: 'custom', name: 'ndarray', module: 'numpy' })).toMatchObject({ + status: 'unresolved', + }); + expect(resolve({ kind: 'custom', name: 'Tensor', module: 'torch' })).toMatchObject({ + status: 'unresolved', + }); + expect(resolve({ + kind: 'generic', + name: 'NDArray', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + })).toMatchObject({ + status: 'supported', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }); }); it('degrades unimplemented dataclass and coroutine outputs with local diagnostics', () => { diff --git a/test/generator.test.ts b/test/generator.test.ts index e0d8a0dd..abcb890d 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -655,6 +655,61 @@ describe('CodeGenerator', () => { expect(code.declaration).toContain('export function getValue(key: number): Promise;'); }); + it('does not expose the implementation signature beside declared overloads', () => { + const key = { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }; + const fallback = { + name: 'fallback', + type: { kind: 'primitive', name: 'int' }, + optional: true, + varArgs: false, + kwArgs: false, + }; + const method = { + name: 'get_value', + signature: { + parameters: [key, fallback], + returnType: { kind: 'primitive', name: 'str' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [key, fallback], + returnType: { kind: 'primitive', name: 'str' }, + methodKind: 'static', + overloads: [{ parameters: [key], returnType: { kind: 'primitive', name: 'str' } }], + } as any; + + const functionCode = gen.generateFunctionWrapper(method, 'fixture'); + expect(functionCode.declaration).toContain( + 'export function getValue(key: string): Promise;' + ); + expect(functionCode.declaration).not.toContain('fallback'); + + const classCode = gen.generateClassWrapper( + { + name: 'Catalog', + bases: [], + methods: [method], + properties: [], + decorators: [], + kind: 'class', + }, + 'fixture' + ); + expect(classCode.declaration).toContain( + 'static getValue(key: string): Promise;' + ); + expect(classCode.declaration).not.toContain('fallback'); + }); + it('models *args as an array parameter when kwargs are present', () => { const code = gen.generateFunctionWrapper( { diff --git a/test/ir-contract-validation.test.ts b/test/ir-contract-validation.test.ts index 984e4667..5337767b 100644 --- a/test/ir-contract-validation.test.ts +++ b/test/ir-contract-validation.test.ts @@ -85,4 +85,33 @@ describe('validateIrContract', () => { ); } }); + + it('reads stable offline contracts without extractor metadata', () => { + const offline = Object.fromEntries( + Object.entries(validContract).filter(([key]) => key !== 'metadata') + ); + const live = validateIrContract(offline, 'live IR'); + expect(live).toMatchObject({ ok: false }); + + const saved = validateIrContract(offline, 'saved contract', { + allowOmittedMetadata: true, + }); + expect(saved.ok).toBe(true); + if (saved.ok) { + expect(saved.contract.metadata).toEqual({}); + } + }); + + it('bounds malformed-contract diagnostics', () => { + const invalid = { + ...validContract, + functions: Array.from({ length: 120 }, () => ({ name: 2 })), + }; + const result = validateIrContract(invalid, 'bad contract'); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics.length).toBeLessThanOrEqual(101); + expect(result.diagnostics.at(-1)?.message).toContain('more than 100 contract errors'); + } + }); }); diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 033bd349..5eba4f59 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -26,6 +26,21 @@ describe('generated return validators', () => { expect(validator(42)).toBe(42); }); + it('checks the value contract number constraints at nested paths', () => { + const validator = createReturnValidator( + { + kind: 'record', + values: { kind: 'primitive', type: 'number', constraint: 'safe-integer' }, + }, + 'fixture.counts' + ); + expect(validator({ count: 42 })).toEqual({ count: 42 }); + expect(() => validator({ count: 1.5 })).toThrow(BridgeValidationError); + expect(() => validator({ count: Number.MAX_SAFE_INTEGER + 1 })).toThrow( + BridgeValidationError + ); + }); + it('checks unions, optionals, tuples, TypedDict records, and no-op schemas', () => { const schema: ReturnSchema = { kind: 'tuple', From 40370ee8bd7f514cf614bd012afa75593955d4d6 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:45:38 -0700 Subject: [PATCH 06/50] fix callable compiler integration regressions --- src/core/callable-compiler.ts | 20 ++++++++++++++++++-- src/core/generator.ts | 16 +++++++++++++--- src/core/ir-contract.ts | 8 ++++++-- test/integration.test.ts | 2 +- test/ir_version_drift.test.ts | 4 ++-- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 5b8b3bd5..3e54d715 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -634,7 +634,23 @@ function resolveCallable( const overloadResults = (func.overloads ?? []).map((overload, index) => resolveValue('output', overload.returnType, `${path}.overloads[${index}].returns`) ); - const values = [...parameters, result, ...overloadParameters.flat(), ...overloadResults]; + // Python binds these receivers before the wrapper sends RPC arguments. + const visibleParameters = parameters.filter( + (_, index) => func.parameters[index]?.name !== 'self' && func.parameters[index]?.name !== 'cls' + ); + const visibleOverloadParameters = overloadParameters.map((values, overloadIndex) => + values.filter( + (_, index) => + func.overloads?.[overloadIndex]?.parameters[index]?.name !== 'self' && + func.overloads?.[overloadIndex]?.parameters[index]?.name !== 'cls' + ) + ); + const values = [ + ...visibleParameters, + result, + ...visibleOverloadParameters.flat(), + ...overloadResults, + ]; const requiredCapabilities = new Set(); for (const value of values) { const diagnostic = diagnosticForResolution(value.resolution, value.path); @@ -644,7 +660,7 @@ function resolveCallable( } for (let index = 0; index < overloadParameters.length; index += 1) { for (let earlier = 0; earlier < index; earlier += 1) { - if (overloadsMayOverlap(overloadParameters[earlier]!, overloadParameters[index]!)) { + if (overloadsMayOverlap(visibleOverloadParameters[earlier]!, visibleOverloadParameters[index]!)) { diagnostics.push({ severity: 'warning', code: 'overload-ambiguous', diff --git a/src/core/generator.ts b/src/core/generator.ts index 36de767d..24ce98ae 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -618,7 +618,7 @@ export class CodeGenerator { : 'positional-or-keyword' as const, optional: parameter.optional, value: contract?.parameterValues[parameterIndex] - ? valueContractToReturnSchema(contract.parameterValues[parameterIndex]!) + ? valueContractToReturnSchema(contract.parameterValues[parameterIndex]) : (this.returnSchema(parameter.type, definitions) as ReturnSchema), })), result: contract?.returnValue @@ -627,7 +627,12 @@ export class CodeGenerator { selectable: contract !== undefined && contract.returnValue !== undefined && - contract.parameterValues.every(value => value !== undefined), + overload.parameters.every( + (parameter, index) => + parameter.name === 'self' || + parameter.name === 'cls' || + contract.parameterValues[index] !== undefined + ), }; }); } @@ -1347,8 +1352,13 @@ ${migrationNote}${declarationMethodsSection} return kind === 'class' && !c.decorators.includes('__typed_dict__'); }); const needsRuntime = module.functions.length > 0 || hasRuntimeClasses; + const hasOverloads = + module.functions.some(func => (func.overloads?.length ?? 0) > 0) || + module.classes.some(cls => + cls.methods.some(method => (method.overloads?.length ?? 0) > 0) + ); const bridgeDecl = needsRuntime - ? `import { createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` + ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` : ''; const ts = `${`${header}${bridgeDecl}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts index 9b48af36..df2b8d3c 100644 --- a/src/core/ir-contract.ts +++ b/src/core/ir-contract.ts @@ -247,12 +247,16 @@ function requiredArray( path: string, state: ValidationState ): unknown[] | null { - const value = requiredValue(object, name, path, state); - if (!Array.isArray(value)) { + if (!(name in object)) { const noun = path === '$' ? `required array field ${name}` : `${path}.${name}`; state.invalid(`${path}.${name}`, `${path} is missing ${noun}.`); return null; } + const value = object[name]; + if (!Array.isArray(value)) { + state.invalid(`${path}.${name}`, `${path}.${name} must be an array.`); + return null; + } return value; } diff --git a/test/integration.test.ts b/test/integration.test.ts index db537357..29e23564 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -232,7 +232,7 @@ describe('IR-only integration', () => { }, 30_000); it('preserves Python overload input-to-return relationships in generated declarations', async () => { - const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-overload-contract-')); + const tempDir = await mkdtemp(join(process.cwd(), 'test', '.tywrap-overload-contract-')); try { const outDir = join(tempDir, 'generated'); const result = await generate({ diff --git a/test/ir_version_drift.test.ts b/test/ir_version_drift.test.ts index a929532b..00f96f21 100644 --- a/test/ir_version_drift.test.ts +++ b/test/ir_version_drift.test.ts @@ -9,7 +9,7 @@ import { describe, expect, it } from 'vitest'; * so this test runs without a Python interpreter and without pulling tywrap.ts' * dependency graph into the test process. The single sources are: * - Python: tywrap_ir/tywrap_ir/__init__.py (IR_VERSION = "...") - * - TypeScript: src/tywrap.ts (const TYWRAP_IR_VERSION = '...') + * - TypeScript: src/core/ir-contract.ts (const TYWRAP_IR_VERSION = '...') */ const repoRoot = process.cwd(); @@ -32,7 +32,7 @@ describe('IR_VERSION drift', () => { 'IR_VERSION' ); const tsVersion = readVersion( - join('src', 'tywrap.ts'), + join('src', 'core', 'ir-contract.ts'), /\bTYWRAP_IR_VERSION\s*=\s*["']([^"']+)["']/, 'TYWRAP_IR_VERSION' ); From c5c02e8e22f0baf41fce4eb1e7b4b81a6365202c Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 00:53:29 -0700 Subject: [PATCH 07/50] refresh menagerie callable contract snapshots --- test/menagerie/__snapshots__/gen.test.ts.snap | 116 +++++++++--------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 36295109..6ea3e405 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -6,29 +6,29 @@ exports[`menagerie generation gate > generates and typechecks fixtures.typing_to // Module: fixtures.typing_torture // DO NOT EDIT MANUALLY -import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime'; +import { createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime'; const __tywrapReturnDefinitions: Record = {"Movie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false}}},"RatedMovie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number"},"optional":true}}}}; -const __validateannotated_echoResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.typing_torture.annotated_echo", __tywrapReturnDefinitions); +const __validateannotated_echoResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.typing_torture.annotated_echo", __tywrapReturnDefinitions); export async function annotatedEcho(value: number): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'annotated_echo', __args, undefined, __validateannotated_echoResult); } -const __validateasync_textResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.typing_torture.async_text", __tywrapReturnDefinitions); +const __validateasync_textResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.async_text", __tywrapReturnDefinitions); -export async function asyncText(): Promise { +export async function asyncText(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args, undefined, __validateasync_textResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args, undefined, __validateasync_textResult); } -const __validatebytes_echoResult = createReturnValidator({"kind":"primitive","type":"Uint8Array"}, "fixtures.typing_torture.bytes_echo", __tywrapReturnDefinitions); +const __validatebytes_echoResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.bytes_echo", __tywrapReturnDefinitions); -export async function bytesEcho(value: Uint8Array): Promise { +export async function bytesEcho(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); } const __validatefrozen_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.frozen_values", __tywrapReturnDefinitions); @@ -52,11 +52,11 @@ export async function genericIdentity(value: T): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'generic_identity', __args, undefined, __validategeneric_identityResult); } -const __validateint_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.int_key_dict", __tywrapReturnDefinitions); +const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.int_key_dict", __tywrapReturnDefinitions); -export async function intKeyDict(value: { [key: number]: string; }): Promise<{ [key: number]: string; }> { +export async function intKeyDict(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call<{ [key: number]: string; }>('fixtures.typing_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); } const __validateiterator_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.iterator_values", __tywrapReturnDefinitions); @@ -73,32 +73,35 @@ export async function literalEcho(value: "red" | "blue" | 7): Promise<"red" | "b return getRuntimeBridge().call<"red" | "blue" | 7>('fixtures.typing_torture', 'literal_echo', __args, undefined, __validateliteral_echoResult); } -const __validatenested_containersResult = createReturnValidator({"kind":"array","element":{"kind":"record","values":{"kind":"array","element":{"kind":"tuple","elements":[{"kind":"primitive","type":"number"},{"kind":"primitive","type":"string"}]}}}}, "fixtures.typing_torture.nested_containers", __tywrapReturnDefinitions); +const __validatenested_containersResult = createReturnValidator({"kind":"array","element":{"kind":"record","fields":{},"values":{"kind":"array","element":{"kind":"tuple","elements":[{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"string"}]}}}}, "fixtures.typing_torture.nested_containers", __tywrapReturnDefinitions); export async function nestedContainers(value: Record[]): Promise[]> { const __args: unknown[] = [value]; return getRuntimeBridge().call[]>('fixtures.typing_torture', 'nested_containers', __args, undefined, __validatenested_containersResult); } -const __validatenone_returnResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.none_return", __tywrapReturnDefinitions); +const __validatenone_returnResult = createReturnValidator({"kind":"primitive","type":"null"}, "fixtures.typing_torture.none_return", __tywrapReturnDefinitions); export async function noneReturn(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.typing_torture', 'none_return', __args, undefined, __validatenone_returnResult); } -const __validateoptional_unionResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"string"},{"kind":"primitive","type":"number"},{"kind":"primitive","type":"null"}]}, "fixtures.typing_torture.optional_union", __tywrapReturnDefinitions); +const __validateoptional_unionResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"string"},{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"null"}]}, "fixtures.typing_torture.optional_union", __tywrapReturnDefinitions); export async function optionalUnion(value: string | number | null): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'optional_union', __args, undefined, __validateoptional_unionResult); } -const __validateoverloadedResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"number"},{"kind":"primitive","type":"string"}]}, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); +const __validateoverloadedResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"string"}]}, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); +export function overloaded(value: number): Promise; +export function overloaded(value: string): Promise; export async function overloaded(value: number | string): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'overloaded', __args, undefined, __validateoverloadedResult); + const __selectedReturnValidator = selectOverloadReturnValidator([{"parameters":[{"name":"value","kind":"positional-or-keyword","optional":false,"value":{"kind":"primitive","type":"number","constraint":"safe-integer"}}],"result":{"kind":"primitive","type":"number","constraint":"safe-integer"},"selectable":true},{"parameters":[{"name":"value","kind":"positional-or-keyword","optional":false,"value":{"kind":"primitive","type":"string"}}],"result":{"kind":"primitive","type":"string"},"selectable":true}], __args, undefined, __validateoverloadedResult, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); + return getRuntimeBridge().call('fixtures.typing_torture', 'overloaded', __args, undefined, __selectedReturnValidator); } const __validateparamspec_applyResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.paramspec_apply", __tywrapReturnDefinitions); @@ -134,18 +137,18 @@ export async function ratedMovie(): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'rated_movie', __args, undefined, __validaterated_movieResult); } -const __validateset_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.typing_torture.set_values", __tywrapReturnDefinitions); +const __validateset_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.typing_torture.set_values", __tywrapReturnDefinitions); export async function setValues(value: number[]): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'set_values', __args, undefined, __validateset_valuesResult); } -const __validatetuple_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.tuple_key_dict", __tywrapReturnDefinitions); +const __validatetuple_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.tuple_key_dict", __tywrapReturnDefinitions); -export async function tupleKeyDict(value: Record): Promise> { +export async function tupleKeyDict(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call>('fixtures.typing_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); } const __validatetyped_dict_echoResult = createReturnValidator({"kind":"ref","name":"Movie"}, "fixtures.typing_torture.typed_dict_echo", __tywrapReturnDefinitions); @@ -189,9 +192,9 @@ export type UserId = number export function annotatedEcho(value: number): Promise; -export function asyncText(): Promise; +export function asyncText(): Promise; -export function bytesEcho(value: Uint8Array): Promise; +export function bytesEcho(value: unknown): Promise; export function frozenValues(value: string[]): Promise; @@ -199,7 +202,7 @@ export function generatorValues(): Promise; export function genericIdentity(value: T): Promise; -export function intKeyDict(value: { [key: number]: string; }): Promise<{ [key: number]: string; }>; +export function intKeyDict(value: unknown): Promise; export function iteratorValues(): Promise; @@ -211,7 +214,8 @@ export function noneReturn(): Promise; export function optionalUnion(value: string | number | null): Promise; -export function overloaded(value: number | string): Promise; +export function overloaded(value: number): Promise; +export function overloaded(value: string): Promise; export function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; @@ -221,7 +225,7 @@ export function ratedMovie(): Promise; export function setValues(value: number[]): Promise; -export function tupleKeyDict(value: Record): Promise>; +export function tupleKeyDict(value: unknown): Promise; export function typedDictEcho(value: Movie): Promise; @@ -256,18 +260,18 @@ import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywr const __tywrapReturnDefinitions: Record = {}; -const __validatebools_and_intsResult = createReturnValidator({"kind":"array","element":{"kind":"union","options":[{"kind":"primitive","type":"boolean"},{"kind":"primitive","type":"number"}]}}, "fixtures.values_torture.bools_and_ints", __tywrapReturnDefinitions); +const __validatebools_and_intsResult = createReturnValidator({"kind":"array","element":{"kind":"union","options":[{"kind":"primitive","type":"boolean"},{"kind":"primitive","type":"number","constraint":"safe-integer"}]}}, "fixtures.values_torture.bools_and_ints", __tywrapReturnDefinitions); export async function boolsAndInts(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'bools_and_ints', __args, undefined, __validatebools_and_intsResult); } -const __validatebytes_echoResult = createReturnValidator({"kind":"primitive","type":"Uint8Array"}, "fixtures.values_torture.bytes_echo", __tywrapReturnDefinitions); +const __validatebytes_echoResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.bytes_echo", __tywrapReturnDefinitions); -export async function bytesEcho(value: Uint8Array): Promise { +export async function bytesEcho(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.values_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); + return getRuntimeBridge().call('fixtures.values_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); } const __validatecomplex_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.complex_value", __tywrapReturnDefinitions); @@ -277,18 +281,18 @@ export async function complexValue(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'complex_value', __args, undefined, __validatecomplex_valueResult); } -const __validatecoroutine_valueResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.values_torture.coroutine_value", __tywrapReturnDefinitions); +const __validatecoroutine_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.coroutine_value", __tywrapReturnDefinitions); -export async function coroutineValue(): Promise { +export async function coroutineValue(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'coroutine_value', __args, undefined, __validatecoroutine_valueResult); + return getRuntimeBridge().call('fixtures.values_torture', 'coroutine_value', __args, undefined, __validatecoroutine_valueResult); } const __validatedataclass_instanceResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.dataclass_instance", __tywrapReturnDefinitions); -export async function dataclassInstance(): Promise { +export async function dataclassInstance(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'dataclass_instance', __args, undefined, __validatedataclass_instanceResult); + return getRuntimeBridge().call('fixtures.values_torture', 'dataclass_instance', __args, undefined, __validatedataclass_instanceResult); } const __validatedecimal_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.decimal_values", __tywrapReturnDefinitions); @@ -305,7 +309,7 @@ export async function deeplyNested(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'deeply_nested', __args, undefined, __validatedeeply_nestedResult); } -const __validateecho_intResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.echo_int", __tywrapReturnDefinitions); +const __validateecho_intResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.echo_int", __tywrapReturnDefinitions); export async function echoInt(value: number): Promise { const __args: unknown[] = [value]; @@ -326,7 +330,7 @@ export async function enumMember(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'enum_member', __args, undefined, __validateenum_memberResult); } -const __validatefinite_float_edgesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.finite_float_edges", __tywrapReturnDefinitions); +const __validatefinite_float_edgesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"finite"}}, "fixtures.values_torture.finite_float_edges", __tywrapReturnDefinitions); export async function finiteFloatEdges(): Promise { const __args: unknown[] = []; @@ -340,56 +344,56 @@ export async function generatorValue(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'generator_value', __args, undefined, __validategenerator_valueResult); } -const __validateint_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.values_torture.int_key_dict", __tywrapReturnDefinitions); +const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.int_key_dict", __tywrapReturnDefinitions); -export async function intKeyDict(): Promise<{ [key: number]: string; }> { +export async function intKeyDict(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call<{ [key: number]: string; }>('fixtures.values_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); + return getRuntimeBridge().call('fixtures.values_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); } -const __validateinteger_boundariesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.integer_boundaries", __tywrapReturnDefinitions); +const __validateinteger_boundariesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.values_torture.integer_boundaries", __tywrapReturnDefinitions); export async function integerBoundaries(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_boundaries', __args, undefined, __validateinteger_boundariesResult); } -const __validateinteger_factorial_30Result = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_factorial_30", __tywrapReturnDefinitions); +const __validateinteger_factorial_30Result = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_factorial_30", __tywrapReturnDefinitions); export async function integerFactorial_30(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_factorial_30', __args, undefined, __validateinteger_factorial_30Result); } -const __validateinteger_first_roundedResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_first_rounded", __tywrapReturnDefinitions); +const __validateinteger_first_roundedResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_first_rounded", __tywrapReturnDefinitions); export async function integerFirstRounded(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_first_rounded', __args, undefined, __validateinteger_first_roundedResult); } -const __validateinteger_first_unsafeResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_first_unsafe", __tywrapReturnDefinitions); +const __validateinteger_first_unsafeResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_first_unsafe", __tywrapReturnDefinitions); export async function integerFirstUnsafe(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_first_unsafe', __args, undefined, __validateinteger_first_unsafeResult); } -const __validateinteger_int64_maxResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_int64_max", __tywrapReturnDefinitions); +const __validateinteger_int64_maxResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_int64_max", __tywrapReturnDefinitions); export async function integerInt64Max(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_int64_max', __args, undefined, __validateinteger_int64_maxResult); } -const __validateinteger_int64_minResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_int64_min", __tywrapReturnDefinitions); +const __validateinteger_int64_minResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_int64_min", __tywrapReturnDefinitions); export async function integerInt64Min(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_int64_min', __args, undefined, __validateinteger_int64_minResult); } -const __validateinteger_safe_maxResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_safe_max", __tywrapReturnDefinitions); +const __validateinteger_safe_maxResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_safe_max", __tywrapReturnDefinitions); export async function integerSafeMax(): Promise { const __args: unknown[] = []; @@ -410,14 +414,14 @@ export async function megabyteText(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'megabyte_text', __args, undefined, __validatemegabyte_textResult); } -const __validateset_and_frozensetResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"array","element":{"kind":"primitive","type":"number"}},{"kind":"array","element":{"kind":"primitive","type":"string"}}]}, "fixtures.values_torture.set_and_frozenset", __tywrapReturnDefinitions); +const __validateset_and_frozensetResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}},{"kind":"array","element":{"kind":"primitive","type":"string"}}]}, "fixtures.values_torture.set_and_frozenset", __tywrapReturnDefinitions); export async function setAndFrozenset(): Promise<[number[], string[]]> { const __args: unknown[] = []; return getRuntimeBridge().call<[number[], string[]]>('fixtures.values_torture', 'set_and_frozenset', __args, undefined, __validateset_and_frozensetResult); } -const __validatespecial_floatsResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.special_floats", __tywrapReturnDefinitions); +const __validatespecial_floatsResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"finite"}}, "fixtures.values_torture.special_floats", __tywrapReturnDefinitions); export function specialFloats(): Promise; export function specialFloats(include?: boolean): Promise; @@ -436,11 +440,11 @@ export async function temporalValues(): Promise> { return getRuntimeBridge().call>('fixtures.values_torture', 'temporal_values', __args, undefined, __validatetemporal_valuesResult); } -const __validatetuple_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.values_torture.tuple_key_dict", __tywrapReturnDefinitions); +const __validatetuple_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.tuple_key_dict", __tywrapReturnDefinitions); -export async function tupleKeyDict(): Promise> { +export async function tupleKeyDict(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); + return getRuntimeBridge().call('fixtures.values_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); } const __validateunicode_textResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.values_torture.unicode_text", __tywrapReturnDefinitions); @@ -476,13 +480,13 @@ export class TrafficLight { export function boolsAndInts(): Promise; -export function bytesEcho(value: Uint8Array): Promise; +export function bytesEcho(value: unknown): Promise; export function complexValue(): Promise; -export function coroutineValue(): Promise; +export function coroutineValue(): Promise; -export function dataclassInstance(): Promise; +export function dataclassInstance(): Promise; export function decimalValues(): Promise; @@ -498,7 +502,7 @@ export function finiteFloatEdges(): Promise; export function generatorValue(): Promise; -export function intKeyDict(): Promise<{ [key: number]: string; }>; +export function intKeyDict(): Promise; export function integerBoundaries(): Promise; @@ -525,7 +529,7 @@ export function specialFloats(include?: boolean): Promise; export function temporalValues(): Promise>; -export function tupleKeyDict(): Promise>; +export function tupleKeyDict(): Promise; export function unicodeText(): Promise; From 8b2bd083b2faa1a64347b7604586cf78504c28e8 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:01:51 -0700 Subject: [PATCH 08/50] harden overload correlation and IR traversal --- src/core/callable-compiler.ts | 32 +++-- src/core/ir-contract.ts | 88 +++++++++----- src/runtime/validators.ts | 8 +- test/callable-compiler.test.ts | 156 +++++++++++++++++++++++++ test/ir-contract-validation.test.ts | 11 ++ test/ir_extraction_advanced.test.ts | 33 ++++++ test/runtime_return_validation.test.ts | 29 +++++ tywrap_ir/tywrap_ir/ir.py | 17 ++- 8 files changed, 331 insertions(+), 43 deletions(-) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 3e54d715..a4bb18c3 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -587,15 +587,25 @@ function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { function overloadsMayOverlap( left: readonly ResolvedCallableValue[], - right: readonly ResolvedCallableValue[] + right: readonly ResolvedCallableValue[], + leftParameters: readonly Parameter[], + rightParameters: readonly Parameter[] ): boolean { - if (left.length !== right.length) { + const countRange = (parameters: readonly Parameter[]): [number, number] => [ + parameters.filter(parameter => !parameter.optional && !parameter.varArgs && !parameter.kwArgs).length, + parameters.some(parameter => parameter.varArgs || parameter.kwArgs) + ? Number.POSITIVE_INFINITY + : parameters.length, + ]; + const [leftMin, leftMax] = countRange(leftParameters); + const [rightMin, rightMax] = countRange(rightParameters); + if (leftMax < rightMin || rightMax < leftMin) { return false; } - return left.every((value, index) => { - const other = right[index]; + return left.slice(0, Math.min(left.length, right.length)).every((value, index) => { + const other = right[index]!; return value.resolution.status !== 'supported' || - other?.resolution.status !== 'supported' || + other.resolution.status !== 'supported' || valuesMayOverlap(value.resolution.value, other.resolution.value); }); } @@ -645,6 +655,9 @@ function resolveCallable( func.overloads?.[overloadIndex]?.parameters[index]?.name !== 'cls' ) ); + const visibleOverloadSignatures = (func.overloads ?? []).map(overload => + overload.parameters.filter(parameter => parameter.name !== 'self' && parameter.name !== 'cls') + ); const values = [ ...visibleParameters, result, @@ -660,12 +673,17 @@ function resolveCallable( } for (let index = 0; index < overloadParameters.length; index += 1) { for (let earlier = 0; earlier < index; earlier += 1) { - if (overloadsMayOverlap(visibleOverloadParameters[earlier]!, visibleOverloadParameters[index]!)) { + if (overloadsMayOverlap( + visibleOverloadParameters[earlier]!, + visibleOverloadParameters[index]!, + visibleOverloadSignatures[earlier]!, + visibleOverloadSignatures[index]! + )) { diagnostics.push({ severity: 'warning', code: 'overload-ambiguous', path: `${path}.overloads[${index}]`, - message: `${path}.overloads[${index}]: input values may also match overload ${earlier}. Runtime return validation uses the implementation result for ambiguous calls.`, + message: `${path}.overloads[${index}]: input values may also match overload ${earlier}. Runtime return validation selects the first declared match.`, }); } } diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts index df2b8d3c..adc80ff5 100644 --- a/src/core/ir-contract.ts +++ b/src/core/ir-contract.ts @@ -126,8 +126,11 @@ export type IrValidationResult = class ValidationState { readonly diagnostics: IrDiagnostic[] = []; private overflowed = false; + private entriesVisited = 0; + private entryBudgetExceeded = false; private static readonly maxDiagnostics = 100; + private static readonly maxEntries = 100_000; constructor(private readonly source: string) {} @@ -151,6 +154,23 @@ class ValidationState { }); } + visit(path: string): boolean { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return false; + } + if (this.entryBudgetExceeded) { + return false; + } + this.entriesVisited += 1; + if (this.entriesVisited > ValidationState.maxEntries) { + this.entryBudgetExceeded = true; + this.invalid(path, `IR exceeds the ${ValidationState.maxEntries} entry validation limit.`); + return false; + } + return true; + } + finish(): readonly IrDiagnostic[] { if (!this.overflowed) { return this.diagnostics; @@ -170,6 +190,21 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } +function visitArray( + values: readonly unknown[], + path: string, + state: ValidationState, + validate: (value: unknown, path: string) => void +): void { + for (let index = 0; index < values.length; index += 1) { + const entryPath = `${path}[${index}]`; + if (!state.visit(entryPath)) { + break; + } + validate(values[index], entryPath); + } +} + function recordAt(value: unknown, path: string, state: ValidationState): Record | null { if (!isRecord(value)) { state.invalid(path, `${path} must be an object.`); @@ -265,9 +300,9 @@ function validateStringArray(value: unknown, path: string, state: ValidationStat state.invalid(path, `${path} must be an array.`); return; } - value.forEach((entry, index) => { + visitArray(value, path, state, (entry, entryPath) => { if (typeof entry !== 'string') { - state.invalid(`${path}[${index}]`, `${path}[${index}] must be a string.`); + state.invalid(entryPath, `${entryPath} must be a string.`); } }); } @@ -321,29 +356,27 @@ function validateFunction(value: unknown, path: string, state: ValidationState): requiredString(callable, 'qualname', path, state); requiredNullableString(callable, 'docstring', path, state); const parameters = requiredArray(callable, 'parameters', path, state); - parameters?.forEach((parameter, index) => validateParameter(parameter, `${path}.parameters[${index}]`, state)); + if (parameters) visitArray(parameters, `${path}.parameters`, state, (parameter, entryPath) => + validateParameter(parameter, entryPath, state)); requiredNullableString(callable, 'returns', path, state); requiredBoolean(callable, 'is_async', path, state); requiredBoolean(callable, 'is_generator', path, state); const typeParameters = requiredArray(callable, 'type_params', path, state); - typeParameters?.forEach((parameter, index) => - validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) - ); + if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); const methodKind = requiredString(callable, 'method_kind', path, state); if (methodKind !== null && !METHOD_KINDS.has(methodKind)) { state.invalid(`${path}.method_kind`, `${path}.method_kind must be instance, class, or static.`); } const overloads = requiredArray(callable, 'overloads', path, state); - overloads?.forEach((overload, index) => { - const overloadPath = `${path}.overloads[${index}]`; + if (overloads) visitArray(overloads, `${path}.overloads`, state, (overload, overloadPath) => { const signature = recordAt(overload, overloadPath, state); if (!signature) { return; } const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); - overloadParameters?.forEach((parameter, parameterIndex) => - validateParameter(parameter, `${overloadPath}.parameters[${parameterIndex}]`, state) - ); + if (overloadParameters) visitArray(overloadParameters, `${overloadPath}.parameters`, state, + (parameter, entryPath) => validateParameter(parameter, entryPath, state)); requiredNullableString(signature, 'returns', overloadPath, state); }); } @@ -373,26 +406,26 @@ function validateClass(value: unknown, path: string, state: ValidationState): vo requiredNullableString(cls, 'docstring', path, state); validateStringArray(requiredValue(cls, 'bases', path, state), `${path}.bases`, state); const methods = requiredArray(cls, 'methods', path, state); - methods?.forEach((method, index) => validateFunction(method, `${path}.methods[${index}]`, state)); + if (methods) visitArray(methods, `${path}.methods`, state, (method, entryPath) => + validateFunction(method, entryPath, state)); requiredBoolean(cls, 'typed_dict', path, state); const total = requiredValue(cls, 'total', path, state); if (total !== null && typeof total !== 'boolean') { state.invalid(`${path}.total`, `${path}.total must be a boolean or null.`); } const fields = requiredArray(cls, 'fields', path, state); - fields?.forEach((field, index) => - validateParameter(field, `${path}.fields[${index}]`, state, FIELD_KINDS) - ); + if (fields) visitArray(fields, `${path}.fields`, state, (field, entryPath) => + validateParameter(field, entryPath, state, FIELD_KINDS)); requiredBoolean(cls, 'is_protocol', path, state); requiredBoolean(cls, 'is_namedtuple', path, state); requiredBoolean(cls, 'is_dataclass', path, state); requiredBoolean(cls, 'is_pydantic', path, state); const typeParameters = requiredArray(cls, 'type_params', path, state); - typeParameters?.forEach((parameter, index) => - validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) - ); + if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); const accessors = requiredArray(cls, 'accessors', path, state); - accessors?.forEach((accessor, index) => validateAccessor(accessor, `${path}.accessors[${index}]`, state)); + if (accessors) visitArray(accessors, `${path}.accessors`, state, (accessor, entryPath) => + validateAccessor(accessor, entryPath, state)); } function validateConstant(value: unknown, path: string, state: ValidationState): void { @@ -415,9 +448,8 @@ function validateTypeAlias(value: unknown, path: string, state: ValidationState) requiredString(alias, 'definition', path, state, true); requiredBoolean(alias, 'is_generic', path, state); const typeParameters = requiredArray(alias, 'type_params', path, state); - typeParameters?.forEach((parameter, index) => - validateTypeParameter(parameter, `${path}.type_params[${index}]`, state) - ); + if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); } /** @@ -443,13 +475,17 @@ export function validateIrContract( } requiredString(contract, 'module', '$', state); const functions = requiredArray(contract, 'functions', '$', state); - functions?.forEach((functionValue, index) => validateFunction(functionValue, `$.functions[${index}]`, state)); + if (functions) visitArray(functions, '$.functions', state, (functionValue, entryPath) => + validateFunction(functionValue, entryPath, state)); const classes = requiredArray(contract, 'classes', '$', state); - classes?.forEach((classValue, index) => validateClass(classValue, `$.classes[${index}]`, state)); + if (classes) visitArray(classes, '$.classes', state, (classValue, entryPath) => + validateClass(classValue, entryPath, state)); const constants = requiredArray(contract, 'constants', '$', state); - constants?.forEach((constant, index) => validateConstant(constant, `$.constants[${index}]`, state)); + if (constants) visitArray(constants, '$.constants', state, (constant, entryPath) => + validateConstant(constant, entryPath, state)); const aliases = requiredArray(contract, 'type_aliases', '$', state); - aliases?.forEach((alias, index) => validateTypeAlias(alias, `$.type_aliases[${index}]`, state)); + if (aliases) visitArray(aliases, '$.type_aliases', state, (alias, entryPath) => + validateTypeAlias(alias, entryPath, state)); const metadata = contract.metadata; if (metadata === undefined && !options.allowOmittedMetadata) { state.invalid('$.metadata', '$ is missing required field metadata.'); diff --git a/src/runtime/validators.ts b/src/runtime/validators.ts index d2db870e..b19d3b69 100644 --- a/src/runtime/validators.ts +++ b/src/runtime/validators.ts @@ -311,7 +311,7 @@ function matchesOverload( return acceptsOtherKeywords || Object.keys(keywords).every(name => usedKeywords.has(name)); } -/** Select one supported overload; ambiguous calls use the implementation validator. */ +/** Select the first supported match, as TypeScript does for declared overloads. */ export function selectOverloadReturnValidator( overloads: readonly OverloadReturnSchema[], args: readonly unknown[], @@ -320,9 +320,9 @@ export function selectOverloadReturnValidator( callSite: string, definitions: Readonly> = {} ): ReturnValidator { - const matches = overloads.filter(overload => matchesOverload(overload, args, kwargs)); - return matches.length === 1 - ? createReturnValidator(matches[0]!.result, callSite, definitions) + const selected = overloads.find(overload => matchesOverload(overload, args, kwargs)); + return selected + ? createReturnValidator(selected.result, callSite, definitions) : fallback; } diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 13f3cad4..428011d4 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -316,6 +316,162 @@ describe('compileContract', () => { } }); + it('warns when an optional parameter makes two overload inputs overlap', () => { + const source = rawIr.functions[0]!; + const key = source.overloads[0]!.parameters[0]!; + const optionalBase = { + name: 'base', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: true, + }; + const ir = validateIrContract({ + ...rawIr, + functions: [{ + ...source, + parameters: [key, optionalBase], + overloads: [ + { parameters: [key], returns: 'str' }, + { parameters: [key, optionalBase], returns: 'int' }, + ], + }], + classes: [], + }, 'optional overload contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const original = moduleModel.functions[0]!; + const keyModel = original.overloads![0]!.parameters[0]!; + const baseModel = { + ...keyModel, + name: 'base', + type: { kind: 'primitive', name: 'int' } as PythonType, + optional: true, + }; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [{ + ...original, + parameters: [keyModel, baseModel], + overloads: [ + { parameters: [keyModel], returnType: { kind: 'primitive', name: 'str' } }, + { parameters: [keyModel, baseModel], returnType: { kind: 'primitive', name: 'int' } }, + ], + }], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: 'overload-ambiguous', path: '$.functions[0].overloads[1]' }), + ])); + }); + + it('selects a class method overload without validating its implicit cls receiver', async () => { + const source = rawIr.functions[0]!; + const receiver = { + name: 'cls', kind: 'POSITIONAL_OR_KEYWORD', annotation: null, default: false, + }; + const stringInput = source.overloads[0]!.parameters[0]!; + const integerInput = source.overloads[1]!.parameters[0]!; + const methodIr = { + ...source, + name: 'convert', + qualname: 'fixture.Converter.convert', + parameters: [receiver, source.parameters[0]!], + method_kind: 'class', + overloads: [ + { parameters: [receiver, stringInput], returns: 'str' }, + { parameters: [receiver, integerInput], returns: 'int' }, + ], + }; + const ir = validateIrContract({ + ...rawIr, + functions: [], + classes: [{ + ...rawIr.classes[0]!, + name: 'Converter', + qualname: 'fixture.Converter', + methods: [methodIr], + fields: [], + is_dataclass: false, + }], + }, 'class overload contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const original = moduleModel.functions[0]!; + const receiverModel = { + ...original.parameters[0]!, + name: 'cls', + type: { kind: 'custom', name: 'Any', module: 'typing' } as PythonType, + }; + const method = { + ...original, + name: 'convert', + methodKind: 'class' as const, + parameters: [receiverModel, original.parameters[0]!], + overloads: original.overloads?.map(overload => ({ + ...overload, + parameters: [receiverModel, ...overload.parameters], + })), + }; + const model: PythonModule = { + ...moduleModel, + functions: [], + classes: [{ + ...moduleModel.classes[0]!, + name: 'Converter', + kind: 'class', + methods: [method], + properties: [], + }], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics.some(item => item.path.endsWith('.parameters[0]'))).toBe(false); + expect(compiled.generated.typescript).toContain('"selectable":true'); + expect(compiled.generated.declaration).toContain('static convert(key: string): Promise;'); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-class-overload-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + _functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = 7 as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + Converter: { convert: (key: string | number) => Promise }; + }; + await expect(generated.Converter.convert('key')).rejects.toThrow(BridgeValidationError); + await expect(generated.Converter.convert(2)).resolves.toBe(7); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('uses revision 2 for selected unions and fixed heterogeneous tuples', () => { const union = DEFAULT_VALUE_CONVERSION.resolve({ direction: 'output', diff --git a/test/ir-contract-validation.test.ts b/test/ir-contract-validation.test.ts index 5337767b..7c165d1c 100644 --- a/test/ir-contract-validation.test.ts +++ b/test/ir-contract-validation.test.ts @@ -114,4 +114,15 @@ describe('validateIrContract', () => { expect(result.diagnostics.at(-1)?.message).toContain('more than 100 contract errors'); } }); + + it('rejects contracts that exceed the shared entry traversal limit', () => { + const oversized = { ...validContract, warnings: Array(100_001).fill('warning') }; + const result = validateIrContract(oversized, 'large contract'); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics.some(diagnostic => + diagnostic.message.includes('100000 entry validation limit') + )).toBe(true); + } + }); }); diff --git a/test/ir_extraction_advanced.test.ts b/test/ir_extraction_advanced.test.ts index a3ddb785..0e2804f5 100644 --- a/test/ir_extraction_advanced.test.ts +++ b/test/ir_extraction_advanced.test.ts @@ -209,6 +209,39 @@ describe('IR Extraction - Complex Fixture Files', () => { }); describe('IR Extraction - Generic Metadata', () => { + it('keeps type parameters declared only in overload signatures', async () => { + const version = await processUtils.exec(PYTHON_EXECUTABLE, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (version.code !== 0) { + return; + } + + createTempModule( + 'overload_only_generic', + `from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def choose(value: T) -> T: ... + +@overload +def choose(value: None) -> None: ... + +def choose(value: object) -> object: + return value +` + ); + + const ir = await extractIR('overload_only_generic'); + const choose = ir.functions.find((item: { name: string }) => item.name === 'choose'); + expect(choose.type_params.map((param: { name: string }) => param.name)).toEqual(['T']); + expect(choose.overloads).toHaveLength(2); + expect(choose.overloads[0].parameters[0].annotation).toMatch(/T$/); + }); + it('extracts ordered type parameters for functions, classes, and type aliases', async () => { if (!(await supportsVariadicTypingFeatures())) { return; diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 5eba4f59..df64f016 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -8,6 +8,7 @@ import { NodeBridge } from '../src/runtime/node.js'; import { createReturnValidator, describeReceivedShape, + selectOverloadReturnValidator, tagDecodedShape, type ReturnSchema, } from '../src/runtime/validators.js'; @@ -41,6 +42,34 @@ describe('generated return validators', () => { ); }); + it('uses declaration order when an optional-arity overload also matches', () => { + const stringValue = { kind: 'primitive', type: 'string' } as const; + const integerValue = { kind: 'primitive', type: 'number', constraint: 'safe-integer' } as const; + const overloads = [ + { + parameters: [ + { name: 'value', kind: 'positional-or-keyword', optional: false, value: stringValue }, + ], + result: stringValue, + selectable: true, + }, + { + parameters: [ + { name: 'value', kind: 'positional-or-keyword', optional: false, value: stringValue }, + { name: 'base', kind: 'positional-or-keyword', optional: true, value: integerValue }, + ], + result: integerValue, + selectable: true, + }, + ] as const; + const fallback = createReturnValidator({ kind: 'any' }, 'fixture.ambiguous'); + const oneArg = selectOverloadReturnValidator(overloads, ['key'], undefined, fallback, 'fixture.ambiguous'); + const twoArgs = selectOverloadReturnValidator(overloads, ['key', 2], undefined, fallback, 'fixture.ambiguous'); + expect(() => oneArg(4)).toThrow(BridgeValidationError); + expect(oneArg('text')).toBe('text'); + expect(twoArgs(4)).toBe(4); + }); + it('checks unions, optionals, tuples, TypedDict records, and no-op schemas', () => { const schema: ReturnSchema = { kind: 'tuple', diff --git a/tywrap_ir/tywrap_ir/ir.py b/tywrap_ir/tywrap_ir/ir.py index 95c4f5bc..d8198719 100644 --- a/tywrap_ir/tywrap_ir/ir.py +++ b/tywrap_ir/tywrap_ir/ir.py @@ -493,11 +493,13 @@ def _extract_function( # Async generators must be marked as generators so callers can distinguish # them from plain coroutines. is_generator = inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) + overloads, overload_annotations = _extract_overloads(obj) type_params = ( _collect_scoped_type_params( obj, *annotations_for_params, returns, + *overload_annotations, inherited_type_params=inherited_type_params, ) if include_type_params @@ -514,7 +516,7 @@ def _extract_function( is_generator=is_generator, type_params=type_params, method_kind=method_kind, - overloads=_extract_overloads(obj), + overloads=overloads, ) @@ -525,8 +527,8 @@ def _extract_function( _overload_warning_recorded = False -def _extract_overloads(obj: Any) -> List[IROverload]: - """Capture @typing.overload signatures for ``obj`` (Python 3.11+). +def _extract_overloads(obj: Any) -> tuple[List[IROverload], List[Any]]: + """Capture overload signatures and their raw annotations (Python 3.11+). On interpreters without ``typing.get_overloads`` (3.10 and earlier) this degrades to an empty list and records a single structured IR warning @@ -541,14 +543,15 @@ def _extract_overloads(obj: Any) -> List[IROverload]: "typing.get_overloads is unavailable (Python < 3.11); " "@overload signatures will not be captured in the IR." ) - return [] + return [], [] try: registered = get_overloads(obj) except Exception: - return [] + return [], [] out: List[IROverload] = [] + annotations: List[Any] = [] for ov in registered or (): try: sig = inspect.signature(ov) @@ -564,6 +567,7 @@ def _extract_overloads(obj: Any) -> List[IROverload]: params: List[IRParam] = [] for pname, p in sig.parameters.items(): ann = hints.get(pname, p.annotation) + annotations.append(ann) params.append( IRParam( name=pname, @@ -573,8 +577,9 @@ def _extract_overloads(obj: Any) -> List[IROverload]: ) ) ret = hints.get("return", sig.return_annotation) + annotations.append(ret) out.append(IROverload(parameters=params, returns=_stringify_annotation(ret))) - return out + return out, annotations def _accessor_return_annotation(getter: Any) -> Optional[str]: From 825a46f9de2ff26c2347c3dfaadf35f0a49c1961 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:05:11 -0700 Subject: [PATCH 09/50] honor Python 3.10 overload fallback in tests --- src/core/ir-contract.ts | 90 ++++++++++++++++++++++++-------------- test/integration.test.ts | 64 +++++++++++++++++++++++++++ test/menagerie/gen.test.ts | 25 ++++++++--- 3 files changed, 141 insertions(+), 38 deletions(-) diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts index adc80ff5..f54b8395 100644 --- a/src/core/ir-contract.ts +++ b/src/core/ir-contract.ts @@ -356,29 +356,37 @@ function validateFunction(value: unknown, path: string, state: ValidationState): requiredString(callable, 'qualname', path, state); requiredNullableString(callable, 'docstring', path, state); const parameters = requiredArray(callable, 'parameters', path, state); - if (parameters) visitArray(parameters, `${path}.parameters`, state, (parameter, entryPath) => - validateParameter(parameter, entryPath, state)); + if (parameters) { + visitArray(parameters, `${path}.parameters`, state, (parameter, entryPath) => + validateParameter(parameter, entryPath, state)); + } requiredNullableString(callable, 'returns', path, state); requiredBoolean(callable, 'is_async', path, state); requiredBoolean(callable, 'is_generator', path, state); const typeParameters = requiredArray(callable, 'type_params', path, state); - if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); + } const methodKind = requiredString(callable, 'method_kind', path, state); if (methodKind !== null && !METHOD_KINDS.has(methodKind)) { state.invalid(`${path}.method_kind`, `${path}.method_kind must be instance, class, or static.`); } const overloads = requiredArray(callable, 'overloads', path, state); - if (overloads) visitArray(overloads, `${path}.overloads`, state, (overload, overloadPath) => { - const signature = recordAt(overload, overloadPath, state); - if (!signature) { - return; - } - const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); - if (overloadParameters) visitArray(overloadParameters, `${overloadPath}.parameters`, state, - (parameter, entryPath) => validateParameter(parameter, entryPath, state)); - requiredNullableString(signature, 'returns', overloadPath, state); - }); + if (overloads) { + visitArray(overloads, `${path}.overloads`, state, (overload, overloadPath) => { + const signature = recordAt(overload, overloadPath, state); + if (!signature) { + return; + } + const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); + if (overloadParameters) { + visitArray(overloadParameters, `${overloadPath}.parameters`, state, + (parameter, entryPath) => validateParameter(parameter, entryPath, state)); + } + requiredNullableString(signature, 'returns', overloadPath, state); + }); + } } function validateAccessor(value: unknown, path: string, state: ValidationState): void { @@ -406,26 +414,34 @@ function validateClass(value: unknown, path: string, state: ValidationState): vo requiredNullableString(cls, 'docstring', path, state); validateStringArray(requiredValue(cls, 'bases', path, state), `${path}.bases`, state); const methods = requiredArray(cls, 'methods', path, state); - if (methods) visitArray(methods, `${path}.methods`, state, (method, entryPath) => - validateFunction(method, entryPath, state)); + if (methods) { + visitArray(methods, `${path}.methods`, state, (method, entryPath) => + validateFunction(method, entryPath, state)); + } requiredBoolean(cls, 'typed_dict', path, state); const total = requiredValue(cls, 'total', path, state); if (total !== null && typeof total !== 'boolean') { state.invalid(`${path}.total`, `${path}.total must be a boolean or null.`); } const fields = requiredArray(cls, 'fields', path, state); - if (fields) visitArray(fields, `${path}.fields`, state, (field, entryPath) => - validateParameter(field, entryPath, state, FIELD_KINDS)); + if (fields) { + visitArray(fields, `${path}.fields`, state, (field, entryPath) => + validateParameter(field, entryPath, state, FIELD_KINDS)); + } requiredBoolean(cls, 'is_protocol', path, state); requiredBoolean(cls, 'is_namedtuple', path, state); requiredBoolean(cls, 'is_dataclass', path, state); requiredBoolean(cls, 'is_pydantic', path, state); const typeParameters = requiredArray(cls, 'type_params', path, state); - if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); + } const accessors = requiredArray(cls, 'accessors', path, state); - if (accessors) visitArray(accessors, `${path}.accessors`, state, (accessor, entryPath) => - validateAccessor(accessor, entryPath, state)); + if (accessors) { + visitArray(accessors, `${path}.accessors`, state, (accessor, entryPath) => + validateAccessor(accessor, entryPath, state)); + } } function validateConstant(value: unknown, path: string, state: ValidationState): void { @@ -448,8 +464,10 @@ function validateTypeAlias(value: unknown, path: string, state: ValidationState) requiredString(alias, 'definition', path, state, true); requiredBoolean(alias, 'is_generic', path, state); const typeParameters = requiredArray(alias, 'type_params', path, state); - if (typeParameters) visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state)); + } } /** @@ -475,17 +493,25 @@ export function validateIrContract( } requiredString(contract, 'module', '$', state); const functions = requiredArray(contract, 'functions', '$', state); - if (functions) visitArray(functions, '$.functions', state, (functionValue, entryPath) => - validateFunction(functionValue, entryPath, state)); + if (functions) { + visitArray(functions, '$.functions', state, (functionValue, entryPath) => + validateFunction(functionValue, entryPath, state)); + } const classes = requiredArray(contract, 'classes', '$', state); - if (classes) visitArray(classes, '$.classes', state, (classValue, entryPath) => - validateClass(classValue, entryPath, state)); + if (classes) { + visitArray(classes, '$.classes', state, (classValue, entryPath) => + validateClass(classValue, entryPath, state)); + } const constants = requiredArray(contract, 'constants', '$', state); - if (constants) visitArray(constants, '$.constants', state, (constant, entryPath) => - validateConstant(constant, entryPath, state)); + if (constants) { + visitArray(constants, '$.constants', state, (constant, entryPath) => + validateConstant(constant, entryPath, state)); + } const aliases = requiredArray(contract, 'type_aliases', '$', state); - if (aliases) visitArray(aliases, '$.type_aliases', state, (alias, entryPath) => - validateTypeAlias(alias, entryPath, state)); + if (aliases) { + visitArray(aliases, '$.type_aliases', state, (alias, entryPath) => + validateTypeAlias(alias, entryPath, state)); + } const metadata = contract.metadata; if (metadata === undefined && !options.allowOmittedMetadata) { state.invalid('$.metadata', '$ is missing required field metadata.'); diff --git a/test/integration.test.ts b/test/integration.test.ts index 29e23564..62b3b0c3 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -232,6 +232,13 @@ describe('IR-only integration', () => { }, 30_000); it('preserves Python overload input-to-return relationships in generated declarations', async () => { + const support = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (support.code !== 0) { + return; + } const tempDir = await mkdtemp(join(process.cwd(), 'test', '.tywrap-overload-contract-')); try { const outDir = join(tempDir, 'generated'); @@ -296,6 +303,63 @@ void wrong; } }, 30_000); + it('preserves type variables declared only by overload signatures', async () => { + const support = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (support.code !== 0) { + return; + } + const tempDir = await mkdtemp(join(process.cwd(), 'test', '.tywrap-generic-overload-')); + try { + const importDir = join(tempDir, 'py'); + await mkdir(importDir, { recursive: true }); + await writeFile(join(importDir, 'generic_overload.py'), `from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def choose(value: T) -> T: ... + +@overload +def choose(value: None) -> None: ... + +def choose(value: object) -> object: + return value +`, 'utf8'); + const outDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { generic_overload: { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: [importDir], + output: { dir: outDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as any); + expect(result.failures).toEqual([]); + const declaration = await fsUtils.readFile(join(outDir, 'generic_overload.generated.d.ts')); + expect(declaration).toContain('export function choose(value: T): Promise;'); + const consumerPath = join(tempDir, 'consumer.ts'); + await writeFile(consumerPath, `import { choose } from './generated/generic_overload.generated.js'; + +const text: Promise = choose('key'); +const integer: Promise = choose(2); +void text; +void integer; +`, 'utf8'); + const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); + const compile = await processUtils.exec(process.execPath, [ + tscPath, '--ignoreConfig', '--noEmit', '--pretty', 'false', '--target', 'ES2022', + '--lib', 'ES2022,DOM,DOM.Iterable', '--module', 'ESNext', '--moduleResolution', + 'bundler', '--skipLibCheck', consumerPath, + ], { cwd: process.cwd(), timeoutMs: 30_000 }); + expect(compile.code).toBe(0); + expect(compile.stderr).toBe(''); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); + it('generates safe generic wrappers and declaration files that typecheck', async () => { if (!(await supportsVariadicTypingFeatures())) { return; diff --git a/test/menagerie/gen.test.ts b/test/menagerie/gen.test.ts index 204495be..405a7d53 100644 --- a/test/menagerie/gen.test.ts +++ b/test/menagerie/gen.test.ts @@ -9,6 +9,14 @@ import { TIER_ONE_MODULES } from './manifest.js'; const defaultPythonPath = getDefaultPythonPath(); const fixtureImportPath = 'test/menagerie'; +async function supportsPythonOverloads(): Promise { + const result = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + return result.code === 0; +} + async function compileGeneratedFile(generatedPath: string): Promise { const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); const compile = await processUtils.exec( @@ -60,12 +68,17 @@ describe('menagerie generation gate', () => { expect(generatedTs).toBeDefined(); expect(generatedDeclaration).toBeDefined(); - await expect( - Promise.all([ - fsUtils.readFile(generatedTs as string), - fsUtils.readFile(generatedDeclaration as string), - ]) - ).resolves.toMatchSnapshot(); + const generated = await Promise.all([ + fsUtils.readFile(generatedTs as string), + fsUtils.readFile(generatedDeclaration as string), + ]); + if (moduleName === 'fixtures.typing_torture' && !(await supportsPythonOverloads())) { + expect(generated[0]).toContain('export async function overloaded(value: number | string)'); + expect(generated[1]).toContain('export function overloaded(value: number | string)'); + await compileGeneratedFile(generatedTs as string); + } else { + expect(generated).toMatchSnapshot(); + } } finally { await rm(tempDir, { recursive: true, force: true }); } From b5a96b6255643c942bcb165d0cc9b49b28372aab Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:11:12 -0700 Subject: [PATCH 10/50] normalize cross-version overload snapshot and verify Torch dtype --- src/core/generator.ts | 2 +- test/callable-compiler.test.ts | 29 +++++++++++++++++++ test/integration.test.ts | 4 +-- test/menagerie/__snapshots__/gen.test.ts.snap | 13 ++------- test/menagerie/gen.test.ts | 29 ++++++++++++++----- 5 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/core/generator.ts b/src/core/generator.ts index 24ce98ae..490cc6e6 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -75,7 +75,7 @@ function valueContractToReturnSchema(value: ValueContract): ReturnSchema { case 'ndarray-float16': return { kind: 'marker', marker: 'ndarray', dtype: value.dtype, dims: value.rank }; case 'torch-float16': - return { kind: 'marker', marker: 'torch.tensor', dtype: value.value.dtype }; + return { kind: 'marker', marker: 'torch.tensor', dtype: `torch.${value.value.dtype}` }; case 'unsupported': return { kind: 'any' }; } diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 428011d4..0d12c3bd 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -532,6 +532,35 @@ describe('compileContract', () => { }); }); + it('matches the outer Torch dtype recorded by the decoder', () => { + const tensorType: PythonType = { + kind: 'generic', + name: 'Tensor', + module: 'torch', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], + }; + const resolution = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', path: '$.functions[0].returns', logicalType: tensorType, + }); + expect(resolution.status).toBe('supported'); + if (resolution.status !== 'supported') { + return; + } + const generated = new CodeGenerator().generateModuleDefinition({ + ...moduleModel, + classes: [], + functions: [{ + ...moduleModel.functions[1]!, + name: 'tensorValue', + returnType: tensorType, + callableContract: { + parameterValues: [], returnValue: resolution.value, overloads: [], + }, + }], + }); + expect(generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); + }); + it('degrades unimplemented dataclass and coroutine outputs with local diagnostics', () => { const validation = validateIrContract(rawIr, 'fixture contract'); expect(validation.ok).toBe(true); diff --git a/test/integration.test.ts b/test/integration.test.ts index 62b3b0c3..ccc8b599 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -224,7 +224,7 @@ describe('IR-only integration', () => { { cwd: process.cwd(), timeoutMs: 30_000 } ); - expect(compile.code).toBe(0); + expect(compile.code, `${compile.stdout}\n${compile.stderr}`).toBe(0); expect(compile.stderr).toBe(''); } finally { await rm(tempDir, { recursive: true, force: true }); @@ -353,7 +353,7 @@ void integer; '--lib', 'ES2022,DOM,DOM.Iterable', '--module', 'ESNext', '--moduleResolution', 'bundler', '--skipLibCheck', consumerPath, ], { cwd: process.cwd(), timeoutMs: 30_000 }); - expect(compile.code).toBe(0); + expect(compile.code, `${compile.stdout}\n${compile.stderr}`).toBe(0); expect(compile.stderr).toBe(''); } finally { await rm(tempDir, { recursive: true, force: true }); diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 6ea3e405..4426809d 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -6,7 +6,7 @@ exports[`menagerie generation gate > generates and typechecks fixtures.typing_to // Module: fixtures.typing_torture // DO NOT EDIT MANUALLY -import { createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime'; +import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime'; const __tywrapReturnDefinitions: Record = {"Movie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false}}},"RatedMovie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number"},"optional":true}}}}; @@ -96,13 +96,7 @@ export async function optionalUnion(value: string | number | null): Promise; -export function overloaded(value: string): Promise; -export async function overloaded(value: number | string): Promise { - const __args: unknown[] = [value]; - const __selectedReturnValidator = selectOverloadReturnValidator([{"parameters":[{"name":"value","kind":"positional-or-keyword","optional":false,"value":{"kind":"primitive","type":"number","constraint":"safe-integer"}}],"result":{"kind":"primitive","type":"number","constraint":"safe-integer"},"selectable":true},{"parameters":[{"name":"value","kind":"positional-or-keyword","optional":false,"value":{"kind":"primitive","type":"string"}}],"result":{"kind":"primitive","type":"string"},"selectable":true}], __args, undefined, __validateoverloadedResult, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); - return getRuntimeBridge().call('fixtures.typing_torture', 'overloaded', __args, undefined, __selectedReturnValidator); -} +/* overload implementation checked separately */ const __validateparamspec_applyResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.paramspec_apply", __tywrapReturnDefinitions); @@ -214,8 +208,7 @@ export function noneReturn(): Promise; export function optionalUnion(value: string | number | null): Promise; -export function overloaded(value: number): Promise; -export function overloaded(value: string): Promise; +/* overload declarations checked separately */ export function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; diff --git a/test/menagerie/gen.test.ts b/test/menagerie/gen.test.ts index 405a7d53..e7e27b3b 100644 --- a/test/menagerie/gen.test.ts +++ b/test/menagerie/gen.test.ts @@ -68,17 +68,32 @@ describe('menagerie generation gate', () => { expect(generatedTs).toBeDefined(); expect(generatedDeclaration).toBeDefined(); - const generated = await Promise.all([ + let generated = await Promise.all([ fsUtils.readFile(generatedTs as string), fsUtils.readFile(generatedDeclaration as string), ]); - if (moduleName === 'fixtures.typing_torture' && !(await supportsPythonOverloads())) { - expect(generated[0]).toContain('export async function overloaded(value: number | string)'); - expect(generated[1]).toContain('export function overloaded(value: number | string)'); - await compileGeneratedFile(generatedTs as string); - } else { - expect(generated).toMatchSnapshot(); + if (moduleName === 'fixtures.typing_torture') { + const supportsOverloads = await supportsPythonOverloads(); + if (supportsOverloads) { + expect(generated[0]).toContain('selectOverloadReturnValidator('); + expect(generated[1]).toContain('export function overloaded(value: number)'); + expect(generated[1]).toContain('export function overloaded(value: string)'); + } else { + expect(generated[0]).toContain('export async function overloaded(value: number | string)'); + expect(generated[1]).toContain('export function overloaded(value: number | string)'); + await compileGeneratedFile(generatedTs as string); + } + generated = [ + generated[0] + .replace('createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge', + 'createReturnValidator, getRuntimeBridge') + .replace(/^(?:export function overloaded[^\n]*\n)*export async function overloaded[^\n]*\n[\s\S]*?^\}/m, + '/* overload implementation checked separately */'), + generated[1].replace(/^export function overloaded[^\n]*(?:\nexport function overloaded[^\n]*)*/m, + '/* overload declarations checked separately */'), + ]; } + expect(generated).toMatchSnapshot(); } finally { await rm(tempDir, { recursive: true, force: true }); } From 8ec57da6026aee76c8c736303517c9fdcb7704fb Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:12:43 -0700 Subject: [PATCH 11/50] feat(codec): define bytes and per-call provenance contracts --- .../value-contract-fixtures.v2.json | 14 ++ docs/maintainers/value-contracts.md | 134 ++++++++++++++---- docs/maintainers/value-contracts.v2.json | 9 ++ src/contracts/value-contract.ts | 2 + src/runtime/decoded-provenance.ts | 59 ++++++++ test/decoded_provenance.test.ts | 35 +++++ 6 files changed, 223 insertions(+), 30 deletions(-) create mode 100644 src/runtime/decoded-provenance.ts create mode 100644 test/decoded_provenance.test.ts diff --git a/docs/maintainers/value-contract-fixtures.v2.json b/docs/maintainers/value-contract-fixtures.v2.json index abe3a5d8..78337545 100644 --- a/docs/maintainers/value-contract-fixtures.v2.json +++ b/docs/maintainers/value-contract-fixtures.v2.json @@ -10,6 +10,20 @@ { "decimal": "9223372036854775807", "outcome": "reject" }, { "decimal": "265252859812191058636308480000000", "outcome": "reject" } ], + "bytesCases": [ + { + "logicalHex": "00ff80", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "AP+A" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "AP+A" }, + "decodedAs": "Uint8Array[0,255,128]" + }, + { + "logicalHex": "", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "" }, + "decodedAs": "Uint8Array[]" + } + ], "binary16Cases": [ { "wordHex": "0000", "outcome": "0" }, { "wordHex": "8000", "outcome": "-0" }, diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index 3229566b..dbc55a31 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -1,38 +1,112 @@ # Value contracts -The frozen internal prototype is [value-contracts.v2.json](value-contracts.v2.json). -That JSON file owns the policy. The TypeScript interface mirrors it and has revision 2. - -Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. Sequences, fixed tuples, records, and unions compose child contracts. A union has two to 32 alternatives. The compiler rejects a union if one wire value could decode two ways. Contract depth stops at 64. Named records can inline within that limit. Recursive aliases stay unsupported until a reviewed reference graph exists. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. - -The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. - -Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow tables. JSON fallback for ndarray and pandas values still rejects unsafe integers. This is a behavior change from silent rounding to a call error. - -The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused tests also compare every finite binary16 storage word with an independent float32 bit reference. They test signed zero with `Object.is`. - -The Python serializer checks plain integers, NumPy scalar integers, and model dumps during its existing value walk. It also checks terminal estimator parameters and Series name metadata before JSON output. A JSON ndarray or sparse matrix checks integer data before it creates an envelope. Sparse metadata checks shapes and indices. The TypeScript decoder converts float16 Arrow storage during extraction, before it reshapes the ndarray. Torch uses that same ndarray path. These checks do not walk Arrow payloads again for schema validation. +The frozen internal prototype is +[value-contracts.v2.json](value-contracts.v2.json). That JSON file owns the +policy. The TypeScript interface mirrors it and has revision 2. + +Each contract names a logical value, its wire encoding, its decoded JavaScript +value, and its constraints. Sequences, fixed tuples, records, and unions compose +child contracts. A union has two to 32 alternatives. The compiler rejects a +union if one wire value could decode two ways. Contract depth stops at 64. Named +records can inline within that limit. Recursive aliases stay unsupported until a +reviewed reference graph exists. The prototype covers float16 ndarrays and Torch +tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 +may decode as `bigint`. Unsupported conversions carry a reason and guidance. + +The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow +envelopes, JSON fallback, and transports keep their shape. The producer rejects +unsafe Python integers before it writes JSON. A generated wrapper that consumes +this interface must keep `number` for Python `int` and add a safe-integer return +constraint. Regenerate wrappers after that compiler change. Do not return +`bigint` under an existing `Promise`. + +The bytes leaf keeps the existing base64 envelopes. Requests use +`__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to +`Uint8Array`. The Torch float16 contract names `torch.float16` on the outer +tensor and `float16` on its nested ndarray. + +Migration for users with large Python integers: convert them to strings in +Python with an explicit return annotation, or keep integer columns in Arrow +tables. JSON fallback for ndarray and pandas values still rejects unsafe +integers. This is a behavior change from silent rounding to a call error. + +The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer +boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused +tests also compare every finite binary16 storage word with an independent +float32 bit reference. They test signed zero with `Object.is`. + +The Python serializer checks plain integers, NumPy scalar integers, and model +dumps during its existing value walk. It also checks terminal estimator +parameters and Series name metadata before JSON output. A JSON ndarray or sparse +matrix checks integer data before it creates an envelope. Sparse metadata checks +shapes and indices. The TypeScript decoder converts float16 Arrow storage during +extraction, before it reshapes the ndarray. Torch uses that same ndarray path. +These checks do not walk Arrow payloads again for schema validation. ## Exact bigint extension for #337 -This extension is a design sketch. The 0.11 policy still rejects unsafe plain Python integers. - -Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map Python `int` to TypeScript `bigint` for both parameters and returns. The return validator must require `typeof value === 'bigint'`, including every record and sequence member. Keep `bool` separate. Calls without the option keep `number` and the safe range. - -Add a version 2 integer value envelope with a canonical signed decimal string. A sample value is `{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. The request encoder and Python request decoder must validate the same grammar and digit limit. The response encoder and TypeScript decoder must do the same. Use `BigInt` only after validation. Keep Arrow int64 columns on their current Arrow path. - -The bridge must advertise exact integer support during capability negotiation. A generated bigint wrapper must reject an older bridge before its first call. The new bridge must emit version 2 integer envelopes only when the call opts in. An older client therefore continues to receive the 0.11 safe integer policy. Unknown integer envelopes, mixed versions, oversized strings, and malformed decimal values must fail with a path. Do not guess a conversion. - -Acceptance evidence still needed: a bounded prototype must round trip positive and negative values beyond 64 bits through nested records and arrays. It must test bigint inputs, malformed tags, payload limits, and both mixed-version directions. An independent review must approve the wire grammar, capability rule, generated types, and migration cost before #337 can close. +This extension is a design sketch. The 0.11 policy still rejects unsafe plain +Python integers. + +Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map +Python `int` to TypeScript `bigint` for both parameters and returns. The return +validator must require `typeof value === 'bigint'`, including every record and +sequence member. Keep `bool` separate. Calls without the option keep `number` +and the safe range. + +Add a version 2 integer value envelope with a canonical signed decimal string. A +sample value is +`{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. +The request encoder and Python request decoder must validate the same grammar +and digit limit. The response encoder and TypeScript decoder must do the same. +Use `BigInt` only after validation. Keep Arrow int64 columns on their current +Arrow path. + +The bridge must advertise exact integer support during capability negotiation. A +generated bigint wrapper must reject an older bridge before its first call. The +new bridge must emit version 2 integer envelopes only when the call opts in. An +older client therefore continues to receive the 0.11 safe integer policy. +Unknown integer envelopes, mixed versions, oversized strings, and malformed +decimal values must fail with a path. Do not guess a conversion. + +Acceptance evidence still needed: a bounded prototype must round trip positive +and negative values beyond 64 bits through nested records and arrays. It must +test bigint inputs, malformed tags, payload limits, and both mixed-version +directions. An independent review must approve the wire grammar, capability +rule, generated types, and migration cost before #337 can close. ## Dataclass return extension for #339 -This extension is a design sketch. Dataclass inputs remain unsupported and need a separate input contract. Users must pass a plain record through an explicitly typed function if they need that behavior now. - -For an annotated dataclass return, compile a structural TypeScript record from `dataclasses.fields`. Include each instance field in the output, including fields with defaults or `init=False`. A default does not make a returned field optional. A field annotated `Optional[T]` permits `null`. Compose nested dataclasses, records, and sequences only when every field has a supported value contract. Reject ordinary class instances and unsupported fields with their result paths. - -Use a version 2 dataclass envelope with a declared type identity and a `fields` record. For example, `Point(1, 2)` would carry `{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must check the type identity, exact field names, field values, and required fields. It must reject extra fields unless the compiled contract allows them. The Python encoder must use the current bounded value walk. It must detect cycles and enforce depth, node, and payload limits without calling `dataclasses.asdict` as a shortcut. - -The bridge must advertise dataclass value support. A generated dataclass wrapper must reject an older bridge before calling it. A new bridge must send the envelope only when the wrapper requests it. This leaves existing value RPC and ordinary class behavior unchanged. - -Acceptance evidence still needed: a generated `Promise` wrapper must return the exact plain record and reject missing, extra, or wrong typed fields. Tests must cover optional and defaulted fields, nested values, cycles, unsupported fields, depth, and payload limits. An independent review must approve the type identity, version rule, and migration cost before #339 can close. +This extension is a design sketch. Dataclass inputs remain unsupported and need +a separate input contract. Users must pass a plain record through an explicitly +typed function if they need that behavior now. + +For an annotated dataclass return, compile a structural TypeScript record from +`dataclasses.fields`. Include each instance field in the output, including +fields with defaults or `init=False`. A default does not make a returned field +optional. A field annotated `Optional[T]` permits `null`. Compose nested +dataclasses, records, and sequences only when every field has a supported value +contract. Reject ordinary class instances and unsupported fields with their +result paths. + +Use a version 2 dataclass envelope with a declared type identity and a `fields` +record. For example, `Point(1, 2)` would carry +`{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. +The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must +check the type identity, exact field names, field values, and required fields. +It must reject extra fields unless the compiled contract allows them. The Python +encoder must use the current bounded value walk. It must detect cycles and +enforce depth, node, and payload limits without calling `dataclasses.asdict` as +a shortcut. + +The bridge must advertise dataclass value support. A generated dataclass wrapper +must reject an older bridge before calling it. A new bridge must send the +envelope only when the wrapper requests it. This leaves existing value RPC and +ordinary class behavior unchanged. + +Acceptance evidence still needed: a generated `Promise` wrapper must +return the exact plain record and reject missing, extra, or wrong typed fields. +Tests must cover optional and defaulted fields, nested values, cycles, +unsupported fields, depth, and payload limits. An independent review must +approve the type identity, version rule, and migration cost before #339 can +close. diff --git a/docs/maintainers/value-contracts.v2.json b/docs/maintainers/value-contracts.v2.json index bfebfa32..827a766c 100644 --- a/docs/maintainers/value-contracts.v2.json +++ b/docs/maintainers/value-contracts.v2.json @@ -19,6 +19,13 @@ "decodedAs": "JavaScript number", "failure": "reject NaN and Infinity" }, + "bytes": { + "logicalKind": "Python bytes or bytearray", + "requestWire": "existing __tywrap_bytes__ base64 envelope", + "responseWire": "existing __type__: bytes base64 envelope", + "decodedAs": "JavaScript Uint8Array", + "failure": "reject malformed base64 or payloads above the existing byte limit" + }, "sequence": { "logicalKind": "list or variadic tuple of one contracted item kind", "wire": "JSON array", @@ -57,6 +64,8 @@ "logicalKind": "Torch float16 tensor", "wire": "codecVersion 1 torch.tensor envelope containing ndarray envelope", "decodedAs": "tensor record with numeric data", + "outerDtype": "torch.float16", + "nestedDtype": "float16", "conversion": "reuse ndarray-float16 rule for nested value" } }, diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts index 52c9351c..d8bc033b 100644 --- a/src/contracts/value-contract.ts +++ b/src/contracts/value-contract.ts @@ -20,6 +20,7 @@ export type ValueContract = readonly constraint: 'finite'; } | { readonly kind: 'string'; readonly wire: 'json'; readonly decodedAs: 'string' } + | { readonly kind: 'bytes'; readonly wire: 'base64-envelope'; readonly decodedAs: 'Uint8Array' } | { readonly kind: 'sequence'; readonly wire: 'json'; @@ -57,6 +58,7 @@ export type ValueContract = readonly kind: 'torch-float16'; readonly wire: 'ndarray-envelope'; readonly decodedAs: 'tensor-record'; + readonly dtype: 'torch.float16'; readonly value: Extract; } | { diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts new file mode 100644 index 00000000..a983406c --- /dev/null +++ b/src/runtime/decoded-provenance.ts @@ -0,0 +1,59 @@ +import type { ScientificMarker } from '../utils/codec.js'; + +/** Metadata from one successfully decoded scientific envelope. */ +export interface DecodedShapeMetadata { + readonly marker: ScientificMarker; + readonly dims?: number; + readonly dtype?: string; +} + +/** Scalar provenance stays with one decoded call, including nested values. */ +export class DecodedProvenance { + private rootScalar?: DecodedShapeMetadata; + private readonly children = new WeakMap>(); + private entries = 0; + private readonly maxEntries: number; + + constructor(maxEntries = 1_000_000) { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + throw new Error('Decoded provenance entry limit must be a positive safe integer'); + } + this.maxEntries = maxEntries; + } + + recordRoot(metadata: DecodedShapeMetadata): void { + if (this.rootScalar !== undefined) { + throw new Error('Duplicate root scalar provenance'); + } + this.claimEntry(); + this.rootScalar = Object.freeze({ ...metadata }); + } + + recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + let entries = this.children.get(parent); + if (!entries) { + entries = new Map(); + this.children.set(parent, entries); + } + if (entries.has(key)) { + throw new Error('Duplicate child scalar provenance'); + } + this.claimEntry(); + entries.set(key, Object.freeze({ ...metadata })); + } + + atRoot(): DecodedShapeMetadata | undefined { + return this.rootScalar; + } + + atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { + return this.children.get(parent)?.get(key); + } + + private claimEntry(): void { + if (this.entries >= this.maxEntries) { + throw new Error(`Decoded scalar provenance exceeds ${this.maxEntries} entries`); + } + this.entries += 1; + } +} diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts new file mode 100644 index 00000000..424890ad --- /dev/null +++ b/test/decoded_provenance.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; + +describe('decoded provenance', () => { + it('keeps equal scalar values separate by call and nested location', () => { + const firstParent = { one: 1.5, two: 1.5 }; + const secondParent = { one: 1.5 }; + const first = new DecodedProvenance(); + const second = new DecodedProvenance(); + + first.recordChild(firstParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + first.recordChild(firstParent, 'two', { marker: 'ndarray', dims: 0, dtype: 'float32' }); + second.recordChild(secondParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float64' }); + + expect(first.atChild(firstParent, 'one')?.dtype).toBe('float16'); + expect(first.atChild(firstParent, 'two')?.dtype).toBe('float32'); + expect(first.atChild(secondParent, 'one')).toBeUndefined(); + expect(second.atChild(firstParent, 'one')).toBeUndefined(); + expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); + }); + + it('has one root slot and a bounded number of entries', () => { + const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + const provenance = new DecodedProvenance(2); + const parent = [1.5]; + provenance.recordRoot(evidence); + provenance.recordChild(parent, 0, evidence); + + expect(provenance.atRoot()).toEqual(evidence); + expect(provenance.atChild(parent, 0)).toEqual(evidence); + expect(() => provenance.recordRoot(evidence)).toThrow(/Duplicate root/); + expect(() => provenance.recordChild(parent, 0, evidence)).toThrow(/Duplicate child/); + expect(() => provenance.recordChild(parent, 1, evidence)).toThrow(/exceeds 2 entries/); + }); +}); From 88ad9de9ec30ad41ce32d6663a3c8baedf0474fb Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:12:43 -0700 Subject: [PATCH 12/50] feat(codec): define bytes and per-call provenance contracts --- .../value-contract-fixtures.v2.json | 14 ++ docs/maintainers/value-contracts.md | 134 ++++++++++++++---- docs/maintainers/value-contracts.v2.json | 9 ++ src/contracts/value-contract.ts | 2 + src/runtime/decoded-provenance.ts | 59 ++++++++ test/decoded_provenance.test.ts | 35 +++++ 6 files changed, 223 insertions(+), 30 deletions(-) create mode 100644 src/runtime/decoded-provenance.ts create mode 100644 test/decoded_provenance.test.ts diff --git a/docs/maintainers/value-contract-fixtures.v2.json b/docs/maintainers/value-contract-fixtures.v2.json index abe3a5d8..78337545 100644 --- a/docs/maintainers/value-contract-fixtures.v2.json +++ b/docs/maintainers/value-contract-fixtures.v2.json @@ -10,6 +10,20 @@ { "decimal": "9223372036854775807", "outcome": "reject" }, { "decimal": "265252859812191058636308480000000", "outcome": "reject" } ], + "bytesCases": [ + { + "logicalHex": "00ff80", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "AP+A" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "AP+A" }, + "decodedAs": "Uint8Array[0,255,128]" + }, + { + "logicalHex": "", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "" }, + "decodedAs": "Uint8Array[]" + } + ], "binary16Cases": [ { "wordHex": "0000", "outcome": "0" }, { "wordHex": "8000", "outcome": "-0" }, diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index dc1ac691..dbc55a31 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -1,38 +1,112 @@ # Value contracts -The frozen internal prototype is [value-contracts.v2.json](value-contracts.v2.json). -That JSON file owns the policy. The TypeScript interface mirrors it and has revision 2. - -Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. Sequences, fixed tuples, records, and unions compose child contracts. A union has two to 32 alternatives. The compiler rejects a union if one wire value could decode two ways. Contract depth stops at 64. Named records can inline within that limit. Recursive aliases stay unsupported until a reviewed reference graph exists. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. - -The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. - -Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow tables. JSON fallback for ndarray and pandas values still rejects unsafe integers. This is a behavior change from silent rounding to a call error. - -The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused tests also compare every finite binary16 storage word with an independent float32 bit reference. They test signed zero with `Object.is`. - -The Python serializer checks plain integers, NumPy scalar integers, and model dumps during its existing value walk. A JSON ndarray or sparse matrix checks integer data before it creates an envelope. The TypeScript decoder converts float16 Arrow storage during extraction, before it reshapes the ndarray. Torch uses that same ndarray path. These checks do not walk the full decoded payload again for schema validation. +The frozen internal prototype is +[value-contracts.v2.json](value-contracts.v2.json). That JSON file owns the +policy. The TypeScript interface mirrors it and has revision 2. + +Each contract names a logical value, its wire encoding, its decoded JavaScript +value, and its constraints. Sequences, fixed tuples, records, and unions compose +child contracts. A union has two to 32 alternatives. The compiler rejects a +union if one wire value could decode two ways. Contract depth stops at 64. Named +records can inline within that limit. Recursive aliases stay unsupported until a +reviewed reference graph exists. The prototype covers float16 ndarrays and Torch +tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 +may decode as `bigint`. Unsupported conversions carry a reason and guidance. + +The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow +envelopes, JSON fallback, and transports keep their shape. The producer rejects +unsafe Python integers before it writes JSON. A generated wrapper that consumes +this interface must keep `number` for Python `int` and add a safe-integer return +constraint. Regenerate wrappers after that compiler change. Do not return +`bigint` under an existing `Promise`. + +The bytes leaf keeps the existing base64 envelopes. Requests use +`__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to +`Uint8Array`. The Torch float16 contract names `torch.float16` on the outer +tensor and `float16` on its nested ndarray. + +Migration for users with large Python integers: convert them to strings in +Python with an explicit return annotation, or keep integer columns in Arrow +tables. JSON fallback for ndarray and pandas values still rejects unsafe +integers. This is a behavior change from silent rounding to a call error. + +The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer +boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused +tests also compare every finite binary16 storage word with an independent +float32 bit reference. They test signed zero with `Object.is`. + +The Python serializer checks plain integers, NumPy scalar integers, and model +dumps during its existing value walk. It also checks terminal estimator +parameters and Series name metadata before JSON output. A JSON ndarray or sparse +matrix checks integer data before it creates an envelope. Sparse metadata checks +shapes and indices. The TypeScript decoder converts float16 Arrow storage during +extraction, before it reshapes the ndarray. Torch uses that same ndarray path. +These checks do not walk Arrow payloads again for schema validation. ## Exact bigint extension for #337 -This extension is a design sketch. The 0.11 policy still rejects unsafe plain Python integers. - -Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map Python `int` to TypeScript `bigint` for both parameters and returns. The return validator must require `typeof value === 'bigint'`, including every record and sequence member. Keep `bool` separate. Calls without the option keep `number` and the safe range. - -Add a version 2 integer value envelope with a canonical signed decimal string. A sample value is `{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. The request encoder and Python request decoder must validate the same grammar and digit limit. The response encoder and TypeScript decoder must do the same. Use `BigInt` only after validation. Keep Arrow int64 columns on their current Arrow path. - -The bridge must advertise exact integer support during capability negotiation. A generated bigint wrapper must reject an older bridge before its first call. The new bridge must emit version 2 integer envelopes only when the call opts in. An older client therefore continues to receive the 0.11 safe integer policy. Unknown integer envelopes, mixed versions, oversized strings, and malformed decimal values must fail with a path. Do not guess a conversion. - -Acceptance evidence still needed: a bounded prototype must round trip positive and negative values beyond 64 bits through nested records and arrays. It must test bigint inputs, malformed tags, payload limits, and both mixed-version directions. An independent review must approve the wire grammar, capability rule, generated types, and migration cost before #337 can close. +This extension is a design sketch. The 0.11 policy still rejects unsafe plain +Python integers. + +Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map +Python `int` to TypeScript `bigint` for both parameters and returns. The return +validator must require `typeof value === 'bigint'`, including every record and +sequence member. Keep `bool` separate. Calls without the option keep `number` +and the safe range. + +Add a version 2 integer value envelope with a canonical signed decimal string. A +sample value is +`{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. +The request encoder and Python request decoder must validate the same grammar +and digit limit. The response encoder and TypeScript decoder must do the same. +Use `BigInt` only after validation. Keep Arrow int64 columns on their current +Arrow path. + +The bridge must advertise exact integer support during capability negotiation. A +generated bigint wrapper must reject an older bridge before its first call. The +new bridge must emit version 2 integer envelopes only when the call opts in. An +older client therefore continues to receive the 0.11 safe integer policy. +Unknown integer envelopes, mixed versions, oversized strings, and malformed +decimal values must fail with a path. Do not guess a conversion. + +Acceptance evidence still needed: a bounded prototype must round trip positive +and negative values beyond 64 bits through nested records and arrays. It must +test bigint inputs, malformed tags, payload limits, and both mixed-version +directions. An independent review must approve the wire grammar, capability +rule, generated types, and migration cost before #337 can close. ## Dataclass return extension for #339 -This extension is a design sketch. Dataclass inputs remain unsupported and need a separate input contract. Users must pass a plain record through an explicitly typed function if they need that behavior now. - -For an annotated dataclass return, compile a structural TypeScript record from `dataclasses.fields`. Include each instance field in the output, including fields with defaults or `init=False`. A default does not make a returned field optional. A field annotated `Optional[T]` permits `null`. Compose nested dataclasses, records, and sequences only when every field has a supported value contract. Reject ordinary class instances and unsupported fields with their result paths. - -Use a version 2 dataclass envelope with a declared type identity and a `fields` record. For example, `Point(1, 2)` would carry `{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must check the type identity, exact field names, field values, and required fields. It must reject extra fields unless the compiled contract allows them. The Python encoder must use the current bounded value walk. It must detect cycles and enforce depth, node, and payload limits without calling `dataclasses.asdict` as a shortcut. - -The bridge must advertise dataclass value support. A generated dataclass wrapper must reject an older bridge before calling it. A new bridge must send the envelope only when the wrapper requests it. This leaves existing value RPC and ordinary class behavior unchanged. - -Acceptance evidence still needed: a generated `Promise` wrapper must return the exact plain record and reject missing, extra, or wrong typed fields. Tests must cover optional and defaulted fields, nested values, cycles, unsupported fields, depth, and payload limits. An independent review must approve the type identity, version rule, and migration cost before #339 can close. +This extension is a design sketch. Dataclass inputs remain unsupported and need +a separate input contract. Users must pass a plain record through an explicitly +typed function if they need that behavior now. + +For an annotated dataclass return, compile a structural TypeScript record from +`dataclasses.fields`. Include each instance field in the output, including +fields with defaults or `init=False`. A default does not make a returned field +optional. A field annotated `Optional[T]` permits `null`. Compose nested +dataclasses, records, and sequences only when every field has a supported value +contract. Reject ordinary class instances and unsupported fields with their +result paths. + +Use a version 2 dataclass envelope with a declared type identity and a `fields` +record. For example, `Point(1, 2)` would carry +`{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. +The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must +check the type identity, exact field names, field values, and required fields. +It must reject extra fields unless the compiled contract allows them. The Python +encoder must use the current bounded value walk. It must detect cycles and +enforce depth, node, and payload limits without calling `dataclasses.asdict` as +a shortcut. + +The bridge must advertise dataclass value support. A generated dataclass wrapper +must reject an older bridge before calling it. A new bridge must send the +envelope only when the wrapper requests it. This leaves existing value RPC and +ordinary class behavior unchanged. + +Acceptance evidence still needed: a generated `Promise` wrapper must +return the exact plain record and reject missing, extra, or wrong typed fields. +Tests must cover optional and defaulted fields, nested values, cycles, +unsupported fields, depth, and payload limits. An independent review must +approve the type identity, version rule, and migration cost before #339 can +close. diff --git a/docs/maintainers/value-contracts.v2.json b/docs/maintainers/value-contracts.v2.json index db1a9e35..bf2b5d5f 100644 --- a/docs/maintainers/value-contracts.v2.json +++ b/docs/maintainers/value-contracts.v2.json @@ -19,6 +19,13 @@ "decodedAs": "JavaScript number", "failure": "reject NaN and Infinity" }, + "bytes": { + "logicalKind": "Python bytes or bytearray", + "requestWire": "existing __tywrap_bytes__ base64 envelope", + "responseWire": "existing __type__: bytes base64 envelope", + "decodedAs": "JavaScript Uint8Array", + "failure": "reject malformed base64 or payloads above the existing byte limit" + }, "sequence": { "logicalKind": "list or variadic tuple of one contracted item kind", "wire": "JSON array", @@ -57,6 +64,8 @@ "logicalKind": "Torch float16 tensor", "wire": "codecVersion 1 torch.tensor envelope containing ndarray envelope", "decodedAs": "tensor record with numeric data", + "outerDtype": "torch.float16", + "nestedDtype": "float16", "conversion": "reuse ndarray-float16 rule for nested value" } }, diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts index 52c9351c..d8bc033b 100644 --- a/src/contracts/value-contract.ts +++ b/src/contracts/value-contract.ts @@ -20,6 +20,7 @@ export type ValueContract = readonly constraint: 'finite'; } | { readonly kind: 'string'; readonly wire: 'json'; readonly decodedAs: 'string' } + | { readonly kind: 'bytes'; readonly wire: 'base64-envelope'; readonly decodedAs: 'Uint8Array' } | { readonly kind: 'sequence'; readonly wire: 'json'; @@ -57,6 +58,7 @@ export type ValueContract = readonly kind: 'torch-float16'; readonly wire: 'ndarray-envelope'; readonly decodedAs: 'tensor-record'; + readonly dtype: 'torch.float16'; readonly value: Extract; } | { diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts new file mode 100644 index 00000000..a983406c --- /dev/null +++ b/src/runtime/decoded-provenance.ts @@ -0,0 +1,59 @@ +import type { ScientificMarker } from '../utils/codec.js'; + +/** Metadata from one successfully decoded scientific envelope. */ +export interface DecodedShapeMetadata { + readonly marker: ScientificMarker; + readonly dims?: number; + readonly dtype?: string; +} + +/** Scalar provenance stays with one decoded call, including nested values. */ +export class DecodedProvenance { + private rootScalar?: DecodedShapeMetadata; + private readonly children = new WeakMap>(); + private entries = 0; + private readonly maxEntries: number; + + constructor(maxEntries = 1_000_000) { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + throw new Error('Decoded provenance entry limit must be a positive safe integer'); + } + this.maxEntries = maxEntries; + } + + recordRoot(metadata: DecodedShapeMetadata): void { + if (this.rootScalar !== undefined) { + throw new Error('Duplicate root scalar provenance'); + } + this.claimEntry(); + this.rootScalar = Object.freeze({ ...metadata }); + } + + recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + let entries = this.children.get(parent); + if (!entries) { + entries = new Map(); + this.children.set(parent, entries); + } + if (entries.has(key)) { + throw new Error('Duplicate child scalar provenance'); + } + this.claimEntry(); + entries.set(key, Object.freeze({ ...metadata })); + } + + atRoot(): DecodedShapeMetadata | undefined { + return this.rootScalar; + } + + atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { + return this.children.get(parent)?.get(key); + } + + private claimEntry(): void { + if (this.entries >= this.maxEntries) { + throw new Error(`Decoded scalar provenance exceeds ${this.maxEntries} entries`); + } + this.entries += 1; + } +} diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts new file mode 100644 index 00000000..424890ad --- /dev/null +++ b/test/decoded_provenance.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; + +describe('decoded provenance', () => { + it('keeps equal scalar values separate by call and nested location', () => { + const firstParent = { one: 1.5, two: 1.5 }; + const secondParent = { one: 1.5 }; + const first = new DecodedProvenance(); + const second = new DecodedProvenance(); + + first.recordChild(firstParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + first.recordChild(firstParent, 'two', { marker: 'ndarray', dims: 0, dtype: 'float32' }); + second.recordChild(secondParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float64' }); + + expect(first.atChild(firstParent, 'one')?.dtype).toBe('float16'); + expect(first.atChild(firstParent, 'two')?.dtype).toBe('float32'); + expect(first.atChild(secondParent, 'one')).toBeUndefined(); + expect(second.atChild(firstParent, 'one')).toBeUndefined(); + expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); + }); + + it('has one root slot and a bounded number of entries', () => { + const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + const provenance = new DecodedProvenance(2); + const parent = [1.5]; + provenance.recordRoot(evidence); + provenance.recordChild(parent, 0, evidence); + + expect(provenance.atRoot()).toEqual(evidence); + expect(provenance.atChild(parent, 0)).toEqual(evidence); + expect(() => provenance.recordRoot(evidence)).toThrow(/Duplicate root/); + expect(() => provenance.recordChild(parent, 0, evidence)).toThrow(/Duplicate child/); + expect(() => provenance.recordChild(parent, 1, evidence)).toThrow(/exceeds 2 entries/); + }); +}); From a2e2773c580d2b0b74ec1e7e80dbd7a66dd62b75 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:17:02 -0700 Subject: [PATCH 13/50] feat(codec): carry scalar ndarray proof per RPC call --- src/runtime/base-bridge.ts | 3 +- src/runtime/bridge-codec.ts | 5 ++- src/runtime/decoded-provenance.ts | 15 ++++++- src/runtime/rpc-client.ts | 11 ++++-- src/types/index.ts | 4 +- src/utils/codec.ts | 42 ++++++++++++++++++-- test/bridge-codec.test.ts | 66 +++++++++++++++++++++++++++++++ test/decoded_provenance.test.ts | 9 +++++ test/rpc-client.test.ts | 65 ++++++++++++++++++++++++++++++ test/value_contracts.test.ts | 40 ++++++++++++++++++- 10 files changed, 247 insertions(+), 13 deletions(-) diff --git a/src/runtime/base-bridge.ts b/src/runtime/base-bridge.ts index 0749a9d2..e3fd049d 100644 --- a/src/runtime/base-bridge.ts +++ b/src/runtime/base-bridge.ts @@ -18,6 +18,7 @@ import type { PythonRuntime, BridgeInfo } from '../types/index.js'; import { DisposableBase } from './bounded-context.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; import type { RpcClient, GetBridgeInfoOptions } from './rpc-client.js'; import type { TransportCapabilities } from './transport.js'; @@ -44,7 +45,7 @@ export abstract class BasePythonBridge extends DisposableBase implements PythonR functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { await this.ensureReady(); return this.getRpcClient().call(module, functionName, args, kwargs, validate); diff --git a/src/runtime/bridge-codec.ts b/src/runtime/bridge-codec.ts index 2edc44bf..f0fc56f8 100644 --- a/src/runtime/bridge-codec.ts +++ b/src/runtime/bridge-codec.ts @@ -16,6 +16,7 @@ import { ScientificDecodeError, } from '../utils/codec.js'; import { PROTOCOL_ID } from './transport.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; // ═══════════════════════════════════════════════════════════════════════════ // TYPES @@ -662,13 +663,13 @@ export class BridgeCodec { * @throws BridgeProtocolError if the RPC response is invalid * @throws BridgeExecutionError if response contains a Python error */ - async decodeResponseAsync(payload: string): Promise { + async decodeResponseAsync(payload: string, provenance?: DecodedProvenance): Promise { const result = this.parseResponseResult(payload); // Decode scientific envelopes in the result. let decoded: unknown; try { - decoded = await decodeScientificValue(result); + decoded = await decodeScientificValue(result, provenance); } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); const decodeError = err instanceof ScientificDecodeError ? err : undefined; diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts index a983406c..945936af 100644 --- a/src/runtime/decoded-provenance.ts +++ b/src/runtime/decoded-provenance.ts @@ -30,6 +30,7 @@ export class DecodedProvenance { } recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + key = this.normalizeKey(parent, key); let entries = this.children.get(parent); if (!entries) { entries = new Map(); @@ -47,7 +48,19 @@ export class DecodedProvenance { } atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { - return this.children.get(parent)?.get(key); + return this.children.get(parent)?.get(this.normalizeKey(parent, key)); + } + + private normalizeKey(parent: object, key: string | number): string | number { + if ( + Array.isArray(parent) && + typeof key === 'string' && + /^(0|[1-9]\d*)$/.test(key) && + Number.isSafeInteger(Number(key)) + ) { + return Number(key); + } + return key; } private claimEntry(): void { diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index 3eee208f..e475d10a 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -23,6 +23,7 @@ import type { BridgeBackend, BridgeInfo } from '../types/index.js'; import { DisposableBase, type ExecuteOptions } from './bounded-context.js'; import { BridgeProtocolError } from './errors.js'; import { BridgeCodec, type CodecOptions } from './bridge-codec.js'; +import { DecodedProvenance } from './decoded-provenance.js'; import { PROTOCOL_ID, TYWRAP_PROTOCOL_VERSION, @@ -431,9 +432,10 @@ export class RpcClient extends DisposableBase { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { - return this.sendMessageAsync( + const provenance = validate ? new DecodedProvenance() : undefined; + return this.sendVia( { method: 'call', params: { @@ -446,11 +448,12 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - validate(result); + validate(result, provenance); return result; }, } - : undefined + : undefined, + responseStr => this.codec.decodeResponseAsync(responseStr, provenance) ); } diff --git a/src/types/index.ts b/src/types/index.ts index c5b0d408..7ff3dd42 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -2,6 +2,8 @@ * Core type definitions for tywrap */ +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; + export interface PythonModule { name: string; path?: string; @@ -503,7 +505,7 @@ export interface PythonRuntime { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise; } diff --git a/src/utils/codec.ts b/src/utils/codec.ts index 01d86601..dec055a9 100644 --- a/src/utils/codec.ts +++ b/src/utils/codec.ts @@ -10,6 +10,7 @@ */ import { tagDecodedShape } from '../runtime/validators.js'; +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; import { isSafeJsonInteger, MAX_SAFE_JSON_INTEGER } from '../contracts/value-contract.js'; const SCIENTIFIC_MARKERS = [ @@ -1334,7 +1335,8 @@ function decodeEnvelope( async function decodeEnvelopeAsync( value: unknown, - decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise + decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise, + provenance?: DecodedProvenance ): Promise { let visitedNodes = 0; @@ -1397,6 +1399,34 @@ async function decodeEnvelopeAsync( } }; + const recordScalarProvenance = ( + original: unknown, + decoded: unknown, + target?: DecodeTarget + ): void => { + if ( + !provenance || + typeof decoded !== 'number' || + !isPlainObject(original) || + original.__tywrap__ !== 'ndarray' || + !isStrictV1Envelope(original) || + !Array.isArray(original.shape) || + original.shape.length !== 0 + ) { + return; + } + const metadata = { + marker: 'ndarray' as const, + dims: 0, + dtype: typeof original.dtype === 'string' ? original.dtype : undefined, + }; + if (target) { + provenance.recordChild(target.container, target.key, metadata); + } else { + provenance.recordRoot(metadata); + } + }; + const settleDecoded = async ( decoded: PromiseLike, original: unknown, @@ -1409,6 +1439,7 @@ async function decodeEnvelopeAsync( } catch (error) { return prefixHandlerError(error, path); } + recordScalarProvenance(original, resolved, path === 'result' ? undefined : target); assignDecoded(target, original, resolved); }; @@ -1455,8 +1486,10 @@ async function decodeEnvelopeAsync( pending.push(settleDecoded(decoded, current, { container: root, key: 'value' }, path)); } } else if (target) { + recordScalarProvenance(current, decoded, target); assignDecoded(target, current, decoded); } else if (decoded !== current) { + recordScalarProvenance(current, decoded); root.value = decoded; } return; @@ -1530,9 +1563,12 @@ async function decodeEnvelopeAsync( /** * Decode values produced by the Python bridge. */ -export async function decodeValueAsync(value: unknown): Promise { +export async function decodeValueAsync( + value: unknown, + provenance?: DecodedProvenance +): Promise { try { - return await decodeEnvelopeAsync(value, tryDecodeArrowTable); + return await decodeEnvelopeAsync(value, tryDecodeArrowTable, provenance); } catch (error) { throw asScientificDecodeError(error, 'unknown'); } diff --git a/test/bridge-codec.test.ts b/test/bridge-codec.test.ts index fe83668f..fd38e01a 100644 --- a/test/bridge-codec.test.ts +++ b/test/bridge-codec.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { BridgeCodec, type CodecOptions } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { BridgeCodecError, BridgeProtocolError, @@ -633,6 +634,71 @@ describe('decodeResponseAsync - Arrow Integration', () => { }); }); + it('records scalar ndarray proof at the root and nested array and record slots', async () => { + const scalar = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype: 'float16', + }; + const rootProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: scalar }), + rootProof + ) + ).toBe(1.5); + expect(rootProof.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + + const nestedProof = new DecodedProvenance(); + const decoded = await codec.decodeResponseAsync<{ tuple: [number, { value: number }] }>( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { tuple: [scalar, { value: scalar }] }, + }), + nestedProof + ); + expect(decoded).toEqual({ tuple: [1.5, { value: 1.5 }] }); + expect(nestedProof.atRoot()).toBeUndefined(); + expect(nestedProof.atChild(decoded.tuple, '0')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple[1], 'value')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple, 1)).toBeUndefined(); + }); + + it('does not record proof for plain numbers or malformed scalar envelopes', async () => { + const plainProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: 1.5 }), + plainProof + ) + ).toBe(1.5); + expect(plainProof.atRoot()).toBeUndefined(); + + const malformedProof = new DecodedProvenance(); + await expect( + codec.decodeResponseAsync( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + }, + }), + malformedProof + ) + ).rejects.toThrow(/must be an array at depth 0/); + expect(malformedProof.atRoot()).toBeUndefined(); + }); + it('labels v1 scientific-envelope validation failures without Arrow wording', async () => { const payload = JSON.stringify({ id: 1, diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts index 424890ad..55f756ef 100644 --- a/test/decoded_provenance.test.ts +++ b/test/decoded_provenance.test.ts @@ -19,6 +19,15 @@ describe('decoded provenance', () => { expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); }); + it('uses the same array slot for a numeric or string index', () => { + const parent = [1.5]; + const provenance = new DecodedProvenance(); + const metadata = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + provenance.recordChild(parent, 0, metadata); + expect(provenance.atChild(parent, '0')).toEqual(metadata); + expect(() => provenance.recordChild(parent, '0', metadata)).toThrow(/Duplicate child/); + }); + it('has one root slot and a bounded number of entries', () => { const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; const provenance = new DecodedProvenance(2); diff --git a/test/rpc-client.test.ts b/test/rpc-client.test.ts index b9aaa1eb..1bbc4255 100644 --- a/test/rpc-client.test.ts +++ b/test/rpc-client.test.ts @@ -503,6 +503,71 @@ describe('BridgeProtocol', () => { const parsed = JSON.parse(transport.lastMessage!); expect(parsed.params.kwargs).toEqual({ key: 'value' }); }); + + it('keeps equal scalar proof with each concurrent call', async () => { + transport.send = async message => { + const request = JSON.parse(message); + const dtype = request.params.functionName === 'half' ? 'float16' : 'float32'; + await new Promise(resolve => setTimeout(resolve, dtype === 'float16' ? 5 : 0)); + return JSON.stringify({ + id: request.id, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype, + }, + }); + }; + + const seen: string[] = []; + const [half, full] = await Promise.all([ + protocol.call('module', 'half', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float16'); + seen.push('half'); + }), + protocol.call('module', 'full', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float32'); + seen.push('full'); + }), + ]); + expect([half, full]).toEqual([1.5, 1.5]); + expect(seen).toHaveLength(2); + }); + + it('does not pass rejected decode proof to validation or a later call', async () => { + let bad = true; + transport.setDynamicResponse(() => + bad + ? { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + } + : 1.5 + ); + const validate = vi.fn((_value: number, provenance?: { atRoot(): unknown }) => { + expect(provenance?.atRoot()).toBeUndefined(); + }); + + await expect(protocol.call('module', 'bad', [], undefined, validate)).rejects.toThrow( + /must be an array at depth 0/ + ); + expect(validate).not.toHaveBeenCalled(); + bad = false; + await expect(protocol.call('module', 'plain', [], undefined, validate)).resolves.toBe( + 1.5 + ); + expect(validate).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/test/value_contracts.test.ts b/test/value_contracts.test.ts index 264ee5d2..db366cc2 100644 --- a/test/value_contracts.test.ts +++ b/test/value_contracts.test.ts @@ -1,5 +1,7 @@ import { readFileSync } from 'node:fs'; import { afterEach, describe, expect, it } from 'vitest'; +import { BridgeCodec } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { MAX_SAFE_JSON_INTEGER, VALUE_CONTRACT_REVISION, @@ -15,7 +17,13 @@ import { const specification = JSON.parse( readFileSync(new URL('../docs/maintainers/value-contracts.v2.json', import.meta.url), 'utf8') -) as { revision: number; rules: { integer: { minimum: number; maximum: number } } }; +) as { + revision: number; + rules: { + integer: { minimum: number; maximum: number }; + bytes: { decodedAs: string }; + }; +}; const fixtures = JSON.parse( readFileSync( new URL('../docs/maintainers/value-contract-fixtures.v2.json', import.meta.url), @@ -23,6 +31,12 @@ const fixtures = JSON.parse( ) ) as { revision: number; + bytesCases: readonly { + logicalHex: string; + requestEnvelope: object; + responseEnvelope: object; + decodedAs: string; + }[]; binary16Cases: readonly { wordHex: string; outcome: string }[]; }; @@ -85,6 +99,23 @@ describe('frozen value policy', () => { expect(isSafeJsonInteger(-(2 ** 53))).toBe(false); expect(isSafeJsonInteger(1.5)).toBe(false); }); + + it('matches the shared bytes request and response fixtures', async () => { + const codec = new BridgeCodec(); + expect(specification.rules.bytes.decodedAs).toBe('JavaScript Uint8Array'); + for (const fixture of fixtures.bytesCases) { + const bytes = Uint8Array.from(Buffer.from(fixture.logicalHex, 'hex')); + const request = JSON.parse(codec.encodeRequest({ value: bytes })); + expect(request.value).toEqual(fixture.requestEnvelope); + + const decoded = await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: fixture.responseEnvelope }) + ); + expect(decoded).toBeInstanceOf(Uint8Array); + expect([...decoded]).toEqual([...bytes]); + expect(`Uint8Array[${[...decoded].join(',')}]`).toBe(fixture.decodedAs); + } + }); }); describe('Arrow float16 value contract', () => { @@ -129,6 +160,13 @@ describe('Arrow float16 value contract', () => { ]); }); + it('carries Arrow scalar float16 proof with the decoded number', async () => { + registerWords([0x3e00]); + const provenance = new DecodedProvenance(); + expect(await decodeValueAsync(envelope([]), provenance)).toBe(1.5); + expect(provenance.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + }); + it('uses the ndarray rule inside nested Torch and record values', async () => { registerWords([0x3e00, 0xc080]); const value = { From 9d9ae8e49ddc84e043c49aafcea2afd5008260bf Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:17:23 -0700 Subject: [PATCH 14/50] consume bytes and scalar provenance in callable validators --- src/core/callable-compiler.ts | 11 ++++- src/core/generator.ts | 26 +++++++--- src/runtime/validators.ts | 49 ++++++++++++------- test/callable-compiler.test.ts | 18 +++++++ test/menagerie/__snapshots__/gen.test.ts.snap | 16 +++--- test/runtime_return_validation.test.ts | 46 +++++++++++++++++ 6 files changed, 132 insertions(+), 34 deletions(-) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index a4bb18c3..47cce79e 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -384,10 +384,16 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { if (type.name === 'str') { return { status: 'supported', value: { kind: 'string', wire: 'json', decodedAs: 'string' } }; } + if (type.name === 'bytes') { + return { + status: 'supported', + value: { kind: 'bytes', wire: 'base64-envelope', decodedAs: 'Uint8Array' }, + }; + } return { status: 'unsupported', - reason: 'The frozen value contract does not specify bytes conversion.', - guidance: 'Use an explicit string or ndarray adapter for byte data.', + reason: 'The value contract does not specify this primitive conversion.', + guidance: 'Use a supported primitive or an explicit value adapter.', }; case 'annotated': return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.base }); @@ -443,6 +449,7 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { kind: 'torch-float16', wire: 'ndarray-envelope', decodedAs: 'tensor-record', + dtype: 'torch.float16', value: ndarray.value, }, }; diff --git a/src/core/generator.ts b/src/core/generator.ts index 490cc6e6..1f718676 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -53,6 +53,8 @@ function valueContractToReturnSchema(value: ValueContract): ReturnSchema { return { kind: 'primitive', type: 'number', constraint: value.constraint }; case 'string': return { kind: 'primitive', type: 'string' }; + case 'bytes': + return { kind: 'primitive', type: 'Uint8Array' }; case 'sequence': return { kind: 'array', element: valueContractToReturnSchema(value.item) }; case 'tuple': @@ -75,7 +77,7 @@ function valueContractToReturnSchema(value: ValueContract): ReturnSchema { case 'ndarray-float16': return { kind: 'marker', marker: 'ndarray', dtype: value.dtype, dims: value.rank }; case 'torch-float16': - return { kind: 'marker', marker: 'torch.tensor', dtype: `torch.${value.value.dtype}` }; + return { kind: 'marker', marker: 'torch.tensor', dtype: value.dtype }; case 'unsupported': return { kind: 'any' }; } @@ -676,6 +678,7 @@ export class CodeGenerator { annotatedJSDoc ? func.parameters.map(p => String(p.type)) : undefined ); const filteredParams = func.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'); + const hasDeclaredOverloads = (func.overloads?.length ?? 0) > 0; const keywordOnlyParams = filteredParams.filter(p => p.keywordOnly); const positionalOnlyNames = filteredParams.filter(p => p.positionalOnly).map(p => p.name); const hasVarKwArgs = filteredParams.some(p => p.kwArgs); @@ -707,6 +710,9 @@ export class CodeGenerator { if (!needsKwargsParam) { return ''; } + if (hasDeclaredOverloads) { + return 'Record'; + } if (keywordOnlyParams.length === 0 && hasVarKwArgs) { return 'Record'; } @@ -723,7 +729,7 @@ export class CodeGenerator { ): string => { const pname = this.escapeIdentifier(p.name); const opt = !forceRequired && p.optional ? '?' : ''; - return `${pname}${opt}: ${tsTypeForValue(p)}`; + return `${pname}${opt}: ${hasDeclaredOverloads ? 'unknown' : tsTypeForValue(p)}`; }; const renderVarArgsParam = (forceRequired = false): string | null => { @@ -762,6 +768,7 @@ export class CodeGenerator { const hasKwArgs = needsKwargsParam; const returnType = this.typeToTsFromPython(func.returnType, genericContext, 'return'); + const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const fname = this.escapeIdentifier(func.name); const moduleId = moduleName ?? '__main__'; const validatorName = `__validate${this.escapeIdentifier(func.name, { preserveCase: true })}Result`; @@ -960,8 +967,8 @@ export class CodeGenerator { : ''; const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; - const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${typeParamDecl}(${paramDecl}): Promise<${returnType}> { -${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${returnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); + const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${hasDeclaredOverloads ? '' : typeParamDecl}(${paramDecl}): Promise<${implementationReturnType}> { +${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${implementationReturnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); } `; @@ -1094,6 +1101,7 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${re // the ordinary module call path and never retain process-local state. const staticPrefix = 'static '; const fparams = method.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'); + const hasDeclaredOverloads = (method.overloads?.length ?? 0) > 0; const methodOwnGenericContext = this.buildGenericRenderContext( this.getTypeParameters(method.typeParameters), [method.returnType, ...fparams.map(param => param.type)], @@ -1125,13 +1133,16 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${re ): string => { const pname = this.escapeIdentifier(p.name); const opt = !forceRequired && p.optional ? '?' : ''; - return `${pname}${opt}: ${methodTsValueType(p)}`; + return `${pname}${opt}: ${hasDeclaredOverloads ? 'unknown' : methodTsValueType(p)}`; }; const kwargsType = (() => { if (!needsKwargsParam) { return ''; } + if (hasDeclaredOverloads) { + return 'Record'; + } if (keywordOnlyParams.length === 0 && hasVarKwArgs) { return 'Record'; } @@ -1163,6 +1174,7 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${re methodGenericContext, 'return' ); + const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const mname = this.escapeIdentifier(method.name); const validatorName = `__validate${this.escapeIdentifier(cls.name, { preserveCase: true })}${this.escapeIdentifier(method.name, { preserveCase: true })}Result`; const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(method, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; @@ -1235,8 +1247,8 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${re : ''; const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; - const callExpr = `getRuntimeBridge().call<${returnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; - methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${methodTypeParamDecl}(${paramsDecl}): Promise<${returnType}> { + const callExpr = `getRuntimeBridge().call<${implementationReturnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; + methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${hasDeclaredOverloads ? '' : methodTypeParamDecl}(${paramsDecl}): Promise<${implementationReturnType}> { ${returnValidator}${callPrelude}${guards}${overloadValidator} return ${callExpr}; }`); methodDeclarations.push( diff --git a/src/runtime/validators.ts b/src/runtime/validators.ts index b19d3b69..dd07557a 100644 --- a/src/runtime/validators.ts +++ b/src/runtime/validators.ts @@ -1,6 +1,9 @@ import { BridgeValidationError } from './errors.js'; +import type { DecodedProvenance, DecodedShapeMetadata } from './decoded-provenance.js'; import type { ScientificMarker } from '../utils/codec.js'; +export type { DecodedShapeMetadata } from './decoded-provenance.js'; + /** * Pure validation functions for runtime value checking. * @@ -43,7 +46,7 @@ export type ReturnSchema = dtype?: string; }; -export type ReturnValidator = (result: T) => T; +export type ReturnValidator = (result: T, provenance?: DecodedProvenance) => T; /** One Python parameter used to select a declared overload after call binding. */ export interface OverloadParameterSchema { @@ -60,12 +63,6 @@ export interface OverloadReturnSchema { selectable: boolean; } -export interface DecodedShapeMetadata { - marker: ScientificMarker; - dims?: number; - dtype?: string; -} - const decodedShapeMetadata = new WeakMap(); /** @internal Preserve wire provenance after codec decoding changes the JS shape. */ @@ -136,9 +133,20 @@ function renderSchema(schema: ReturnSchema): string { interface CheckState { readonly definitions: Readonly>; readonly pairs: WeakMap>; + readonly provenance?: DecodedProvenance; } -function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean { +interface ValueLocation { + parent: object; + key: string | number; +} + +function check( + schema: ReturnSchema, + value: unknown, + state: CheckState, + location?: ValueLocation +): boolean { if (schema.kind === 'any') { return true; } @@ -156,7 +164,7 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean seen.add(schema.name); state.pairs.set(value, seen); } - return check(definition, value, state); + return check(definition, value, state, location); } switch (schema.kind) { @@ -183,12 +191,14 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean case 'literal': return Object.is(value, schema.value); case 'array': - return Array.isArray(value) && value.every(item => check(schema.element, item, state)); + return Array.isArray(value) && value.every((item, index) => + check(schema.element, item, state, { parent: value, key: index })); case 'tuple': return ( Array.isArray(value) && value.length === schema.elements.length && - schema.elements.every((entry, index) => check(entry, value[index], state)) + schema.elements.every((entry, index) => + check(entry, value[index], state, { parent: value, key: index })) ); case 'record': { if (!isPlainObject(value)) { @@ -202,20 +212,25 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean } return false; } - if (!check(field.schema, value[key], state)) { + if (!check(field.schema, value[key], state, { parent: value, key })) { return false; } } } return ( !schema.values || - Object.values(value).every(item => check(schema.values as ReturnSchema, item, state)) + Object.entries(value).every(([key, item]) => + check(schema.values as ReturnSchema, item, state, { parent: value, key })) ); } case 'union': - return schema.options.some(option => check(option, value, state)); + return schema.options.some(option => check(option, value, state, location)); case 'marker': { - const metadata = isObjectLike(value) ? decodedShapeMetadata.get(value) : undefined; + const metadata = isObjectLike(value) + ? decodedShapeMetadata.get(value) + : location + ? state.provenance?.atChild(location.parent, location.key) + : state.provenance?.atRoot(); if (metadata?.marker !== schema.marker) { return false; } @@ -237,8 +252,8 @@ export function createReturnValidator( definitions: Readonly> = {} ): ReturnValidator { const declaredType = renderSchema(schema); - return (result: T): T => { - if (!check(schema, result, { definitions, pairs: new WeakMap>() })) { + return (result: T, provenance?: DecodedProvenance): T => { + if (!check(schema, result, { definitions, pairs: new WeakMap>(), provenance })) { throw new BridgeValidationError({ declaredType, receivedShape: describeReceivedShape(result), diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 0d12c3bd..d5513087 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -246,6 +246,9 @@ describe('compileContract', () => { expect(compiled.generated.declaration).toContain( 'export function select(key: number): Promise;' ); + expect(compiled.generated.typescript).toContain( + 'export async function select(key: unknown): Promise' + ); const nested = compiled.callables.find(callable => callable.name === 'nested'); expect(nested?.result.resolution).toMatchObject({ status: 'supported', @@ -439,6 +442,9 @@ describe('compileContract', () => { expect(compiled.diagnostics.some(item => item.path.endsWith('.parameters[0]'))).toBe(false); expect(compiled.generated.typescript).toContain('"selectable":true'); expect(compiled.generated.declaration).toContain('static convert(key: string): Promise;'); + expect(compiled.generated.typescript).toContain( + 'static async convert(key: unknown): Promise' + ); const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-class-overload-')); try { @@ -532,6 +538,18 @@ describe('compileContract', () => { }); }); + it('keeps the existing bytes RPC shape in the value contract', () => { + const resolution = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { kind: 'primitive', name: 'bytes' }, + }); + expect(resolution).toMatchObject({ + status: 'supported', + value: { kind: 'bytes', wire: 'base64-envelope', decodedAs: 'Uint8Array' }, + }); + }); + it('matches the outer Torch dtype recorded by the decoder', () => { const tensorType: PythonType = { kind: 'generic', diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 4426809d..76ecb63e 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -24,11 +24,11 @@ export async function asyncText(): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args, undefined, __validateasync_textResult); } -const __validatebytes_echoResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.bytes_echo", __tywrapReturnDefinitions); +const __validatebytes_echoResult = createReturnValidator({"kind":"primitive","type":"Uint8Array"}, "fixtures.typing_torture.bytes_echo", __tywrapReturnDefinitions); -export async function bytesEcho(value: unknown): Promise { +export async function bytesEcho(value: Uint8Array): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); } const __validatefrozen_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.frozen_values", __tywrapReturnDefinitions); @@ -188,7 +188,7 @@ export function annotatedEcho(value: number): Promise; export function asyncText(): Promise; -export function bytesEcho(value: unknown): Promise; +export function bytesEcho(value: Uint8Array): Promise; export function frozenValues(value: string[]): Promise; @@ -260,11 +260,11 @@ export async function boolsAndInts(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'bools_and_ints', __args, undefined, __validatebools_and_intsResult); } -const __validatebytes_echoResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.bytes_echo", __tywrapReturnDefinitions); +const __validatebytes_echoResult = createReturnValidator({"kind":"primitive","type":"Uint8Array"}, "fixtures.values_torture.bytes_echo", __tywrapReturnDefinitions); -export async function bytesEcho(value: unknown): Promise { +export async function bytesEcho(value: Uint8Array): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.values_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); + return getRuntimeBridge().call('fixtures.values_torture', 'bytes_echo', __args, undefined, __validatebytes_echoResult); } const __validatecomplex_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.complex_value", __tywrapReturnDefinitions); @@ -473,7 +473,7 @@ export class TrafficLight { export function boolsAndInts(): Promise; -export function bytesEcho(value: unknown): Promise; +export function bytesEcho(value: Uint8Array): Promise; export function complexValue(): Promise; diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index df64f016..04cb620e 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -3,6 +3,7 @@ import type { AddressInfo } from 'node:net'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { BridgeValidationError } from '../src/runtime/errors.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { HttpBridge } from '../src/runtime/http.js'; import { NodeBridge } from '../src/runtime/node.js'; import { @@ -42,6 +43,51 @@ describe('generated return validators', () => { ); }); + it('requires per-call provenance for a scalar float16 ndarray', () => { + const validate = createReturnValidator( + { kind: 'marker', marker: 'ndarray', dims: 0, dtype: 'float16' }, + 'fixture.scalar' + ); + const valid = new DecodedProvenance(); + valid.recordRoot({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + const wrongDtype = new DecodedProvenance(); + wrongDtype.recordRoot({ marker: 'ndarray', dims: 0, dtype: 'float32' }); + const wrongShape = new DecodedProvenance(); + wrongShape.recordRoot({ marker: 'ndarray', dims: 1, dtype: 'float16' }); + + expect(() => validate(1.5)).toThrow(BridgeValidationError); + expect(validate(1.5, valid)).toBe(1.5); + expect(() => validate(1.5, wrongDtype)).toThrow(BridgeValidationError); + expect(() => validate(1.5, wrongShape)).toThrow(BridgeValidationError); + expect(validate(1.5, valid)).toBe(1.5); + }); + + it('checks nested scalar proofs at array indices and record keys', () => { + const scalar = { kind: 'marker', marker: 'ndarray', dims: 0, dtype: 'float16' } as const; + const rows = [[1.5]]; + const result = { rows, scalar: 1.5 }; + const proof = new DecodedProvenance(); + proof.recordChild(rows[0]!, 0, { marker: 'ndarray', dims: 0, dtype: 'float16' }); + proof.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + const validate = createReturnValidator({ + kind: 'record', + fields: { + rows: { schema: { kind: 'array', element: { kind: 'tuple', elements: [scalar] } } }, + scalar: { schema: scalar }, + }, + }, 'fixture.nestedScalar'); + expect(validate(result, proof)).toBe(result); + expect(() => validate(result)).toThrow(BridgeValidationError); + + const wrongIndex = new DecodedProvenance(); + wrongIndex.recordChild(rows[0]!, '0', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + wrongIndex.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + expect(() => validate(result, wrongIndex)).toThrow(BridgeValidationError); + + const other = { rows: [[1.5]], scalar: 1.5 }; + expect(() => validate(other, proof)).toThrow(BridgeValidationError); + }); + it('uses declaration order when an optional-arity overload also matches', () => { const stringValue = { kind: 'primitive', type: 'string' } as const; const integerValue = { kind: 'primitive', type: 'number', constraint: 'safe-integer' } as const; From 7e144140ba93e1f4534a82457eb6a3c03b123081 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:18:36 -0700 Subject: [PATCH 15/50] test(architecture): add independent numeric oracle and fixed cases --- test/architecture_numeric_oracle.test.ts | 236 ++++++++++++++++++ .../fixtures/architecture-value-cases.v1.json | 67 +++++ test/python/architecture_numeric_oracle.py | 56 +++++ 3 files changed, 359 insertions(+) create mode 100644 test/architecture_numeric_oracle.test.ts create mode 100644 test/fixtures/architecture-value-cases.v1.json create mode 100644 test/python/architecture_numeric_oracle.py diff --git a/test/architecture_numeric_oracle.test.ts b/test/architecture_numeric_oracle.test.ts new file mode 100644 index 00000000..52a1eb58 --- /dev/null +++ b/test/architecture_numeric_oracle.test.ts @@ -0,0 +1,236 @@ +import { execFile } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import * as arrow from 'apache-arrow'; +import * as fc from 'fast-check'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { VALUE_CONTRACT_REVISION, isSafeJsonInteger } from '../src/contracts/value-contract.js'; +import { + clearArrowDecoder, + decodeValue, + decodeValueAsync, + registerArrowDecoder, + type ArrowTable, +} from '../src/utils/codec.js'; + +const fixturePath = fileURLToPath( + new URL('./fixtures/architecture-value-cases.v1.json', import.meta.url) +); +const oraclePath = fileURLToPath( + new URL('./python/architecture_numeric_oracle.py', import.meta.url) +); +const python = process.env.PYTHON ?? (process.platform === 'win32' ? 'python' : 'python3'); +const execFileAsync = promisify(execFile); + +interface FixedCase { + wordHex: string; + decimal: string; + negativeZero: boolean; +} + +interface ArchitectureFixture { + schemaVersion: number; + contractRevision: number; + propertySeed: number; + safeIntegers: Array<{ decimal: string; accepted: boolean }>; + binary16: FixedCase[]; + nonFiniteBinary16: Array<{ wordHex: string; kind: 'infinity' | 'nan' }>; + nestedCases: Array<{ path: string; float16Words?: string[] }>; +} + +interface NumericOracle { + contractRevision: number; + method: string; + words: Array<[decimalOrKind: string, negativeZero: boolean]>; + integers: Array<{ decimal: string; accepted: boolean }>; +} + +const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as ArchitectureFixture; +const lockfile = JSON.parse( + readFileSync(new URL('../package-lock.json', import.meta.url), 'utf8') +) as { packages: Record }; +let oracle: NumericOracle; + +function isNonFinite(label: string): boolean { + return label === 'NaN' || label === '+Infinity' || label === '-Infinity'; +} + +function decimalFor(word: number): number { + const entry = oracle.words[word]; + if (!entry || isNonFinite(entry[0])) { + throw new Error(`Expected a finite Python oracle value for word 0x${word.toString(16)}`); + } + return Number(entry[0]); +} + +function envelope(shape: number[], b64 = 'AA=='): object { + return { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'arrow', + b64, + shape, + dtype: 'float16', + }; +} + +function registerStorageWords(words: readonly number[]): void { + registerArrowDecoder( + () => + ({ + getChildAt: () => ({ + type: 'Float16', + nullCount: 0, + toArray: () => Uint16Array.from(words), + }), + }) as ArrowTable + ); +} + +beforeAll(async () => { + const { stdout } = await execFileAsync(python, [oraclePath, fixturePath], { + timeout: 20_000, + maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, PYTHONNOUSERSITE: '1' }, + }); + oracle = JSON.parse(stdout) as NumericOracle; + expect(fixture.schemaVersion).toBe(1); + expect(fixture.contractRevision).toBe(VALUE_CONTRACT_REVISION); + expect(oracle.contractRevision).toBe(VALUE_CONTRACT_REVISION); + expect(oracle.method).toBe('python-stdlib-struct-unpack-binary16'); + expect(lockfile.packages['node_modules/apache-arrow']?.version).toBe('21.1.0'); + expect(oracle.words).toHaveLength(0x10000); + expect(oracle.integers).toHaveLength(fixture.safeIntegers.length); +}, 30_000); + +afterEach(() => clearArrowDecoder()); + +describe('architecture numeric oracle', () => { + it('agrees with Python on the fixed safe-integer domain', () => { + for (const [index, fixed] of fixture.safeIntegers.entries()) { + expect(oracle.integers[index]).toEqual(fixed); + const asNumber = Number(fixed.decimal); + expect(isSafeJsonInteger(asNumber), fixed.decimal).toBe(fixed.accepted); + if (fixed.accepted) { + expect(BigInt(asNumber)).toBe(BigInt(fixed.decimal)); + } + } + }); + + it('matches every finite binary16 word against Python struct.unpack', () => { + const words: number[] = []; + const expected: number[] = []; + let nonFiniteCount = 0; + for (let word = 0; word < oracle.words.length; word += 1) { + const entry = oracle.words[word]; + if (!entry) throw new Error(`Python oracle omitted word ${word}`); + if (isNonFinite(entry[0])) { + nonFiniteCount += 1; + expect(entry[1]).toBe(false); + continue; + } + const value = Number(entry[0]); + expect(Number.isFinite(value), `word 0x${word.toString(16)}`).toBe(true); + expect(Object.is(value, -0), `word 0x${word.toString(16)}`).toBe(entry[1]); + words.push(word); + expected.push(value); + } + expect(words).toHaveLength(63_488); + expect(nonFiniteCount).toBe(2_048); + + // This injects Arrow storage words. The clean consumer test covers real Python Arrow output. + registerStorageWords(words); + const decoded = decodeValue(envelope([words.length])) as number[]; + expect(decoded).toHaveLength(words.length); + for (let index = 0; index < words.length; index += 1) { + if (!Object.is(decoded[index], expected[index])) { + throw new Error(`binary16 word 0x${words[index]?.toString(16)} differs from Python`); + } + } + }, 30_000); + + it('matches fixed finite cases and rejects NaN and infinity', () => { + for (const fixed of fixture.binary16) { + const word = Number.parseInt(fixed.wordHex, 16); + const expected = Number(fixed.decimal); + expect(Object.is(decimalFor(word), expected), fixed.wordHex).toBe(true); + expect(oracle.words[word]?.[1], fixed.wordHex).toBe(fixed.negativeZero); + registerStorageWords([word]); + const decoded = decodeValue(envelope([1])) as number[]; + expect(Object.is(decoded[0], expected), fixed.wordHex).toBe(true); + } + for (const fixed of fixture.nonFiniteBinary16) { + const word = Number.parseInt(fixed.wordHex, 16); + const label = oracle.words[word]?.[0]; + expect(fixed.kind === 'nan' ? label === 'NaN' : label?.endsWith('Infinity')).toBe(true); + registerStorageWords([word]); + expect(() => decodeValue(envelope([1]))).toThrow(/non-finite float16/); + } + }); + + it('decodes scalar and multidimensional shapes through real Arrow IPC', () => { + const words = Uint16Array.of(0x3e00, 0xc080, 0x0001, 0x8000); + const vector = arrow.makeVector({ type: new arrow.Float16(), data: words }); + const table = arrow.tableFromArrays({ value: vector }); + const b64 = Buffer.from(arrow.tableToIPC(table, 'stream')).toString('base64'); + registerArrowDecoder(bytes => arrow.tableFromIPC(bytes) as unknown as ArrowTable); + + const decoded = decodeValue(envelope([2, 2], b64)) as number[][]; + expect(decoded).toHaveLength(2); + expect(decoded[0]).toEqual([decimalFor(0x3e00), decimalFor(0xc080)]); + expect(decoded[1]?.[0]).toBe(decimalFor(0x0001)); + expect(Object.is(decoded[1]?.[1], -0)).toBe(true); + + const scalar = arrow.tableFromArrays({ + value: arrow.makeVector({ type: new arrow.Float16(), data: Uint16Array.of(0x3e00) }), + }); + const scalarB64 = Buffer.from(arrow.tableToIPC(scalar, 'stream')).toString('base64'); + expect(decodeValue(envelope([], scalarB64))).toBe(decimalFor(0x3e00)); + }); + + it('reuses float16 conversion inside Torch and records with a retained seed', async () => { + const nested = fixture.nestedCases.find(testCase => testCase.float16Words); + if (!nested?.float16Words) throw new Error('Missing nested Torch fixture'); + expect(nested.path).toBe('result.outer[0].tensor.value'); + const fixedWords = nested.float16Words.map(word => Number.parseInt(word, 16)); + + const finiteWord = fc.integer({ min: 0, max: 0xffff }).filter(word => { + const entry = oracle.words[word]; + return entry !== undefined && !isNonFinite(entry[0]); + }); + await fc.assert( + fc.asyncProperty(fc.array(finiteWord, { minLength: 1, maxLength: 8 }), async sampledWords => { + const words = [...fixedWords, ...sampledWords]; + registerStorageWords(words); + const decoded = (await decodeValueAsync({ + outer: [ + { + tensor: { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + value: envelope([words.length]), + shape: [words.length], + dtype: 'torch.float16', + device: 'cpu', + }, + }, + ], + })) as { outer: Array<{ tensor: { data: number[] } }> }; + const values = decoded.outer[0]?.tensor.data; + if (!values || values.length !== words.length) { + throw new Error('Nested Torch value did not keep its shape'); + } + for (let index = 0; index < words.length; index += 1) { + if (!Object.is(values[index], decimalFor(words[index] as number))) { + throw new Error(`Nested float16 word 0x${words[index]?.toString(16)} differs`); + } + } + }), + { seed: fixture.propertySeed, numRuns: 64, verbose: true } + ); + }, 30_000); +}); diff --git a/test/fixtures/architecture-value-cases.v1.json b/test/fixtures/architecture-value-cases.v1.json new file mode 100644 index 00000000..ff610f07 --- /dev/null +++ b/test/fixtures/architecture-value-cases.v1.json @@ -0,0 +1,67 @@ +{ + "schemaVersion": 1, + "contractRevision": 2, + "propertySeed": 20260920, + "safeIntegers": [ + { "decimal": "-9007199254740991", "accepted": true }, + { "decimal": "9007199254740991", "accepted": true }, + { "decimal": "-9007199254740992", "accepted": false }, + { "decimal": "9007199254740992", "accepted": false }, + { "decimal": "9007199254740993", "accepted": false }, + { "decimal": "-9223372036854775808", "accepted": false }, + { "decimal": "9223372036854775807", "accepted": false }, + { "decimal": "265252859812191058636308480000000", "accepted": false } + ], + "binary16": [ + { "wordHex": "0000", "decimal": "0", "negativeZero": false }, + { "wordHex": "8000", "decimal": "-0", "negativeZero": true }, + { "wordHex": "0001", "decimal": "5.960464477539063e-8", "negativeZero": false }, + { "wordHex": "03ff", "decimal": "0.00006097555160522461", "negativeZero": false }, + { "wordHex": "0400", "decimal": "0.00006103515625", "negativeZero": false }, + { "wordHex": "3e00", "decimal": "1.5", "negativeZero": false }, + { "wordHex": "c080", "decimal": "-2.25", "negativeZero": false }, + { "wordHex": "7bff", "decimal": "65504", "negativeZero": false }, + { "wordHex": "fbff", "decimal": "-65504", "negativeZero": false } + ], + "nonFiniteBinary16": [ + { "wordHex": "7c00", "kind": "infinity" }, + { "wordHex": "fc00", "kind": "infinity" }, + { "wordHex": "7d00", "kind": "nan" }, + { "wordHex": "7e00", "kind": "nan" }, + { "wordHex": "fe00", "kind": "nan" } + ], + "nestedCases": [ + { + "path": "result.outer[0].value", + "value": "9007199254740993", + "outcome": "reject" + }, + { + "path": "result.outer[0].tensor.value", + "float16Words": ["3e00", "c080"], + "outcome": "numbers" + } + ], + "overloadCases": [ + { + "function": "select_record", + "inputType": "str", + "argument": "label", + "returnType": "dict[str, dict[str, str]]", + "expected": { "outer": { "value": "label" } } + }, + { + "function": "select_record", + "inputType": "int", + "argument": 42, + "returnType": "dict[str, dict[str, int]]", + "expected": { "outer": { "value": 42 } } + } + ], + "wrongReturn": { + "function": "wrong_integer_return", + "declaredType": "int", + "returnedValue": "not an integer", + "expectedError": "BridgeValidationError" + } +} diff --git a/test/python/architecture_numeric_oracle.py b/test/python/architecture_numeric_oracle.py new file mode 100644 index 00000000..d33ff21b --- /dev/null +++ b/test/python/architecture_numeric_oracle.py @@ -0,0 +1,56 @@ +"""Produce independent binary16 and safe-integer expectations for conformance tests.""" + +from __future__ import annotations + +import json +import math +import struct +import sys +from pathlib import Path + + +CONTRACT_REVISION = 2 +SAFE_INTEGER_MAX = (1 << 53) - 1 + + +def binary16_word(word: int) -> tuple[str, bool]: + """Read one raw binary16 word with Python's standard IEEE 754 decoder.""" + value = struct.unpack('>e', word.to_bytes(2, 'big'))[0] + if math.isnan(value): + return 'NaN', False + if math.isinf(value): + return ('-Infinity' if value < 0 else '+Infinity'), False + negative_zero = value == 0.0 and math.copysign(1.0, value) < 0 + return repr(value), negative_zero + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit('usage: architecture_numeric_oracle.py FIXTURE_JSON') + + fixture = json.loads(Path(sys.argv[1]).read_text(encoding='utf8')) + if fixture['schemaVersion'] != 1 or fixture['contractRevision'] != CONTRACT_REVISION: + raise ValueError('architecture fixture version does not match the oracle') + + integers = [] + for case in fixture['safeIntegers']: + value = int(case['decimal']) + integers.append( + {'decimal': case['decimal'], 'accepted': -SAFE_INTEGER_MAX <= value <= SAFE_INTEGER_MAX} + ) + + json.dump( + { + 'contractRevision': CONTRACT_REVISION, + 'method': 'python-stdlib-struct-unpack-binary16', + 'words': [binary16_word(word) for word in range(1 << 16)], + 'integers': integers, + }, + sys.stdout, + separators=(',', ':'), + allow_nan=False, + ) + + +if __name__ == '__main__': + main() From 16e2c48a73a4c17d512f5b3a57bf55703915ed39 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:19:20 -0700 Subject: [PATCH 16/50] test(architecture): bound exhaustive oracle assertion cost --- test/architecture_numeric_oracle.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/architecture_numeric_oracle.test.ts b/test/architecture_numeric_oracle.test.ts index 52a1eb58..ec366682 100644 --- a/test/architecture_numeric_oracle.test.ts +++ b/test/architecture_numeric_oracle.test.ts @@ -129,12 +129,16 @@ describe('architecture numeric oracle', () => { if (!entry) throw new Error(`Python oracle omitted word ${word}`); if (isNonFinite(entry[0])) { nonFiniteCount += 1; - expect(entry[1]).toBe(false); + if (entry[1]) throw new Error(`Non-finite word 0x${word.toString(16)} has a zero flag`); continue; } const value = Number(entry[0]); - expect(Number.isFinite(value), `word 0x${word.toString(16)}`).toBe(true); - expect(Object.is(value, -0), `word 0x${word.toString(16)}`).toBe(entry[1]); + if (!Number.isFinite(value)) { + throw new Error(`Python oracle word 0x${word.toString(16)} is not finite`); + } + if (Object.is(value, -0) !== entry[1]) { + throw new Error(`Python oracle word 0x${word.toString(16)} has the wrong zero flag`); + } words.push(word); expected.push(value); } From 05a891e906fec9a0ba4a2b9c1c359d8ddd38a9af Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:19:38 -0700 Subject: [PATCH 17/50] test: expect broad hidden overload implementation --- test/generator.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/generator.test.ts b/test/generator.test.ts index abcb890d..8816ada9 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -649,7 +649,7 @@ describe('CodeGenerator', () => { expect(code.typescript).toContain('export function getValue(key: string): Promise;'); expect(code.typescript).toContain('export function getValue(key: number): Promise;'); expect(code.typescript).toContain( - 'export async function getValue(key: string | number): Promise' + 'export async function getValue(key: unknown): Promise' ); expect(code.declaration).toContain('export function getValue(key: string): Promise;'); expect(code.declaration).toContain('export function getValue(key: number): Promise;'); From 855115eeaba478e82db99a8b28954c6756aded7a Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:20:14 -0700 Subject: [PATCH 18/50] style(test): use braces in architecture oracle --- test/architecture_numeric_oracle.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/architecture_numeric_oracle.test.ts b/test/architecture_numeric_oracle.test.ts index ec366682..fcded9fa 100644 --- a/test/architecture_numeric_oracle.test.ts +++ b/test/architecture_numeric_oracle.test.ts @@ -126,10 +126,14 @@ describe('architecture numeric oracle', () => { let nonFiniteCount = 0; for (let word = 0; word < oracle.words.length; word += 1) { const entry = oracle.words[word]; - if (!entry) throw new Error(`Python oracle omitted word ${word}`); + if (!entry) { + throw new Error(`Python oracle omitted word ${word}`); + } if (isNonFinite(entry[0])) { nonFiniteCount += 1; - if (entry[1]) throw new Error(`Non-finite word 0x${word.toString(16)} has a zero flag`); + if (entry[1]) { + throw new Error(`Non-finite word 0x${word.toString(16)} has a zero flag`); + } continue; } const value = Number(entry[0]); @@ -197,7 +201,9 @@ describe('architecture numeric oracle', () => { it('reuses float16 conversion inside Torch and records with a retained seed', async () => { const nested = fixture.nestedCases.find(testCase => testCase.float16Words); - if (!nested?.float16Words) throw new Error('Missing nested Torch fixture'); + if (!nested?.float16Words) { + throw new Error('Missing nested Torch fixture'); + } expect(nested.path).toBe('result.outer[0].tensor.value'); const fixedWords = nested.float16Words.map(word => Number.parseInt(word, 16)); From 7e2301cdd6878a1423cf6787c219ac81925e7d47 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:20:22 -0700 Subject: [PATCH 19/50] fix(rpc): keep one-argument validator call shape --- src/runtime/rpc-client.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index e475d10a..aa1ad6a6 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -448,7 +448,12 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - validate(result, provenance); + // Keep the original call shape for validators that accept one argument. + if (validate.length > 1) { + validate(result, provenance); + } else { + validate(result); + } return result; }, } From a642527bc2acd74387aa81b7bc8d6c80fb7ffea5 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:21:30 -0700 Subject: [PATCH 20/50] test(rpc): assert additive validator provenance --- src/runtime/rpc-client.ts | 7 +------ test/runtime_return_validation.test.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index aa1ad6a6..e475d10a 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -448,12 +448,7 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - // Keep the original call shape for validators that accept one argument. - if (validate.length > 1) { - validate(result, provenance); - } else { - validate(result); - } + validate(result, provenance); return result; }, } diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 033bd349..e4a78b18 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -3,6 +3,7 @@ import type { AddressInfo } from 'node:net'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { BridgeValidationError } from '../src/runtime/errors.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { HttpBridge } from '../src/runtime/http.js'; import { NodeBridge } from '../src/runtime/node.js'; import { @@ -220,7 +221,17 @@ describe('return validator bridge propagation', () => { try { await expect(bridge.call('math', 'sqrt', [16], undefined, validate)).resolves.toBe(4); expect(validate).toHaveBeenCalledOnce(); - expect(validate).toHaveBeenCalledWith(4); + expect(validate).toHaveBeenCalledWith(4, expect.any(DecodedProvenance)); + const [, provenance] = validate.mock.calls[0] as unknown as [number, DecodedProvenance]; + expect(provenance.atRoot()).toBeUndefined(); + + const reject = vi.fn((_value: number) => { + throw new Error('one-argument validator rejected the value'); + }); + await expect(bridge.call('math', 'sqrt', [16], undefined, reject)).rejects.toThrow( + 'one-argument validator rejected the value' + ); + expect(reject).toHaveBeenCalledOnce(); } finally { await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve())) From 552b530e8ffe7d40d8d51640e30f61df3ab9f253 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:24:23 -0700 Subject: [PATCH 21/50] test: prove scalar float16 through generated HTTP wrapper --- test/callable-compiler.test.ts | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index d5513087..f3da30d7 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from 'vitest'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import ts from 'typescript'; @@ -11,6 +13,7 @@ import { import { CodeGenerator } from '../src/core/generator.js'; import { validateIrContract } from '../src/core/ir-contract.js'; import { BridgeValidationError } from '../src/runtime/errors.js'; +import { HttpBridge } from '../src/runtime/http.js'; import { clearRuntimeBridge, setRuntimeBridge } from 'tywrap/runtime'; import type { PythonModule, PythonType } from '../src/types/index.js'; @@ -478,6 +481,81 @@ describe('compileContract', () => { } }); + it('validates a generated scalar float16 wrapper from the HTTP codec proof', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract({ + ...rawIr, + functions: [{ ...source, name: 'scalar_value', qualname: 'fixture.scalar_value', + returns: 'numpy.typing.NDArray[numpy.float16]' }], + classes: [], + }, 'scalar contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const scalarType: PythonType = { + kind: 'generic', name: 'NDArray', module: 'numpy.typing', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }; + const original = moduleModel.functions[1]!; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [{ + ...original, + name: 'scalar_value', + returnType: scalarType, + signature: { ...original.signature, returnType: scalarType }, + }], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.typescript).toContain('"marker":"ndarray","dtype":"float16"'); + + let dtype = 'float16'; + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'ndarray', codecVersion: 1, encoding: 'json', + shape: [], dtype, data: 1.5, + }, + })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-scalar-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + scalarValue: () => Promise; + }; + await expect(generated.scalarValue()).resolves.toBe(1.5); + dtype = 'float32'; + await expect(generated.scalarValue()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('uses revision 2 for selected unions and fixed heterogeneous tuples', () => { const union = DEFAULT_VALUE_CONVERSION.resolve({ direction: 'output', From 1fa6a87aa6104a557057b5518ac2370dfb653c3d Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:24:42 -0700 Subject: [PATCH 22/50] test(architecture): add overloaded nested-record fixtures --- .../architecture-overload.typecheck.ts.txt | 18 ++++++++++ .../fixtures/architecture_callable_fixture.py | 35 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 test/fixtures/architecture-overload.typecheck.ts.txt create mode 100644 test/fixtures/architecture_callable_fixture.py diff --git a/test/fixtures/architecture-overload.typecheck.ts.txt b/test/fixtures/architecture-overload.typecheck.ts.txt new file mode 100644 index 00000000..8e18d6ce --- /dev/null +++ b/test/fixtures/architecture-overload.typecheck.ts.txt @@ -0,0 +1,18 @@ +import { selectRecord } from './generated/architecture_callable_fixture.generated.js'; + +const textRecord: Promise>> = selectRecord('label'); +const integerRecord: Promise>> = selectRecord(42); + +// @ts-expect-error An integer input must not infer a string record. +const wrongTextRecord: Promise>> = selectRecord(42); + +// @ts-expect-error A string input must not infer an integer record. +const wrongIntegerRecord: Promise>> = selectRecord('label'); + +// @ts-expect-error A boolean input matches no overload. +selectRecord(true); + +void textRecord; +void integerRecord; +void wrongTextRecord; +void wrongIntegerRecord; diff --git a/test/fixtures/architecture_callable_fixture.py b/test/fixtures/architecture_callable_fixture.py new file mode 100644 index 00000000..193df756 --- /dev/null +++ b/test/fixtures/architecture_callable_fixture.py @@ -0,0 +1,35 @@ +"""Controlled overload and return-validation cases for the clean consumer.""" + +from __future__ import annotations + +from typing import overload + + +@overload +def select_record(key: str) -> dict[str, dict[str, str]]: ... + + +@overload +def select_record(key: int) -> dict[str, dict[str, int]]: ... + + +def select_record(key: str | int) -> dict[str, dict[str, str | int]]: + return {'outer': {'value': key}} + + +@overload +def select_record_wrong(key: str) -> dict[str, dict[str, str]]: ... + + +@overload +def select_record_wrong(key: int) -> dict[str, dict[str, int]]: ... + + +def select_record_wrong(key: str | int) -> dict[str, dict[str, str | int]]: + if isinstance(key, str): + return {'outer': {'value': 42}} + return {'outer': {'value': 'not an integer'}} + + +def wrong_integer_return() -> int: + return 'not an integer' From e14d2f31c93c66a22ffa6695e97355f2c799cae5 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:26:31 -0700 Subject: [PATCH 23/50] fix(codec): bind Torch float16 to nested ndarray dtype --- docs/maintainers/value-contracts.md | 3 +- src/utils/codec.ts | 13 +++++++ test/value_contracts.test.ts | 56 +++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index dbc55a31..20c8293f 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -23,7 +23,8 @@ constraint. Regenerate wrappers after that compiler change. Do not return The bytes leaf keeps the existing base64 envelopes. Requests use `__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to `Uint8Array`. The Torch float16 contract names `torch.float16` on the outer -tensor and `float16` on its nested ndarray. +tensor and `float16` on its nested version 1 ndarray. The decoder rejects a +different nested dtype before it decodes the data. Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow diff --git a/src/utils/codec.ts b/src/utils/codec.ts index dec055a9..83d26a2b 100644 --- a/src/utils/codec.ts +++ b/src/utils/codec.ts @@ -1109,6 +1109,19 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = ( if (strictV1 && 'dtype' in nested) { assertOptionalNonEmptyDtype(nested.dtype, 'torch.tensor', 'value.dtype', nestedShape); } + if (strictV1 && dtype === 'torch.float16') { + if (!isStrictV1Envelope(nested)) { + throw new Error( + 'Invalid torch.tensor envelope: float16 value must use ndarray codecVersion 1' + ); + } + if (nested.dtype !== 'float16') { + throw new Error( + `Invalid torch.tensor envelope: value.dtype at path value.dtype must be "float16" ` + + `for outer dtype "torch.float16"; got ${JSON.stringify(nested.dtype)}` + ); + } + } const deviceValue = value.device; if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) { throw new Error( diff --git a/test/value_contracts.test.ts b/test/value_contracts.test.ts index db366cc2..71a7eaed 100644 --- a/test/value_contracts.test.ts +++ b/test/value_contracts.test.ts @@ -187,6 +187,62 @@ describe('Arrow float16 value contract', () => { }); }); + it.each([ + { shape: [], encoding: 'json' }, + { shape: [1], encoding: 'json' }, + { shape: [], encoding: 'arrow' }, + { shape: [1], encoding: 'arrow' }, + ] as const)( + 'requires a matching nested dtype for Torch float16 shape $shape with $encoding data', + async ({ shape, encoding }) => { + registerWords([0x3e00]); + const nested = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding, + shape, + dtype: 'float16', + ...(encoding === 'arrow' ? { b64: 'AA==' } : { data: shape.length ? [1.5] : 1.5 }), + }; + const tensor = { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + value: nested, + shape, + dtype: 'torch.float16', + device: 'cpu', + }; + + await expect(decodeValueAsync(tensor)).resolves.toMatchObject({ + data: shape.length ? [1.5] : 1.5, + dtype: 'torch.float16', + }); + await expect( + decodeValueAsync({ ...tensor, value: { ...nested, dtype: 'float32' } }) + ).rejects.toThrow(/value\.dtype.*must be "float16".*"torch\.float16"/); + } + ); + + it('requires the nested version 1 envelope for Torch float16', async () => { + await expect( + decodeValueAsync({ + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + shape: [], + dtype: 'torch.float16', + value: { + __tywrap__: 'ndarray', + encoding: 'json', + shape: [], + dtype: 'float16', + data: 1.5, + }, + }) + ).rejects.toThrow(/float16 value must use ndarray codecVersion 1/); + }); + it.each([0x7c00, 0xfc00, 0x7e00])('rejects non-finite word %s', word => { registerWords([word]); expect(() => decodeValue(envelope([1]))).toThrow(/non-finite float16 value at b64\[0\]/); From e9fa9b2a7c80cb3f7d0cc5c221b006134de89a8f Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:27:41 -0700 Subject: [PATCH 24/50] test: align nested scalar proof with array index normalization --- docs/maintainers/callable-contracts.md | 17 +++++++++++------ test/runtime_return_validation.test.ts | 7 ++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 15cc309d..4f276d96 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -1,8 +1,8 @@ # Callable compilation contracts -`tywrap_ir` emits IR 0.4. `validateIrContract()` accepts unknown JSON and checks every required IR field. Each diagnostic has a JSON path. +`tywrap_ir` emits IR 0.4. `validateIrContract()` checks required fields in unknown JSON. Each diagnostic has a JSON path. Validation stops after 100,000 array entries. -`compileContract()` is an internal pure step. It accepts validated IR, a parser-mapped module, a value conversion description, capability descriptions, and a generator. It returns the resolved module, callable records, local diagnostics, and TypeScript and declaration file content. +`compileContract()` is an internal pure step. It accepts validated IR, a mapped module, value conversions, capabilities, and a generator. It returns the resolved module, callable records, diagnostics, and TypeScript and declaration content. The compiler does not read files. It does not start Python. It does not read a cache. It does not write files. It does not access a runtime registry. @@ -10,21 +10,24 @@ The compiler does not read files. It does not start Python. It does not read a c ## Resolved callable data -The compiler records each parameter, result, overload parameter, and overload result with its logical Python type, direction, source path, and conversion result. It creates one resolved module model for declarations, call wrappers, and return validators. +The compiler records each parameter and result, including overloads. Each record has a Python type, direction, source path, and conversion result. One resolved module model serves declarations, call wrappers, and return validators. Each value conversion uses revision 2 of [value-contracts.v2.json](value-contracts.v2.json). Do not infer a wire rule from the TypeScript type. | Case | Result | | --- | --- | | `int` | JSON number, JavaScript number, safe-integer constraint | +| `bytes` | Existing base64 envelope, decoded as `Uint8Array` | | nested `dict[str, T]` | JSON record with the child conversion at every value path | | `str | int` | Selected union alternative with one child contract per option | | `tuple[int, str]` | Exact JSON array with a contract for each index | -| float16 ndarray | Existing Arrow or JSON-fallback envelope and finite number result | -| float16 Torch tensor | Existing Torch envelope containing the float16 ndarray conversion | +| float16 ndarray | Existing Arrow or JSON-fallback envelope, decoded as a numeric scalar or nested array | +| float16 Torch tensor | Outer `torch.float16` envelope with a nested `float16` ndarray | The compiler requires an explicit float16 dtype. An untyped ndarray or Torch tensor stays unresolved. +The codec records scalar ndarray dtype and rank for one call. A generated return validator accepts a scalar only with matching proof. Nested proof follows its parent and array index or record key. Array index `0` and `"0"` refer to the same child. Equal numbers from separate calls do not share proof. Array and tensor objects keep their identity-based marker proof. + The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. ## Diagnostics and fallback @@ -37,7 +40,9 @@ Known unsupported cases create error diagnostics. The compiler changes the affec Unresolved annotations create warning diagnostics. They retain the generator's current fallback and do not claim a supported conversion. -IR overload signatures stay separate from the implementation signature. Generated declarations retain each input-to-result relation. At runtime, the wrapper selects a result validator when exactly one supported overload matches the bound arguments. Ambiguous calls use the implementation validator. +IR overload signatures stay separate from the implementation signature. The extractor collects type variables from overload annotations. Generated declarations retain each input-to-result relation. The hidden TypeScript implementation accepts `unknown` values so it can cover every declared overload. + +At runtime, the wrapper selects the first supported overload that matches bound arguments. This follows TypeScript declaration order. The compiler warns when overload inputs may overlap, including an omitted optional argument. Python 3.10 cannot expose registered overloads through `typing.get_overloads`; it keeps the implementation signature and emits an IR warning. ## Deferred work diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index e0820bf3..8004b203 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -79,8 +79,13 @@ describe('generated return validators', () => { expect(validate(result, proof)).toBe(result); expect(() => validate(result)).toThrow(BridgeValidationError); + const stringIndex = new DecodedProvenance(); + stringIndex.recordChild(rows[0]!, '0', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + stringIndex.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + expect(validate(result, stringIndex)).toBe(result); + const wrongIndex = new DecodedProvenance(); - wrongIndex.recordChild(rows[0]!, '0', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + wrongIndex.recordChild(rows[0]!, 1, { marker: 'ndarray', dims: 0, dtype: 'float16' }); wrongIndex.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); expect(() => validate(result, wrongIndex)).toThrow(BridgeValidationError); From f20b4a73a5f4fe36fda7076c73ef279c88650d36 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:29:48 -0700 Subject: [PATCH 25/50] test: exercise generated Torch float16 wrapper through HTTP codec --- test/callable-compiler.test.ts | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index f3da30d7..e8c1e12b 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -657,6 +657,87 @@ describe('compileContract', () => { expect(generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); }); + it('rejects a Torch float16 response with a mismatched nested dtype', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract({ + ...rawIr, + functions: [{ ...source, name: 'tensor_value', qualname: 'fixture.tensor_value', + returns: 'torch.Tensor[torch.float16]' }], + classes: [], + }, 'Torch contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const tensorType: PythonType = { + kind: 'generic', name: 'Tensor', module: 'torch', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], + }; + const original = moduleModel.functions[1]!; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [{ + ...original, + name: 'tensor_value', + returnType: tensorType, + signature: { ...original.signature, returnType: tensorType }, + }], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); + + let nestedDtype = 'float16'; + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'torch.tensor', codecVersion: 1, encoding: 'ndarray', + shape: [], dtype: 'torch.float16', device: 'cpu', + value: { + __tywrap__: 'ndarray', codecVersion: 1, encoding: 'json', + shape: [], dtype: nestedDtype, data: 1.5, + }, + }, + })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-torch-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + tensorValue: () => Promise<{ data: number; dtype: string }>; + }; + await expect(generated.tensorValue()).resolves.toMatchObject({ + data: 1.5, dtype: 'torch.float16', + }); + nestedDtype = 'float32'; + await expect(generated.tensorValue()).rejects.toThrow(/value\.dtype.*must be "float16"/); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('degrades unimplemented dataclass and coroutine outputs with local diagnostics', () => { const validation = validateIrContract(rawIr, 'fixture contract'); expect(validation.ok).toBe(true); From 0ac3823ac97a9e12a889e5d6284434b51c659274 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:31:09 -0700 Subject: [PATCH 26/50] test: exercise generated callable contracts in isolated consumer --- test/isolated_consumer.conformance.ts | 70 ++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/test/isolated_consumer.conformance.ts b/test/isolated_consumer.conformance.ts index bbdbe6eb..c28061cd 100644 --- a/test/isolated_consumer.conformance.ts +++ b/test/isolated_consumer.conformance.ts @@ -1,5 +1,6 @@ import { execFile } from 'node:child_process'; import { + copyFileSync, existsSync, mkdtempSync, mkdirSync, @@ -85,6 +86,14 @@ describe('isolated npm consumer', () => { await run(python, ['-m', 'venv', venv], { cwd: tempRoot }); const isolatedPython = venvPython(venv); expect(existsSync(isolatedPython)).toBe(true); + await run( + isolatedPython, + [ + '-c', + 'import sys\nif sys.version_info < (3, 11): raise SystemExit("Python 3.11 is required for overload extraction")', + ], + { cwd: consumer } + ); mkdirSync(wheelDir, { recursive: true }); await run( @@ -145,6 +154,9 @@ describe('isolated npm consumer', () => { writeFileSync( join(fixtures, 'consumer_fixture.py'), [ + 'import numpy as np', + 'from numpy.typing import NDArray', + '', 'def add(left: int, right: int) -> int:', ' return left + right', '', @@ -160,8 +172,7 @@ describe('isolated npm consumer', () => { 'def unsafe_negative() -> int:', ' return -(2**53)', '', - 'def float16_values() -> object:', - ' import numpy as np', + 'def float16_values() -> NDArray[np.float16]:', ' return np.array([1.5, -2.25, -0.0], dtype=np.float16)', '', 'def wrong_return() -> int:', @@ -170,6 +181,10 @@ describe('isolated npm consumer', () => { ].join('\n'), 'utf8' ); + copyFileSync( + join(repoRoot, 'test', 'fixtures', 'architecture_callable_fixture.py'), + join(fixtures, 'architecture_callable_fixture.py') + ); writeFileSync( join(consumer, 'package.json'), `${JSON.stringify({ name: 'tywrap-consumer-check', private: true, type: 'module' }, null, 2)}\n`, @@ -211,7 +226,10 @@ describe('isolated npm consumer', () => { join(consumer, 'tywrap.config.json'), `${JSON.stringify( { - pythonModules: { consumer_fixture: { typeHints: 'strict' } }, + pythonModules: { + consumer_fixture: { typeHints: 'strict' }, + architecture_callable_fixture: { typeHints: 'strict' }, + }, pythonImportPath: [fixtures], output: { dir: generated, format: 'esm', declaration: false, sourceMap: false }, runtime: { node: { pythonPath: isolatedPython } }, @@ -237,6 +255,13 @@ describe('isolated npm consumer', () => { const generatedSource = join(generated, 'consumer_fixture.generated.ts'); expect(existsSync(generatedSource)).toBe(true); expect(readFileSync(generatedSource, 'utf8')).not.toContain(repoRoot); + const architectureSource = join(generated, 'architecture_callable_fixture.generated.ts'); + expect(existsSync(architectureSource)).toBe(true); + expect(readFileSync(architectureSource, 'utf8')).not.toContain(repoRoot); + copyFileSync( + join(repoRoot, 'test', 'fixtures', 'architecture-overload.typecheck.ts.txt'), + join(consumer, 'architecture-overload.typecheck.ts') + ); writeFileSync( join(consumer, 'tsconfig.json'), `${JSON.stringify( @@ -250,7 +275,7 @@ describe('isolated npm consumer', () => { strict: true, skipLibCheck: true, }, - include: ['generated/**/*.ts'], + include: ['generated/**/*.ts', 'architecture-overload.typecheck.ts'], }, null, 2 @@ -272,6 +297,7 @@ describe('isolated npm consumer', () => { import { NodeBridge } from 'tywrap/node'; import { clearRuntimeBridge, setRuntimeBridge } from 'tywrap/runtime'; import * as fixture from './built/generated/consumer_fixture.generated.js'; +import * as architecture from './built/generated/architecture_callable_fixture.generated.js'; const [pythonPath, fixtures] = process.argv.slice(2); const bridge = new NodeBridge({ @@ -280,7 +306,7 @@ const bridge = new NodeBridge({ env: { PYTHONNOUSERSITE: '1', PYTHONPATH: fixtures, - TYWRAP_ALLOWED_MODULES: 'consumer_fixture', + TYWRAP_ALLOWED_MODULES: 'consumer_fixture,architecture_callable_fixture', }, }); @@ -319,7 +345,37 @@ try { if (!(validationError instanceof BridgeValidationError)) { throw new Error('wrong_return did not throw BridgeValidationError'); } - process.stdout.write(JSON.stringify({ add, safeMax, safeMin, float16Values, negativeZero })); + const textRecord = await architecture.selectRecord('label'); + const integerRecord = await architecture.selectRecord(42); + for (const key of ['label', 42]) { + let wrongOverloadError; + try { + await architecture.selectRecordWrong(key); + } catch (error) { + wrongOverloadError = error; + } + if ( + !(wrongOverloadError instanceof BridgeValidationError) || + wrongOverloadError.callSite !== 'architecture_callable_fixture.select_record_wrong' + ) { + throw new Error('selected overload accepted a wrong nested return'); + } + } + let wrongIntegerError; + try { + await architecture.wrongIntegerReturn(); + } catch (error) { + wrongIntegerError = error; + } + if ( + !(wrongIntegerError instanceof BridgeValidationError) || + wrongIntegerError.callSite !== 'architecture_callable_fixture.wrong_integer_return' + ) { + throw new Error('wrong_integer_return did not throw BridgeValidationError'); + } + process.stdout.write(JSON.stringify({ + add, safeMax, safeMin, float16Values, negativeZero, textRecord, integerRecord, + })); } finally { clearRuntimeBridge(); await bridge.dispose(); @@ -338,6 +394,8 @@ try { safeMin: Number.MIN_SAFE_INTEGER, float16Values: [1.5, -2.25, 0], negativeZero: true, + textRecord: { outer: { value: 'label' } }, + integerRecord: { outer: { value: 42 } }, }); }, 240_000); }); From b831054b223bb97f79348e605dcff85f8d03b8e3 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:33:12 -0700 Subject: [PATCH 27/50] test: reject architecture contract mutations structurally --- test/architecture_mutations.test.ts | 196 ++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 test/architecture_mutations.test.ts diff --git a/test/architecture_mutations.test.ts b/test/architecture_mutations.test.ts new file mode 100644 index 00000000..d180f7c7 --- /dev/null +++ b/test/architecture_mutations.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it } from 'vitest'; + +import { + compileContract, + DEFAULT_CALLABLE_CAPABILITIES, + DEFAULT_VALUE_CONVERSION, + type CapabilityDescription, +} from '../src/core/callable-compiler.js'; +import { CodeGenerator } from '../src/core/generator.js'; +import { validateIrContract } from '../src/core/ir-contract.js'; +import { BridgeCodec } from '../src/runtime/bridge-codec.js'; +import { BridgeCodecError } from '../src/runtime/errors.js'; +import type { PythonModule, PythonType } from '../src/types/index.js'; + +function tensorEnvelope(): Record { + return { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + dtype: 'torch.float16', + shape: [2], + device: 'cpu', + value: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + dtype: 'float16', + shape: [2], + data: [1.5, -2.25], + }, + }; +} + +async function decode(result: unknown): Promise { + return new BridgeCodec().decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result }) + ); +} + +async function expectDecodeFailure(result: unknown, marker: string): Promise { + let caught: unknown; + try { + await decode(result); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(BridgeCodecError); + expect(caught).toMatchObject({ codecPhase: 'decode', valueType: marker }); +} + +function compileReturn( + returnType: PythonType, + annotation: string, + capabilities: readonly CapabilityDescription[] = DEFAULT_CALLABLE_CAPABILITIES +) { + const rawIr = { + ir_version: '0.4.0', + module: 'mutation_fixture', + functions: [ + { + name: 'probe', + qualname: 'mutation_fixture.probe', + docstring: null, + parameters: [], + returns: annotation, + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + ], + classes: [], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [], + }; + const validation = validateIrContract(rawIr, 'mutation fixture'); + if (!validation.ok) { + throw new Error(`Mutation fixture did not validate: ${JSON.stringify(validation.diagnostics)}`); + } + const module: PythonModule = { + name: 'mutation_fixture', + functions: [ + { + name: 'probe', + signature: { parameters: [], returnType, isAsync: false, isGenerator: false }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [], + returnType, + }, + ], + classes: [], + imports: [], + exports: [], + }; + return compileContract(validation.contract, { + module, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities, + }); +} + +describe('architecture contract mutations', () => { + it('accepts the matching strict v1 Torch envelope', async () => { + await expect(decode(tensorEnvelope())).resolves.toMatchObject({ + data: [1.5, -2.25], + shape: [2], + dtype: 'torch.float16', + device: 'cpu', + }); + }); + + it('rejects an outer Torch float16 dtype paired with nested float32', async () => { + const envelope = tensorEnvelope(); + envelope.value = { ...(envelope.value as object), dtype: 'float32' }; + await expectDecodeFailure(envelope, 'torch.tensor'); + }); + + it.each([ + { + name: 'outer shape', + mutate: (envelope: Record) => { + envelope.shape = [1, 2]; + }, + }, + { + name: 'codec version', + mutate: (envelope: Record) => { + envelope.codecVersion = 2; + }, + }, + { + name: 'encoding', + mutate: (envelope: Record) => { + envelope.encoding = 'raw'; + }, + }, + ])('rejects a mutated $name with a structured decode error', async ({ mutate }) => { + const envelope = tensorEnvelope(); + mutate(envelope); + await expectDecodeFailure(envelope, 'torch.tensor'); + }); + + it('reports an unsupported record key conversion at the return path', () => { + const compiled = compileReturn( + { + kind: 'collection', + name: 'dict', + itemTypes: [ + { kind: 'primitive', name: 'int' }, + { kind: 'primitive', name: 'str' }, + ], + }, + 'dict[int, str]' + ); + expect(compiled.diagnostics).toContainEqual( + expect.objectContaining({ + severity: 'error', + code: 'conversion-unsupported', + path: '$.functions[0].returns', + }) + ); + expect(compiled.callables[0]?.result.resolution.status).toBe('unsupported'); + expect(compiled.module.functions[0]?.callableContract?.returnValue).toBeUndefined(); + }); + + it('reports an unavailable scientific capability at the return path', () => { + const capabilities = DEFAULT_CALLABLE_CAPABILITIES.map(capability => + capability.name === 'scientific-ndarray' ? { ...capability, available: false } : capability + ); + const compiled = compileReturn( + { + kind: 'generic', + name: 'NDArray', + module: 'numpy.typing', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }, + 'numpy.typing.NDArray[numpy.float16]', + capabilities + ); + expect(compiled.diagnostics).toContainEqual( + expect.objectContaining({ + severity: 'error', + code: 'capability-unavailable', + path: '$.functions[0].returns', + }) + ); + expect(compiled.callables[0]?.result.resolution.status).toBe('unsupported'); + expect(compiled.module.functions[0]?.callableContract?.returnValue).toBeUndefined(); + }); +}); From 64087dc2e48b0a09169f61f7f4eba203b7349aa9 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:35:31 -0700 Subject: [PATCH 28/50] test: keep overload helper out of extracted module callables --- test/fixtures/architecture_callable_fixture.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/fixtures/architecture_callable_fixture.py b/test/fixtures/architecture_callable_fixture.py index 193df756..d600eb5a 100644 --- a/test/fixtures/architecture_callable_fixture.py +++ b/test/fixtures/architecture_callable_fixture.py @@ -2,14 +2,14 @@ from __future__ import annotations -from typing import overload +import typing -@overload +@typing.overload def select_record(key: str) -> dict[str, dict[str, str]]: ... -@overload +@typing.overload def select_record(key: int) -> dict[str, dict[str, int]]: ... @@ -17,11 +17,11 @@ def select_record(key: str | int) -> dict[str, dict[str, str | int]]: return {'outer': {'value': key}} -@overload +@typing.overload def select_record_wrong(key: str) -> dict[str, dict[str, str]]: ... -@overload +@typing.overload def select_record_wrong(key: int) -> dict[str, dict[str, int]]: ... From 5c1728f27c1a33ca7d7593ae35645a9b944f7b2f Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:38:48 -0700 Subject: [PATCH 29/50] fix(compiler): derive callable types from IR and detect bound overload overlap --- docs/maintainers/callable-contracts.md | 4 +- src/core/callable-compiler.ts | 162 ++++++++++++++++------ src/core/ir-model.ts | 165 +++++++++++++++++++++++ src/tywrap.ts | 179 +------------------------ test/callable-compiler.test.ts | 75 +++++++++++ 5 files changed, 365 insertions(+), 220 deletions(-) create mode 100644 src/core/ir-model.ts diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 4f276d96..b2b3c653 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -2,7 +2,7 @@ `tywrap_ir` emits IR 0.4. `validateIrContract()` checks required fields in unknown JSON. Each diagnostic has a JSON path. Validation stops after 100,000 array entries. -`compileContract()` is an internal pure step. It accepts validated IR, a mapped module, value conversions, capabilities, and a generator. It returns the resolved module, callable records, diagnostics, and TypeScript and declaration content. +`compileContract()` is an internal pure step. It accepts validated IR, exported-name selection, value conversions, capabilities, and a generator. It maps callable types from the validated IR. The selection input cannot change those types. It returns the resolved module, callable records, diagnostics, and TypeScript and declaration content. The compiler does not read files. It does not start Python. It does not read a cache. It does not write files. It does not access a runtime registry. @@ -35,7 +35,7 @@ The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow i Known unsupported cases create error diagnostics. The compiler changes the affected generated type to `unknown`. `generate()` returns each diagnostic as a warning, so `--fail-on-warn` rejects strict builds. - Coroutines require the capability owned by #338. -- Dataclasses require an adapter owned by #339. +- Dataclasses require an adapter owned by #339. The default capability stays unavailable. An explicit output record conversion can use the adapter capability when enabled. - Non-string record keys require an explicit conversion. Unresolved annotations create warning diagnostics. They retain the generator's current fallback and do not claim a supported conversion. diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 47cce79e..f3912af9 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -17,7 +17,8 @@ import type { PythonType, } from '../types/index.js'; import type { CodeGenerator } from './generator.js'; -import type { IrFunction, ValidatedIrContract } from './ir-contract.js'; +import type { ValidatedIrContract } from './ir-contract.js'; +import { transformIrToTsModel } from './ir-model.js'; export type CallableDirection = 'input' | 'output'; @@ -121,7 +122,7 @@ export interface CompiledContract { } export interface CompileContractOptions { - /** Parser-mapped model derived from the supplied validated IR. */ + /** Select exported names only. The compiler derives all types from validated IR. */ module: PythonModule; /** The generator is synchronous and performs no filesystem or runtime work. */ generator: Pick; @@ -567,6 +568,9 @@ function diagnosticForResolution( } function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { + if (left.kind === 'unsupported' || right.kind === 'unsupported') { + return true; + } if (left.kind === 'union') { return left.options.some(option => valuesMayOverlap(option, right)); } @@ -579,6 +583,16 @@ function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { ) { return true; } + if (left.kind === 'ndarray-float16' || right.kind === 'ndarray-float16' || + left.kind === 'torch-float16' || right.kind === 'torch-float16') { + return true; + } + if (left.kind === 'tuple' && right.kind === 'sequence') { + return left.items.every(item => valuesMayOverlap(item, right.item)); + } + if (left.kind === 'sequence' && right.kind === 'tuple') { + return right.items.every(item => valuesMayOverlap(left.item, item)); + } if (left.kind !== right.kind) { return false; } @@ -587,11 +601,50 @@ function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { left.items.every((item, index) => valuesMayOverlap(item, right.items[index]!)); } if (left.kind === 'sequence' && right.kind === 'sequence') { - return valuesMayOverlap(left.item, right.item); + return true; // An empty array matches both element schemas. } return true; } +function positionalSlot(parameters: readonly Parameter[], index: number): number | null { + const parameter = parameters[index]!; + if (parameter.keywordOnly || parameter.kwArgs || parameter.varArgs) { + return null; + } + let slot = 0; + for (const earlier of parameters.slice(0, index)) { + if (earlier.varArgs) { + return null; + } + if (!earlier.keywordOnly && !earlier.kwArgs) { + slot += 1; + } + } + return slot; +} + +function sharesRequiredBinding( + left: readonly Parameter[], + leftIndex: number, + right: readonly Parameter[], + rightIndex: number +): boolean { + const a = left[leftIndex]!; + const b = right[rightIndex]!; + if (a.optional || b.optional || a.varArgs || b.varArgs || a.kwArgs || b.kwArgs || + a.name !== b.name) { + return false; + } + if (a.keywordOnly && b.keywordOnly) { + return true; + } + if (a.keywordOnly || b.keywordOnly) { + return !a.positionalOnly && !b.positionalOnly; + } + const leftSlot = positionalSlot(left, leftIndex); + return leftSlot !== null && leftSlot === positionalSlot(right, rightIndex); +} + function overloadsMayOverlap( left: readonly ResolvedCallableValue[], right: readonly ResolvedCallableValue[], @@ -609,12 +662,30 @@ function overloadsMayOverlap( if (leftMax < rightMin || rightMax < leftMin) { return false; } - return left.slice(0, Math.min(left.length, right.length)).every((value, index) => { - const other = right[index]!; - return value.resolution.status !== 'supported' || - other.resolution.status !== 'supported' || - valuesMayOverlap(value.resolution.value, other.resolution.value); - }); + for (let leftIndex = 0; leftIndex < leftParameters.length; leftIndex += 1) { + if (leftParameters.filter(parameter => + parameter.name === leftParameters[leftIndex]!.name + ).length !== 1) { + continue; + } + const rightIndex = rightParameters.findIndex(parameter => + parameter.name === leftParameters[leftIndex]!.name + ); + if (rightIndex < 0 || rightParameters.filter(parameter => + parameter.name === leftParameters[leftIndex]!.name + ).length !== 1 || !sharesRequiredBinding( + leftParameters, leftIndex, rightParameters, rightIndex + )) { + continue; + } + const a = left[leftIndex]?.resolution; + const b = right[rightIndex]?.resolution; + if (a?.status === 'supported' && b?.status === 'supported' && + !valuesMayOverlap(a.value, b.value)) { + return false; + } + } + return true; } function resolveCallable( @@ -696,7 +767,18 @@ function resolveCallable( } } for (const value of values) { - if (containsNamedType(value.logicalType, dataclassNames)) { + const directDataclass = + (value.logicalType.kind === 'custom' || value.logicalType.kind === 'generic') && + dataclassNames.has(leafName(value.logicalType) ?? ''); + const adaptedDataclass = + value.direction === 'output' && + directDataclass && + capabilities.get('dataclass-adapter')?.available === true && + value.resolution.status === 'supported' && + value.resolution.value.kind === 'record'; + if (adaptedDataclass) { + requiredCapabilities.add('dataclass-adapter'); + } else if (containsNamedType(value.logicalType, dataclassNames)) { diagnostics.push({ severity: 'error', code: 'dataclass-unsupported', @@ -803,42 +885,39 @@ function resolveCallable( /** * Compile already-validated IR without filesystem, subprocess, cache, or - * registry access. `generate()` supplies the model and writes these files. + * registry access. `generate()` selects exports and writes these files. */ export function compileContract( ir: ValidatedIrContract, options: CompileContractOptions ): CompiledContract { - const matchesFunction = (source: IrFunction, mapped: PythonFunction): boolean => - source.name === mapped.name && - source.parameters.length === mapped.parameters.length && - source.parameters.every((parameter, index) => parameter.name === mapped.parameters[index]?.name) && - source.overloads.length === (mapped.overloads?.length ?? 0) && - source.overloads.every((overload, index) => - overload.parameters.length === mapped.overloads?.[index]?.parameters.length && - overload.parameters.every((parameter, parameterIndex) => - parameter.name === mapped.overloads?.[index]?.parameters[parameterIndex]?.name - ) - ); if (ir.module !== options.module.name) { throw new Error(`Compiled module ${options.module.name} does not match IR module ${ir.module}.`); } - for (const func of options.module.functions) { - const source = ir.functions.find(entry => entry.name === func.name); - if (!source || !matchesFunction(source, func)) { - throw new Error(`Compiled function ${func.name} does not match validated IR.`); - } - } - for (const cls of options.module.classes) { - const source = ir.classes.find(entry => entry.name === cls.name); - if (!source || cls.methods.some(method => - !source.methods.some(entry => - matchesFunction(entry, method) - ) - )) { - throw new Error(`Compiled class ${cls.name} does not match validated IR.`); - } - } + const canonical = transformIrToTsModel(ir); + const select = ( + kind: string, + source: readonly T[], + requested: readonly { name: string }[] + ): T[] => { + const byName = new Map(source.map(item => [item.name, item] as const)); + const seen = new Set(); + return requested.map(item => { + const match = byName.get(item.name); + if (!match || seen.has(item.name)) { + throw new Error(`Selected ${kind} ${item.name} does not match validated IR.`); + } + seen.add(item.name); + return match; + }); + }; + const selectedFunctions = select('function', canonical.functions, options.module.functions); + const selectedClasses = select('class', canonical.classes, options.module.classes); + const selectedAliases = select( + 'type alias', + canonical.typeAliases ?? [], + options.module.typeAliases ?? canonical.typeAliases ?? [] + ); const capabilities = new Map(options.capabilities.map(capability => [capability.name, capability])); const dataclassNames = new Set( ir.classes.filter(cls => cls.is_dataclass).map(cls => cls.name) @@ -858,11 +937,11 @@ export function compileContract( return result.function; }; const module: PythonModule = { - ...options.module, - functions: options.module.functions.map(func => + ...canonical, + functions: selectedFunctions.map(func => compileFunction(func, `$.functions[${ir.functions.findIndex(entry => entry.name === func.name)}]`) ), - classes: options.module.classes.map(cls => ({ + classes: selectedClasses.map(cls => ({ ...cls, methods: cls.methods.map(method => { const sourceClassIndex = ir.classes.findIndex(entry => entry.name === cls.name); @@ -875,6 +954,7 @@ export function compileContract( ); }), })), + typeAliases: selectedAliases, }; const generated = options.generator.generateModuleDefinition(module, options.annotatedJSDoc); return { diff --git a/src/core/ir-model.ts b/src/core/ir-model.ts new file mode 100644 index 00000000..f8e93384 --- /dev/null +++ b/src/core/ir-model.ts @@ -0,0 +1,165 @@ +/** Map validated Python IR to the generator model without runtime effects. */ + +import { parseAnnotationToPythonType } from './annotation-parser.js'; +import type { + IrClass, + IrFunction, + IrParameter, + IrTypeAlias, + IrTypeParameter, + ValidatedIrContract, +} from './ir-contract.js'; +import type { + Parameter, + PythonClass, + PythonFunction, + PythonGenericParameter, + PythonModule, + PythonType, + PythonTypeAlias, +} from '../types/index.js'; + +function collectModuleTypeVarNames(ir: ValidatedIrContract): Set { + const names = new Set(); + for (const constant of ir.constants) { + const name = constant.name.trim(); + if (!name) { + continue; + } + const valueRepr = constant.value_repr?.trim() ?? ''; + if (/^~?[A-Za-z_][A-Za-z0-9_]*$/.test(valueRepr) && valueRepr.replace(/^~/, '') === name) { + names.add(name); + continue; + } + const annotation = constant.annotation?.trim() ?? ''; + if (/(^|\.)(TypeVar|ParamSpec|TypeVarTuple)(\[|\(|$)/.test(annotation)) { + names.add(name); + } + } + return names; +} + +export function transformIrToTsModel( + ir: ValidatedIrContract, + onUnknownTypeName?: (name: string) => void +): PythonModule { + const moduleTypeVarNames = collectModuleTypeVarNames(ir); + const parseType = ( + annotation: unknown, + typeParameters: readonly PythonGenericParameter[] = [] + ): PythonType => + parseAnnotationToPythonType(annotation, { + onUnknownTypeName, + knownTypeVarNames: moduleTypeVarNames, + typeParameters, + }); + const mapTypeParameters = ( + typeParameters: readonly IrTypeParameter[] + ): PythonGenericParameter[] => + typeParameters.map(param => ({ + name: param.name, + kind: param.kind, + bound: param.bound ? parseType(param.bound) : undefined, + constraints: param.constraints ? param.constraints.map(item => parseType(item)) : undefined, + variance: param.variance ?? undefined, + })); + const mapParam = ( + param: IrParameter, + typeParameters: readonly PythonGenericParameter[] = [] + ): Parameter => ({ + name: param.name, + type: parseType(param.annotation, typeParameters), + optional: param.default, + varArgs: param.kind === 'VAR_POSITIONAL', + kwArgs: param.kind === 'VAR_KEYWORD', + positionalOnly: param.kind === 'POSITIONAL_ONLY', + keywordOnly: param.kind === 'KEYWORD_ONLY', + }); + const mapMethodKind = (value: unknown): PythonFunction['methodKind'] => + value === 'class' || value === 'static' ? value : 'instance'; + const mapFunc = ( + func: IrFunction, + inheritedTypeParameters: readonly PythonGenericParameter[] = [] + ): PythonFunction => { + const localTypeParameters = mapTypeParameters(func.type_params); + const annotationTypeParameters = [...inheritedTypeParameters, ...localTypeParameters]; + return { + name: func.name, + signature: { + parameters: func.parameters.map(param => mapParam(param, annotationTypeParameters)), + returnType: parseType(func.returns, annotationTypeParameters), + isAsync: func.is_async, + isGenerator: func.is_generator, + }, + docstring: func.docstring ?? undefined, + decorators: [], + isAsync: func.is_async, + isGenerator: func.is_generator, + typeParameters: [...localTypeParameters], + returnType: parseType(func.returns, annotationTypeParameters), + parameters: func.parameters.map(param => mapParam(param, annotationTypeParameters)), + overloads: func.overloads.map(overload => ({ + parameters: overload.parameters.map(param => mapParam(param, annotationTypeParameters)), + returnType: parseType(overload.returns, annotationTypeParameters), + })), + methodKind: mapMethodKind(func.method_kind), + }; + }; + const mapClass = (cls: IrClass): PythonClass => { + const classTypeParameters = mapTypeParameters(cls.type_params); + return { + name: cls.name, + bases: cls.bases, + methods: cls.methods.map(method => mapFunc(method, classTypeParameters)), + properties: cls.fields.map(field => ({ + name: field.name, + type: parseType(field.annotation, classTypeParameters), + readonly: false, + setter: false, + getter: true, + optional: field.default, + })), + accessors: cls.accessors.map(accessor => ({ + name: accessor.name, + type: parseType(accessor.returns, classTypeParameters), + docstring: accessor.docstring ?? undefined, + readOnly: accessor.read_only ?? undefined, + isCached: accessor.is_cached, + })), + docstring: cls.docstring ?? undefined, + decorators: cls.typed_dict ? ['__typed_dict__'] : [], + kind: cls.typed_dict + ? 'typed_dict' + : cls.is_protocol + ? 'protocol' + : cls.is_namedtuple + ? 'namedtuple' + : cls.is_dataclass + ? 'dataclass' + : cls.is_pydantic + ? 'pydantic' + : 'class', + typeParameters: classTypeParameters, + }; + }; + const mapTypeAlias = (alias: IrTypeAlias): PythonTypeAlias => { + const typeParameters = mapTypeParameters(alias.type_params); + return { + name: alias.name, + type: parseType(alias.definition, typeParameters), + typeParameters, + }; + }; + return { + name: ir.module, + path: undefined, + version: typeof ir.metadata.package_version === 'string' + ? ir.metadata.package_version + : undefined, + functions: ir.functions.map(mapFunc), + classes: ir.classes.map(mapClass), + typeAliases: ir.type_aliases.map(mapTypeAlias), + imports: [], + exports: [], + }; +} diff --git a/src/tywrap.ts b/src/tywrap.ts index d29d3103..172f8e10 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -11,25 +11,14 @@ import { import { TYWRAP_IR_VERSION, validateIrContract, - type IrClass, - type IrFunction, - type IrParameter, - type IrTypeAlias, - type IrTypeParameter, type ValidatedIrContract, } from './core/ir-contract.js'; +import { transformIrToTsModel } from './core/ir-model.js'; import { TypeMapper } from './core/mapper.js'; -import { parseAnnotationToPythonType } from './core/annotation-parser.js'; import { createConfig } from './config/index.js'; import type { TywrapOptions, - PythonGenericParameter, - PythonFunction, PythonModule as TSPythonModule, - PythonClass, - PythonTypeAlias, - Parameter, - PythonType, PythonModuleConfig, } from './types/index.js'; import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js'; @@ -465,7 +454,7 @@ export async function generate( } } - const moduleModel = transformIrToTsModel(ir); + const moduleModel = transformIrToTsModel(ir, recordUnknown); // Apply module-level export filtering (functions/classes + excludes). filterModuleExports(moduleModel, moduleConfig, moduleKey, warnings); @@ -632,170 +621,6 @@ async function fetchPythonIr( } } -/** - * Convert JSON IR from Python into the internal TypeScript model used by the generator. - */ -function collectModuleTypeVarNames(obj: Record): Set { - const names = new Set(); - const constants = Array.isArray(obj.constants) ? (obj.constants as unknown[]) : []; - - for (const constant of constants) { - const entry = constant as Record; - const name = typeof entry.name === 'string' ? entry.name.trim() : ''; - if (!name) { - continue; - } - - const valueRepr = typeof entry.value_repr === 'string' ? entry.value_repr.trim() : ''; - if (/^~?[A-Za-z_][A-Za-z0-9_]*$/.test(valueRepr) && valueRepr.replace(/^~/, '') === name) { - names.add(name); - continue; - } - - const annotation = typeof entry.annotation === 'string' ? entry.annotation.trim() : ''; - if (/(^|\.)(TypeVar|ParamSpec|TypeVarTuple)(\[|\(|$)/.test(annotation)) { - names.add(name); - } - } - - return names; -} - -function transformIrToTsModel(ir: ValidatedIrContract): TSPythonModule { - const obj = ir; - const functions = ir.functions; - const classes = ir.classes; - const aliases = ir.type_aliases; - const moduleTypeVarNames = collectModuleTypeVarNames(obj); - const parseType = ( - annotation: unknown, - typeParameters: readonly PythonGenericParameter[] = [] - ): PythonType => - parseAnnotationToPythonType(annotation, { - onUnknownTypeName: recordUnknown, - knownTypeVarNames: moduleTypeVarNames, - typeParameters, - }); - const mapTypeParameters = ( - typeParameters: readonly IrTypeParameter[] - ): PythonGenericParameter[] => - typeParameters.map(param => ({ - name: param.name, - kind: param.kind, - bound: param.bound ? parseType(param.bound) : undefined, - constraints: param.constraints ? param.constraints.map(item => parseType(item)) : undefined, - variance: param.variance ?? undefined, - })); - const mapParam = ( - p: IrParameter, - typeParameters: readonly PythonGenericParameter[] = [] - ): Parameter => ({ - name: p.name, - type: parseType(p.annotation, typeParameters), - optional: p.default, - varArgs: p.kind === 'VAR_POSITIONAL', - kwArgs: p.kind === 'VAR_KEYWORD', - positionalOnly: p.kind === 'POSITIONAL_ONLY', - keywordOnly: p.kind === 'KEYWORD_ONLY', - }); - - const mapMethodKind = (value: unknown): PythonFunction['methodKind'] => - value === 'class' || value === 'static' ? value : 'instance'; - - const mapFunc = ( - f: IrFunction, - inheritedTypeParameters: readonly PythonGenericParameter[] = [] - ): PythonFunction => { - const localTypeParameters = mapTypeParameters(f.type_params); - const annotationTypeParameters = [...inheritedTypeParameters, ...localTypeParameters]; - return { - name: String(f.name ?? ''), - signature: { - parameters: f.parameters.map(parameter => mapParam(parameter, annotationTypeParameters)), - returnType: parseType(f.returns, annotationTypeParameters), - isAsync: Boolean(f.is_async), - isGenerator: Boolean(f.is_generator), - }, - docstring: f.docstring ?? undefined, - decorators: [], - isAsync: Boolean(f.is_async), - isGenerator: Boolean(f.is_generator), - typeParameters: [...localTypeParameters], - returnType: parseType(f.returns, annotationTypeParameters), - parameters: f.parameters.map(parameter => mapParam(parameter, annotationTypeParameters)), - overloads: f.overloads.map(overload => ({ - parameters: overload.parameters.map(parameter => - mapParam(parameter, annotationTypeParameters) - ), - returnType: parseType(overload.returns, annotationTypeParameters), - })), - methodKind: mapMethodKind(f.method_kind), - }; - }; - - const mapClass = (c: IrClass): PythonClass => { - const classTypeParameters = mapTypeParameters(c.type_params); - return { - name: c.name, - bases: c.bases, - methods: c.methods.map(method => mapFunc(method, classTypeParameters)), - properties: c.fields.map(p => ({ - name: p.name, - type: parseType(p.annotation, classTypeParameters), - readonly: false, - setter: false, - getter: true, - optional: p.default, - })), - accessors: c.accessors.map(a => ({ - name: a.name, - type: parseType(a.returns, classTypeParameters), - docstring: a.docstring ?? undefined, - readOnly: a.read_only ?? undefined, - isCached: a.is_cached, - })), - docstring: c.docstring ?? undefined, - decorators: c.typed_dict ? ['__typed_dict__'] : [], - kind: c.typed_dict - ? 'typed_dict' - : c.is_protocol - ? 'protocol' - : c.is_namedtuple - ? 'namedtuple' - : c.is_dataclass - ? 'dataclass' - : c.is_pydantic - ? 'pydantic' - : 'class', - typeParameters: classTypeParameters, - }; - }; - - const mapTypeAlias = (alias: IrTypeAlias): PythonTypeAlias => { - const typeParameters = mapTypeParameters(alias.type_params); - return { - name: alias.name, - type: parseType(alias.definition, typeParameters), - typeParameters, - }; - }; - - const moduleModel: TSPythonModule = { - name: obj.module, - path: undefined, - version: - typeof obj.metadata.package_version === 'string' - ? obj.metadata.package_version - : undefined, - functions: functions.map(functionValue => mapFunc(functionValue)), - classes: classes.map(mapClass), - typeAliases: aliases.map(mapTypeAlias), - imports: [], - exports: [], - }; - return moduleModel; -} - /** * Compute a stable cache key filename for a module and options */ diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index e8c1e12b..e019f40e 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -375,6 +375,81 @@ describe('compileContract', () => { ])); }); + it('warns when keyword-only overloads overlap despite declaration order', () => { + const source = rawIr.functions[0]!; + const x = { name: 'x', kind: 'KEYWORD_ONLY', annotation: 'str', default: false }; + const y = { name: 'y', kind: 'KEYWORD_ONLY', annotation: 'int', default: false }; + const ir = validateIrContract({ + ...rawIr, + functions: [{ + ...source, + name: 'choose', qualname: 'fixture.choose', + parameters: [x, y], + overloads: [ + { parameters: [x, y], returns: 'str' }, + { parameters: [y, x], returns: 'int' }, + ], + }], + classes: [], + }, 'keyword overload contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: [{ ...moduleModel.functions[0]!, name: 'choose' }], + classes: [], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: 'overload-ambiguous', path: '$.functions[0].overloads[1]' }), + ])); + }); + + it('takes callable types from validated IR, not from the export selection model', () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract({ + ...rawIr, + functions: [{ + ...source, + name: 'echo', qualname: 'fixture.echo', + parameters: [{ name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }], + returns: 'str', + }], + classes: [], + }, 'IR authority contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const forged = { + ...moduleModel.functions[1]!, + name: 'echo', + parameters: [{ + name: 'value', type: { kind: 'primitive', name: 'int' } as PythonType, + optional: false, varArgs: false, kwArgs: false, + }], + returnType: { kind: 'primitive', name: 'int' } as PythonType, + }; + const compiled = compileContract(ir.contract, { + module: { ...moduleModel, functions: [forged], classes: [] }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.module.functions[0]?.parameters[0]?.type).toEqual({ kind: 'primitive', name: 'str' }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', value: { kind: 'string' }, + }); + expect(compiled.generated.declaration).toContain('echo(value: string): Promise'); + expect(compiled.generated.declaration).not.toContain('echo(value: number)'); + }); + it('selects a class method overload without validating its implicit cls receiver', async () => { const source = rawIr.functions[0]!; const receiver = { From fda3d6d4eddb1d671a79d5490077081e804630e4 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:41:27 -0700 Subject: [PATCH 30/50] fix(ir): avoid array index as inherited type parameters --- src/core/ir-model.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ir-model.ts b/src/core/ir-model.ts index f8e93384..74214921 100644 --- a/src/core/ir-model.ts +++ b/src/core/ir-model.ts @@ -156,7 +156,7 @@ export function transformIrToTsModel( version: typeof ir.metadata.package_version === 'string' ? ir.metadata.package_version : undefined, - functions: ir.functions.map(mapFunc), + functions: ir.functions.map(func => mapFunc(func)), classes: ir.classes.map(mapClass), typeAliases: ir.type_aliases.map(mapTypeAlias), imports: [], From 5d0b3add33c8d9a1d6625afc512bc6ba7489c39f Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:47:13 -0700 Subject: [PATCH 31/50] fix(scientific): resolve NumPy float16 dtype from expanded annotation --- docs/maintainers/callable-contracts.md | 2 + src/core/annotation-parser.ts | 20 ++++++++- src/core/callable-compiler.ts | 11 ++++- src/core/generator.ts | 60 +++++++++++++++++++++----- src/tywrap.ts | 51 ++++++++++++++++------ test/annotation-parser.test.ts | 21 +++++++++ test/callable-compiler.test.ts | 16 +++++++ test/ir_contract.test.ts | 38 ++++++++++++++++ 8 files changed, 193 insertions(+), 26 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index b2b3c653..1e3393f4 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -26,6 +26,8 @@ Each value conversion uses revision 2 of [value-contracts.v2.json](value-contrac The compiler requires an explicit float16 dtype. An untyped ndarray or Torch tensor stays unresolved. +NumPy can spell `NDArray[np.float16]` as `numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]`. The compiler reads the dtype argument, not the shape wildcard. Only a supported scientific result clears its matching external-annotation IR warning. Other external annotations remain warnings. + The codec records scalar ndarray dtype and rank for one call. A generated return validator accepts a scalar only with matching proof. Nested proof follows its parent and array index or record key. Array index `0` and `"0"` refer to the same child. Equal numbers from separate calls do not share proof. Array and tensor objects keep their identity-based marker proof. The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. diff --git a/src/core/annotation-parser.ts b/src/core/annotation-parser.ts index 5dfc2864..a636f7d6 100644 --- a/src/core/annotation-parser.ts +++ b/src/core/annotation-parser.ts @@ -16,10 +16,14 @@ export function parseAnnotationToPythonType( (options.typeParameters ?? []).map(param => [param.name, param] as const) ); const modulePrefixes = ['', 'typing.', 'typing_extensions.', 'collections.abc.'] as const; + let numpyShapeDepth = 0; const unknownType = (): PythonType => ({ kind: 'custom', name: 'Any', module: 'typing' }); const recordUnknown = (name: string): void => { + if (numpyShapeDepth > 0) { + return; + } try { onUnknownTypeName?.(name); } catch { @@ -495,8 +499,22 @@ export function parseAnnotationToPythonType( const generic = splitGenericInvocation(raw); if (generic) { - const typeArgs = splitTopLevel(generic.inner, ',').map(part => parse(part.trim(), depth + 1)); const qualified = splitQualifiedName(generic.name); + const parts = splitTopLevel(generic.inner, ','); + const numpyArrayShape = qualified.module === 'numpy' && + qualified.name === 'ndarray' && parts.length === 2; + const typeArgs = parts.map((part, index) => { + if (!numpyArrayShape || index !== 0) { + return parse(part.trim(), depth + 1); + } + // NumPy's first ndarray argument describes shape, not decoded value type. + numpyShapeDepth += 1; + try { + return parse(part.trim(), depth + 1); + } finally { + numpyShapeDepth -= 1; + } + }); return { kind: 'generic', name: qualified.name, diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index f3912af9..2c525b75 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -321,7 +321,16 @@ function resolveUnion( } function resolveNdarray(type: PythonType): ValueResolution { - const typeArgument = type.kind === 'generic' ? type.typeArgs[0] : undefined; + const typeArgument = type.kind === 'generic' && type.typeArgs.length === 2 && + type.typeArgs[1]?.kind === 'generic' && + type.typeArgs[1].name === 'dtype' && + type.typeArgs[1].module === 'numpy' && + (type.typeArgs[0]?.kind === 'collection' || + (type.typeArgs[0]?.kind === 'custom' && type.typeArgs[0].name === 'Any')) + ? type.typeArgs[1].typeArgs[0] + : type.kind === 'generic' && type.typeArgs.length === 1 + ? type.typeArgs[0] + : undefined; const dtype = typeArgument ? leafName(typeArgument) : undefined; const normalizedDtype = dtype?.toLowerCase(); if (normalizedDtype !== 'float16') { diff --git a/src/core/generator.ts b/src/core/generator.ts index 1f718676..d616b9c8 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -394,6 +394,20 @@ export class CodeGenerator { ); } + private resolvedReturnType( + contract: ValueContract | undefined, + logicalType: PythonType, + ctx: GenericRenderContext + ): string { + if (contract?.kind === 'ndarray-float16') { + return '__tywrapFloat16Value'; + } + if (contract?.kind === 'torch-float16') { + return '__tywrapFloat16Tensor'; + } + return this.typeToTsFromPython(logicalType, ctx, 'return'); + } + private isLocalTypeIdentity( type: { name: string; module?: string }, ctx: GenericRenderContext @@ -767,14 +781,21 @@ export class CodeGenerator { const paramDecl = implParams.join(', '); const hasKwArgs = needsKwargsParam; - const returnType = this.typeToTsFromPython(func.returnType, genericContext, 'return'); + const returnType = this.resolvedReturnType( + func.callableContract?.returnValue, + func.returnType, + genericContext + ); const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const fname = this.escapeIdentifier(func.name); const moduleId = moduleName ?? '__main__'; const validatorName = `__validate${this.escapeIdentifier(func.name, { preserveCase: true })}Result`; const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(func, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; - const renderDeclaredOverload = (overload: NonNullable[number]): string[] => { + const renderDeclaredOverload = ( + overload: NonNullable[number], + overloadIndex: number + ): string[] => { const overloadParams = overload.parameters.filter( parameter => parameter.name !== 'self' && parameter.name !== 'cls' ); @@ -817,10 +838,10 @@ export class CodeGenerator { overloadNeedsKwargs ? `kwargs${forceRequired ? '' : '?'}: ${overloadKwargsType}` : null; - const overloadReturnType = this.typeToTsFromPython( + const overloadReturnType = this.resolvedReturnType( + func.callableContract?.overloads[overloadIndex]?.returnValue, overload.returnType, - genericContext, - 'return' + genericContext ); const signatures: string[] = []; const addSignature = (parameters: string[]): void => { @@ -1169,10 +1190,10 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${im const paramsDecl = paramsDeclParts.join(', '); const requiredKwOnlyNames = keywordOnlyParams.filter(p => !p.optional).map(p => p.name); - const returnType = this.typeToTsFromPython( + const returnType = this.resolvedReturnType( + method.callableContract?.returnValue, method.returnType, - methodGenericContext, - 'return' + methodGenericContext ); const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const mname = this.escapeIdentifier(method.name); @@ -1369,12 +1390,31 @@ ${migrationNote}${declarationMethodsSection} module.classes.some(cls => cls.methods.some(method => (method.overloads?.length ?? 0) > 0) ); + const emittedCallables = [ + ...module.functions, + ...module.classes.flatMap(cls => cls.methods.filter(method => + method.name !== '__init__' && + (method.methodKind === 'class' || method.methodKind === 'static') + )), + ]; + const resolvedReturns = emittedCallables.flatMap(func => [ + func.callableContract?.returnValue, + ...(func.callableContract?.overloads.map(overload => overload.returnValue) ?? []), + ]); + const needsTorchFloat16 = resolvedReturns.some(value => value?.kind === 'torch-float16'); + const needsFloat16 = needsTorchFloat16 || + resolvedReturns.some(value => value?.kind === 'ndarray-float16'); + const scientificTypes = needsFloat16 + ? `type __tywrapFloat16Value = number | __tywrapFloat16Value[];\n${needsTorchFloat16 + ? `type __tywrapFloat16Tensor = { data: __tywrapFloat16Value; shape: number[]; dtype: 'torch.float16'; device?: string; sourceDtype?: string; sourceDevice?: string };\n` + : ''}\n` + : ''; const bridgeDecl = needsRuntime ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` : ''; - const ts = `${`${header}${bridgeDecl}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; - const declaration = `${`${declarationHeader}${functionResults + const ts = `${`${header}${bridgeDecl}${scientificTypes}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; + const declaration = `${`${declarationHeader}${scientificTypes}${functionResults .map(result => result.declaration) .join('\n')}\n${classResults .map(result => result.declaration) diff --git a/src/tywrap.ts b/src/tywrap.ts index 172f8e10..2f32f773 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -439,20 +439,6 @@ export async function generate( const irWarnings = Array.isArray((ir as Record).warnings) ? ((ir as Record).warnings as unknown[]) : []; - for (const warning of irWarnings) { - if (typeof warning !== 'string') { - continue; - } - // Only type-honesty degrades feed --fail-on-warn. Environment capability - // notices (e.g. typing.get_overloads missing on Python < 3.11) describe - // the analyzing interpreter, not the generated types, and must not fail - // an otherwise clean build. - if (warning.startsWith('Return annotation for ')) { - recordUnknown(`Python IR warning: ${warning}`); - } else { - logger.info(`Python IR notice: ${warning}`, { component: 'Generate' }); - } - } const moduleModel = transformIrToTsModel(ir, recordUnknown); @@ -472,6 +458,43 @@ export async function generate( for (const diagnostic of compiled.diagnostics) { warnings.push(diagnostic.message); } + for (const warning of irWarnings) { + if (typeof warning !== 'string') { + continue; + } + const external = warning.match( + /^Return annotation for (.+) resolves outside analyzed module: (.+)\.$/ + ); + if (external) { + const qualname = external[1]; + const candidate = external[2]; + const functionIndex = ir.functions.findIndex(func => func.qualname === qualname); + const classIndex = ir.classes.findIndex(cls => + cls.methods.some(method => method.qualname === qualname) + ); + const methodIndex = classIndex < 0 ? -1 : ir.classes[classIndex]!.methods.findIndex( + method => method.qualname === qualname + ); + const path = functionIndex >= 0 + ? `$.functions[${functionIndex}]` + : methodIndex >= 0 + ? `$.classes[${classIndex}].methods[${methodIndex}]` + : ''; + const result = compiled.callables.find(callable => callable.path === path)?.result.resolution; + if (result?.status === 'supported' && ( + (result.value.kind === 'ndarray-float16' && candidate?.startsWith('numpy.')) || + (result.value.kind === 'torch-float16' && candidate?.startsWith('torch.')) + )) { + continue; + } + } + // Runtime notices do not fail strict mode. Type-honesty warnings do. + if (warning.startsWith('Return annotation for ')) { + recordUnknown(`Python IR warning: ${warning}`); + } else { + logger.info(`Python IR notice: ${warning}`, { component: 'Generate' }); + } + } const baseName = moduleModel.name || 'module'; const filesToEmit: Array<{ path: string; content: string }> = [ diff --git a/test/annotation-parser.test.ts b/test/annotation-parser.test.ts index 9d87863e..8226434a 100644 --- a/test/annotation-parser.test.ts +++ b/test/annotation-parser.test.ts @@ -68,4 +68,25 @@ describe('annotation parser', () => { expect((t as any).name).toBe('args'); expect((t as any).module).toBe('Request'); }); + + it('keeps NumPy dtype detail without warning on its shape wildcard', () => { + const unknown: string[] = []; + const type = parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', + { onUnknownTypeName: name => unknown.push(name) } + ); + expect(type).toMatchObject({ + kind: 'generic', name: 'ndarray', module: 'numpy', + typeArgs: [ + { kind: 'collection', name: 'tuple' }, + { kind: 'generic', name: 'dtype', module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }] }, + ], + }); + expect(unknown).toEqual([]); + parseAnnotationToPythonType('list[typing.Any]', { + onUnknownTypeName: name => unknown.push(name), + }); + expect(unknown).toEqual(['Any']); + }); }); diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index e019f40e..59860a3a 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -11,6 +11,7 @@ import { DEFAULT_VALUE_CONVERSION, } from '../src/core/callable-compiler.js'; import { CodeGenerator } from '../src/core/generator.js'; +import { parseAnnotationToPythonType } from '../src/core/annotation-parser.js'; import { validateIrContract } from '../src/core/ir-contract.js'; import { BridgeValidationError } from '../src/runtime/errors.js'; import { HttpBridge } from '../src/runtime/http.js'; @@ -590,6 +591,9 @@ describe('compileContract', () => { capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); expect(compiled.generated.typescript).toContain('"marker":"ndarray","dtype":"float16"'); + expect(compiled.generated.declaration).toContain( + 'scalarValue(): Promise<__tywrapFloat16Value>' + ); let dtype = 'float16'; let requestId = 0; @@ -689,6 +693,15 @@ describe('compileContract', () => { status: 'supported', value: { kind: 'ndarray-float16', dtype: 'float16' }, }); + expect(resolve(parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]' + ))).toMatchObject({ + status: 'supported', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }); + expect(resolve(parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float32]]' + ))).toMatchObject({ status: 'unresolved' }); }); it('keeps the existing bytes RPC shape in the value contract', () => { @@ -730,6 +743,9 @@ describe('compileContract', () => { }], }); expect(generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); + expect(generated.declaration).toContain( + 'tensorValue(): Promise<__tywrapFloat16Tensor>' + ); }); it('rejects a Torch float16 response with a mismatched nested dtype', async () => { diff --git a/test/ir_contract.test.ts b/test/ir_contract.test.ts index cf57e5da..80be1a74 100644 --- a/test/ir_contract.test.ts +++ b/test/ir_contract.test.ts @@ -96,6 +96,44 @@ describe('pinned IR contracts', () => { } }); + it('keeps a supported NumPy float16 contract strict in offline and check modes', async () => { + const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-ir-float16-')); + try { + const contractPath = join(tempDir, 'scientific.contract.json'); + await writeFile(contractPath, JSON.stringify({ + ir_version: '0.4.0', module: 'scientific_fixture', + functions: [{ + name: 'value', qualname: 'scientific_fixture.value', docstring: null, + parameters: [], + returns: 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', + is_async: false, is_generator: false, type_params: [], + method_kind: 'instance', overloads: [], + }], + classes: [], constants: [], type_aliases: [], metadata: {}, + warnings: [ + 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.ndarray.', + ], + }), 'utf8'); + const configured = { + ...options(join(tempDir, 'generated')), + pythonModules: { scientific_fixture: { typeHints: 'strict' as const } }, + contractInput: contractPath, + }; + const first = await generate(configured); + expect(first.failures).toEqual([]); + expect(first.warnings).toEqual([]); + const emitted = await readFile(join(tempDir, 'generated', 'scientific_fixture.generated.ts'), 'utf8'); + expect(emitted).toContain('value(): Promise<__tywrapFloat16Value>'); + expect(emitted).toContain('"marker":"ndarray","dtype":"float16"'); + const check = await generate(configured, { check: true }); + expect(check.failures).toEqual([]); + expect(check.warnings).toEqual([]); + expect(check.outOfDate).toEqual([]); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + // Windows cannot run the fake-interpreter shim: extensionless sh scripts // fail with ENOENT and .cmd files fail with EINVAL (Node's batch-file spawn // mitigation; the production spawn rightly never sets shell:true). The From 96041aa1c0a87d306846e06c331b7038eef5877b Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:49:30 -0700 Subject: [PATCH 32/50] test: preserve strict NumPy warning control and required dataclass fields --- src/core/ir-model.ts | 3 ++- test/annotation-parser.test.ts | 6 +++++- test/callable-compiler.test.ts | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/core/ir-model.ts b/src/core/ir-model.ts index 74214921..7ef07590 100644 --- a/src/core/ir-model.ts +++ b/src/core/ir-model.ts @@ -117,7 +117,8 @@ export function transformIrToTsModel( readonly: false, setter: false, getter: true, - optional: field.default, + // A constructor default does not make the returned dataclass field absent. + optional: cls.is_dataclass ? false : field.default, })), accessors: cls.accessors.map(accessor => ({ name: accessor.name, diff --git a/test/annotation-parser.test.ts b/test/annotation-parser.test.ts index 8226434a..daad8a50 100644 --- a/test/annotation-parser.test.ts +++ b/test/annotation-parser.test.ts @@ -84,7 +84,11 @@ describe('annotation parser', () => { ], }); expect(unknown).toEqual([]); - parseAnnotationToPythonType('list[typing.Any]', { + parseAnnotationToPythonType('numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]', { + onUnknownTypeName: name => unknown.push(name), + }); + expect(unknown).toEqual([]); + parseAnnotationToPythonType('list[Any]', { onUnknownTypeName: name => unknown.push(name), }); expect(unknown).toEqual(['Any']); diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 59860a3a..60552f80 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -860,4 +860,28 @@ describe('compileContract', () => { module: 'typing', }); }); + + it('requires returned dataclass fields even when their constructor has defaults', () => { + const point = rawIr.classes[0]!; + const ir = validateIrContract({ + ...rawIr, + functions: [], + classes: [{ + ...point, + fields: [{ ...point.fields[0]!, default: true }, point.fields[1]!], + }], + }, 'Point default contract'); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { ...moduleModel, functions: [] }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.declaration).toContain('x: number;'); + expect(compiled.generated.declaration).not.toContain('x?: number;'); + }); }); From 461ff6a39560208aa3b3ad82a8253006d6d1e555 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:51:03 -0700 Subject: [PATCH 33/50] fix(scientific): require NumPy and Torch dtype identity --- src/core/callable-compiler.ts | 12 ++++++++---- test/callable-compiler.test.ts | 13 +++++++++++++ test/ir_contract.test.ts | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 2c525b75..cd99e57c 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -331,9 +331,8 @@ function resolveNdarray(type: PythonType): ValueResolution { : type.kind === 'generic' && type.typeArgs.length === 1 ? type.typeArgs[0] : undefined; - const dtype = typeArgument ? leafName(typeArgument) : undefined; - const normalizedDtype = dtype?.toLowerCase(); - if (normalizedDtype !== 'float16') { + if (typeArgument?.kind !== 'custom' || typeArgument.name !== 'float16' || + typeArgument.module !== 'numpy') { return { status: 'unresolved', annotation: annotationName(type), @@ -444,11 +443,16 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { return resolveNdarray(type); } if (leaf === 'Tensor' && type.module?.startsWith('torch')) { + const tensorDtype = type.typeArgs[0]; + if (type.typeArgs.length !== 1 || tensorDtype?.kind !== 'custom' || + tensorDtype.name !== 'float16' || tensorDtype.module !== 'torch') { + return { status: 'unresolved', annotation: annotationName(type) }; + } const ndarray = resolveNdarray({ kind: 'generic', name: 'ndarray', module: 'numpy', - typeArgs: type.typeArgs, + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], }); if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { return ndarray; diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 60552f80..5bd900f6 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -702,6 +702,12 @@ describe('compileContract', () => { expect(resolve(parseAnnotationToPythonType( 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float32]]' ))).toMatchObject({ status: 'unresolved' }); + expect(resolve(parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]' + ))).toMatchObject({ status: 'unresolved' }); + expect(resolve(parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[torch.float16]]' + ))).toMatchObject({ status: 'unresolved' }); }); it('keeps the existing bytes RPC shape in the value contract', () => { @@ -746,6 +752,13 @@ describe('compileContract', () => { expect(generated.declaration).toContain( 'tensorValue(): Promise<__tywrapFloat16Tensor>' ); + expect(DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', path: '$.functions[0].returns', + logicalType: { + ...tensorType, + typeArgs: [{ kind: 'custom', name: 'float16', module: 'external' }], + }, + })).toMatchObject({ status: 'unresolved' }); }); it('rejects a Torch float16 response with a mismatched nested dtype', async () => { diff --git a/test/ir_contract.test.ts b/test/ir_contract.test.ts index 80be1a74..7d0701b3 100644 --- a/test/ir_contract.test.ts +++ b/test/ir_contract.test.ts @@ -129,6 +129,20 @@ describe('pinned IR contracts', () => { expect(check.failures).toEqual([]); expect(check.warnings).toEqual([]); expect(check.outOfDate).toEqual([]); + + const foreign = JSON.parse(await readFile(contractPath, 'utf8')) as { + functions: Array<{ returns: string }>; + }; + foreign.functions[0]!.returns = + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]'; + await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); + const rejected = await generate(configured); + expect(rejected.warnings.some(warning => + warning.includes('no value conversion is defined for numpy.ndarray') + )).toBe(true); + expect(rejected.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.ndarray') + )).toBe(true); } finally { await rm(tempDir, { recursive: true, force: true }); } From ecba0e03b924149120ad16cb85a87f8912c4250f Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:51:04 -0700 Subject: [PATCH 34/50] test: typecheck generated float16 scalar and array results --- .../architecture-overload.typecheck.ts.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/fixtures/architecture-overload.typecheck.ts.txt b/test/fixtures/architecture-overload.typecheck.ts.txt index 8e18d6ce..7581b02a 100644 --- a/test/fixtures/architecture-overload.typecheck.ts.txt +++ b/test/fixtures/architecture-overload.typecheck.ts.txt @@ -1,4 +1,15 @@ import { selectRecord } from './generated/architecture_callable_fixture.generated.js'; +import { float16Values } from './generated/consumer_fixture.generated.js'; + +type Float16Value = number | Float16Value[]; +type GeneratedFloat16Value = Awaited>; + +const float16Result: Promise = float16Values(); +const acceptsScalar: number extends GeneratedFloat16Value ? true : never = true; +const acceptsArray: number[] extends GeneratedFloat16Value ? true : never = true; + +// @ts-expect-error A decoded float16 array is not a string. +const wrongFloat16Result: Promise = float16Values(); const textRecord: Promise>> = selectRecord('label'); const integerRecord: Promise>> = selectRecord(42); @@ -16,3 +27,7 @@ void textRecord; void integerRecord; void wrongTextRecord; void wrongIntegerRecord; +void float16Result; +void wrongFloat16Result; +void acceptsScalar; +void acceptsArray; From 6f82ccb201b8ecaca94ab330c5a4cfce6288ae88 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:52:01 -0700 Subject: [PATCH 35/50] fix(ir): clear only matching scientific annotation warnings --- src/tywrap.ts | 16 ++++++++++++---- test/ir_contract.test.ts | 11 +++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/tywrap.ts b/src/tywrap.ts index 2f32f773..3731dc33 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -480,10 +480,18 @@ export async function generate( : methodIndex >= 0 ? `$.classes[${classIndex}].methods[${methodIndex}]` : ''; - const result = compiled.callables.find(callable => callable.path === path)?.result.resolution; - if (result?.status === 'supported' && ( - (result.value.kind === 'ndarray-float16' && candidate?.startsWith('numpy.')) || - (result.value.kind === 'torch-float16' && candidate?.startsWith('torch.')) + const callableResult = compiled.callables.find(callable => callable.path === path)?.result; + const logicalType = callableResult?.logicalType; + const outerName = logicalType && + (logicalType.kind === 'custom' || logicalType.kind === 'generic') && + logicalType.module + ? `${logicalType.module}.${logicalType.name}` + : undefined; + const resolution = callableResult?.resolution; + if (resolution?.status === 'supported' && candidate === outerName && ( + (resolution.value.kind === 'ndarray-float16' && + (candidate === 'numpy.ndarray' || candidate === 'numpy.typing.NDArray')) || + (resolution.value.kind === 'torch-float16' && candidate === 'torch.Tensor') )) { continue; } diff --git a/test/ir_contract.test.ts b/test/ir_contract.test.ts index 7d0701b3..48d81678 100644 --- a/test/ir_contract.test.ts +++ b/test/ir_contract.test.ts @@ -132,6 +132,7 @@ describe('pinned IR contracts', () => { const foreign = JSON.parse(await readFile(contractPath, 'utf8')) as { functions: Array<{ returns: string }>; + warnings: string[]; }; foreign.functions[0]!.returns = 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]'; @@ -143,6 +144,16 @@ describe('pinned IR contracts', () => { expect(rejected.warnings.some(warning => warning.includes('resolves outside analyzed module: numpy.ndarray') )).toBe(true); + + foreign.functions[0]!.returns = + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]'; + foreign.warnings[0] = + 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.Opaque.'; + await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); + const unrelated = await generate(configured); + expect(unrelated.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.Opaque') + )).toBe(true); } finally { await rm(tempDir, { recursive: true, force: true }); } From 0006a3f64a83acb501adaff9a28024425921f484 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:53:57 -0700 Subject: [PATCH 36/50] feat(generator): expose isolated runtime binding template --- src/core/generator.ts | 52 +++++++++++++++++++++++++++++++++++------- test/generator.test.ts | 28 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/core/generator.ts b/src/core/generator.ts index d616b9c8..75ffbbe7 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -146,6 +146,13 @@ export class CodeGenerator { this.onTypeDegrade = options.onTypeDegrade; } + private assertRuntimeGetterIdentifier(identifier: string): void { + if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier) || + this.reservedTsIdentifiers.has(identifier)) { + throw new Error(`Invalid runtime getter identifier: ${identifier}`); + } + } + /** * Convert Python snake_case to TypeScript camelCase and escape reserved words */ @@ -685,8 +692,10 @@ export class CodeGenerator { moduleName?: string, annotatedJSDoc = false, localDeclaredNames: Set = new Set(), - returnDefinitions: ReturnDefinitionNames = new Set() + returnDefinitions: ReturnDefinitionNames = new Set(), + runtimeGetterIdentifier = 'getRuntimeBridge' ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const jsdoc = this.generateJsDoc( func.docstring, annotatedJSDoc ? func.parameters.map(p => String(p.type)) : undefined @@ -989,7 +998,7 @@ export class CodeGenerator { const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${hasDeclaredOverloads ? '' : typeParamDecl}(${paramDecl}): Promise<${implementationReturnType}> { -${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${implementationReturnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); +${callPrelude}${guards}${overloadValidator} return ${runtimeGetterIdentifier}().call<${implementationReturnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); } `; @@ -1006,8 +1015,10 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${im moduleName?: string, _annotatedJSDoc = false, localDeclaredNames: Set = new Set(), - returnDefinitions: ReturnDefinitionNames = new Set() + returnDefinitions: ReturnDefinitionNames = new Set(), + runtimeGetterIdentifier = 'getRuntimeBridge' ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const moduleDeclaredNames = new Set(localDeclaredNames); moduleDeclaredNames.add(cls.name); const jsdoc = this.generateJsDoc(cls.docstring); @@ -1213,7 +1224,8 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${im moduleName, false, moduleDeclaredNames, - returnDefinitions + returnDefinitions, + runtimeGetterIdentifier ).declaration.match(/^export function .*;$/gm)?.map(line => ` static ${line.slice('export function '.length)}` ) ?? [] @@ -1268,7 +1280,7 @@ ${callPrelude}${guards}${overloadValidator} return getRuntimeBridge().call<${im : ''; const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; - const callExpr = `getRuntimeBridge().call<${implementationReturnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; + const callExpr = `${runtimeGetterIdentifier}().call<${implementationReturnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${hasDeclaredOverloads ? '' : methodTypeParamDecl}(${paramsDecl}): Promise<${implementationReturnType}> { ${returnValidator}${callPrelude}${guards}${overloadValidator} return ${callExpr}; }`); @@ -1344,6 +1356,28 @@ ${migrationNote}${declarationMethodsSection} } generateModuleDefinition(module: PythonModule, annotatedJSDoc = false): GeneratedCode { + return this.generateModuleWithGetter(module, annotatedJSDoc, 'getRuntimeBridge', true); + } + + /** Build an intermediate template. The caller must bind its provider before emission. */ + generateModuleBindingTemplate( + module: PythonModule, + annotatedJSDoc = false, + runtimeGetterIdentifier = '__tywrapRuntimeProvider' + ): GeneratedCode { + if (runtimeGetterIdentifier === 'getRuntimeBridge') { + throw new Error('Binding templates require a distinct runtime getter'); + } + return this.generateModuleWithGetter(module, annotatedJSDoc, runtimeGetterIdentifier, false); + } + + private generateModuleWithGetter( + module: PythonModule, + annotatedJSDoc: boolean, + runtimeGetterIdentifier: string, + importRuntimeGetter: boolean + ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const localDeclaredNames = new Set([ ...module.classes.map(cls => cls.name), ...(module.typeAliases ?? []).map(alias => alias.name), @@ -1356,7 +1390,8 @@ ${migrationNote}${declarationMethodsSection} module.name, annotatedJSDoc, localDeclaredNames, - this.returnDefinitions(module) + this.returnDefinitions(module), + runtimeGetterIdentifier ) ); const classResults = [...module.classes] @@ -1367,7 +1402,8 @@ ${migrationNote}${declarationMethodsSection} module.name, annotatedJSDoc, localDeclaredNames, - this.returnDefinitions(module) + this.returnDefinitions(module), + runtimeGetterIdentifier ) ); const typeAliasResults = [...(module.typeAliases ?? [])] @@ -1410,7 +1446,7 @@ ${migrationNote}${declarationMethodsSection} : ''}\n` : ''; const bridgeDecl = needsRuntime - ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` + ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}${importRuntimeGetter ? 'getRuntimeBridge, ' : ''}type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` : ''; const ts = `${`${header}${bridgeDecl}${scientificTypes}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; diff --git a/test/generator.test.ts b/test/generator.test.ts index 8816ada9..6a7fe588 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -59,6 +59,34 @@ describe('CodeGenerator', () => { expect(code.typescript).toContain('createReturnValidator'); }); + it('uses one opt-in runtime provider for free and static wrappers', () => { + const ping = { + name: 'ping', + signature: { parameters: [], returnType: { kind: 'primitive', name: 'int' }, + isAsync: false, isGenerator: false }, + decorators: [], isAsync: false, isGenerator: false, + returnType: { kind: 'primitive', name: 'int' }, parameters: [], + } as any; + const module = { + name: 'binding_fixture', functions: [ping], + classes: [{ + name: 'Box', kind: 'class', bases: [], properties: [], decorators: [], + methods: [{ ...ping, name: 'pong', methodKind: 'static' }], + }], + typeAliases: [], imports: [], exports: [], + } as any; + const normal = gen.generateModuleDefinition(module); + const bound = gen.generateModuleBindingTemplate(module); + expect(normal.typescript).toContain("getRuntimeBridge().call('binding_fixture', 'ping'"); + expect(normal.typescript).toContain("getRuntimeBridge().call('binding_fixture', 'Box.pong'"); + expect(bound.typescript).toContain("__tywrapRuntimeProvider().call('binding_fixture', 'ping'"); + expect(bound.typescript).toContain("__tywrapRuntimeProvider().call('binding_fixture', 'Box.pong'"); + expect(bound.typescript).not.toContain('getRuntimeBridge'); + expect(bound.declaration).toBe(normal.declaration); + expect(() => gen.generateModuleBindingTemplate(module, false, 'provider;inject')) + .toThrow(/Invalid runtime getter identifier/); + }); + it('emits a null union member for X | None returns, not an accept-everything any', () => { const code = gen.generateFunctionWrapper( { From 6a72dced38469bb06c4ebb89f4c850984a9de5a0 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:59:16 -0700 Subject: [PATCH 37/50] fix(generator): isolate binding provider namespace --- src/core/generator.ts | 49 ++++++++++++++++++++++++++++++++++++++++-- test/generator.test.ts | 11 ++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/core/generator.ts b/src/core/generator.ts index 75ffbbe7..ff112b6c 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -147,12 +147,55 @@ export class CodeGenerator { } private assertRuntimeGetterIdentifier(identifier: string): void { - if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier) || - this.reservedTsIdentifiers.has(identifier)) { + if (identifier !== 'getRuntimeBridge' && + !/^__tywrapRuntimeProvider(?:[1-9][0-9]*)?$/.test(identifier)) { throw new Error(`Invalid runtime getter identifier: ${identifier}`); } } + private assertBindingGetterAvailable(module: PythonModule, identifier: string): void { + const occupied = new Set([ + 'getRuntimeBridge', + 'createReturnValidator', + 'selectOverloadReturnValidator', + 'ReturnSchema', + '__tywrapReturnDefinitions', + '__tywrapFloat16Value', + '__tywrapFloat16Tensor', + '__tywrapCreateApi', + '__args', + '__kwargs', + '__candidate', + '__varargs', + '__positionalOnly', + '__requiredKwOnly', + '__missing', + '__selectedReturnValidator', + 'kwargs', + 'key', + ]); + const includeCallable = (func: PythonFunction, owner = ''): void => { + occupied.add(this.escapeIdentifier(func.name)); + occupied.add(`__validate${owner}${this.escapeIdentifier(func.name, { preserveCase: true })}Result`); + for (const parameter of [ + ...func.parameters, + ...(func.overloads ?? []).flatMap(overload => overload.parameters), + ]) { + occupied.add(this.escapeIdentifier(parameter.name)); + } + }; + module.functions.forEach(func => includeCallable(func)); + for (const cls of module.classes) { + occupied.add(this.escapeIdentifier(cls.name)); + const owner = this.escapeIdentifier(cls.name, { preserveCase: true }); + cls.methods.forEach(method => includeCallable(method, owner)); + } + (module.typeAliases ?? []).forEach(alias => occupied.add(this.escapeIdentifier(alias.name))); + if (occupied.has(identifier)) { + throw new Error(`Runtime getter identifier ${identifier} conflicts with a generated binding`); + } + } + /** * Convert Python snake_case to TypeScript camelCase and escape reserved words */ @@ -1368,6 +1411,8 @@ ${migrationNote}${declarationMethodsSection} if (runtimeGetterIdentifier === 'getRuntimeBridge') { throw new Error('Binding templates require a distinct runtime getter'); } + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); + this.assertBindingGetterAvailable(module, runtimeGetterIdentifier); return this.generateModuleWithGetter(module, annotatedJSDoc, runtimeGetterIdentifier, false); } diff --git a/test/generator.test.ts b/test/generator.test.ts index 6a7fe588..5f7ccdc3 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -83,8 +83,15 @@ describe('CodeGenerator', () => { expect(bound.typescript).toContain("__tywrapRuntimeProvider().call('binding_fixture', 'Box.pong'"); expect(bound.typescript).not.toContain('getRuntimeBridge'); expect(bound.declaration).toBe(normal.declaration); - expect(() => gen.generateModuleBindingTemplate(module, false, 'provider;inject')) - .toThrow(/Invalid runtime getter identifier/); + for (const identifier of [ + 'ping', 'provider', 'createReturnValidator', '__tywrapReturnDefinitions', + '__args', 'for', 'super', 'this', 'arguments', 'undefined', 'provider;inject', + ]) { + expect(() => gen.generateModuleBindingTemplate(module, false, identifier)) + .toThrow(/Invalid runtime getter identifier/); + } + expect(gen.generateModuleBindingTemplate(module, false, '__tywrapRuntimeProvider1').typescript) + .toContain('__tywrapRuntimeProvider1().call'); }); it('emits a null union member for X | None returns, not an accept-everything any', () => { From fb93225e225e9748d0b2e46f0e418e837f01513e Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:03:30 -0700 Subject: [PATCH 38/50] style: format callable compiler source and tests --- src/core/annotation-parser.ts | 4 +- src/core/callable-compiler.ts | 152 +++++--- src/core/generator.ts | 110 +++--- src/core/ir-contract.ts | 47 ++- src/core/ir-model.ts | 5 +- src/runtime/validators.ts | 29 +- src/tywrap.ts | 35 +- test/annotation-parser.test.ts | 12 +- test/callable-compiler.test.ts | 486 ++++++++++++++++--------- test/generator.test.ts | 79 ++-- test/integration.test.ts | 42 ++- test/ir-contract-validation.test.ts | 8 +- test/ir_contract.test.ts | 72 ++-- test/menagerie/gen.test.ts | 22 +- test/runtime_return_validation.test.ts | 35 +- 15 files changed, 747 insertions(+), 391 deletions(-) diff --git a/src/core/annotation-parser.ts b/src/core/annotation-parser.ts index a636f7d6..9bf6543a 100644 --- a/src/core/annotation-parser.ts +++ b/src/core/annotation-parser.ts @@ -501,8 +501,8 @@ export function parseAnnotationToPythonType( if (generic) { const qualified = splitQualifiedName(generic.name); const parts = splitTopLevel(generic.inner, ','); - const numpyArrayShape = qualified.module === 'numpy' && - qualified.name === 'ndarray' && parts.length === 2; + const numpyArrayShape = + qualified.module === 'numpy' && qualified.name === 'ndarray' && parts.length === 2; const typeArgs = parts.map((part, index) => { if (!numpyArrayShape || index !== 0) { return parse(part.trim(), depth + 1); diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index cd99e57c..fb6db356 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -5,10 +5,7 @@ * deterministic step between validated IR and generated wrapper files. */ -import { - VALUE_CONTRACT_REVISION, - type ValueContract, -} from '../contracts/value-contract.js'; +import { VALUE_CONTRACT_REVISION, type ValueContract } from '../contracts/value-contract.js'; import type { GeneratedCode, Parameter, @@ -162,7 +159,8 @@ export const DEFAULT_CALLABLE_CAPABILITIES: readonly CapabilityDescription[] = [ { name: 'dataclass-adapter', available: false, - guidance: 'Use a TypedDict or an explicit record adapter until #339 defines dataclass conversion.', + guidance: + 'Use a TypedDict or an explicit record adapter until #339 defines dataclass conversion.', }, ]; @@ -271,7 +269,12 @@ function resolveTuple( const values = entries as SupportedValueResolution[]; return { status: 'supported', - value: { kind: 'tuple', wire: 'json', decodedAs: 'array', items: values.map(entry => entry.value) }, + value: { + kind: 'tuple', + wire: 'json', + decodedAs: 'array', + items: values.map(entry => entry.value), + }, }; } @@ -321,18 +324,23 @@ function resolveUnion( } function resolveNdarray(type: PythonType): ValueResolution { - const typeArgument = type.kind === 'generic' && type.typeArgs.length === 2 && + const typeArgument = + type.kind === 'generic' && + type.typeArgs.length === 2 && type.typeArgs[1]?.kind === 'generic' && type.typeArgs[1].name === 'dtype' && type.typeArgs[1].module === 'numpy' && (type.typeArgs[0]?.kind === 'collection' || (type.typeArgs[0]?.kind === 'custom' && type.typeArgs[0].name === 'Any')) - ? type.typeArgs[1].typeArgs[0] - : type.kind === 'generic' && type.typeArgs.length === 1 - ? type.typeArgs[0] - : undefined; - if (typeArgument?.kind !== 'custom' || typeArgument.name !== 'float16' || - typeArgument.module !== 'numpy') { + ? type.typeArgs[1].typeArgs[0] + : type.kind === 'generic' && type.typeArgs.length === 1 + ? type.typeArgs[0] + : undefined; + if ( + typeArgument?.kind !== 'custom' || + typeArgument.name !== 'float16' || + typeArgument.module !== 'numpy' + ) { return { status: 'unresolved', annotation: annotationName(type), @@ -391,7 +399,10 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { }; } if (type.name === 'str') { - return { status: 'supported', value: { kind: 'string', wire: 'json', decodedAs: 'string' } }; + return { + status: 'supported', + value: { kind: 'string', wire: 'json', decodedAs: 'string' }, + }; } if (type.name === 'bytes') { return { @@ -436,7 +447,11 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { if (type.name === 'tuple') { return resolveTuple(type.itemTypes, request, DEFAULT_VALUE_CONVERSION); } - return resolveSequence(type.itemTypes[0] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + return resolveSequence( + type.itemTypes[0] ?? UNKNOWN_TYPE, + request, + DEFAULT_VALUE_CONVERSION + ); case 'generic': { const leaf = leafName(type); if (leaf === 'NDArray' || leaf === 'ndarray') { @@ -444,8 +459,12 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { } if (leaf === 'Tensor' && type.module?.startsWith('torch')) { const tensorDtype = type.typeArgs[0]; - if (type.typeArgs.length !== 1 || tensorDtype?.kind !== 'custom' || - tensorDtype.name !== 'float16' || tensorDtype.module !== 'torch') { + if ( + type.typeArgs.length !== 1 || + tensorDtype?.kind !== 'custom' || + tensorDtype.name !== 'float16' || + tensorDtype.module !== 'torch' + ) { return { status: 'unresolved', annotation: annotationName(type) }; } const ndarray = resolveNdarray({ @@ -469,7 +488,11 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { }; } if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { - return resolveSequence(type.typeArgs[0] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + return resolveSequence( + type.typeArgs[0] ?? UNKNOWN_TYPE, + request, + DEFAULT_VALUE_CONVERSION + ); } if (['dict', 'Dict', 'Mapping', 'MutableMapping'].includes(leaf ?? '')) { const key = type.typeArgs[0]; @@ -596,8 +619,12 @@ function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { ) { return true; } - if (left.kind === 'ndarray-float16' || right.kind === 'ndarray-float16' || - left.kind === 'torch-float16' || right.kind === 'torch-float16') { + if ( + left.kind === 'ndarray-float16' || + right.kind === 'ndarray-float16' || + left.kind === 'torch-float16' || + right.kind === 'torch-float16' + ) { return true; } if (left.kind === 'tuple' && right.kind === 'sequence') { @@ -610,8 +637,10 @@ function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { return false; } if (left.kind === 'tuple' && right.kind === 'tuple') { - return left.items.length === right.items.length && - left.items.every((item, index) => valuesMayOverlap(item, right.items[index]!)); + return ( + left.items.length === right.items.length && + left.items.every((item, index) => valuesMayOverlap(item, right.items[index]!)) + ); } if (left.kind === 'sequence' && right.kind === 'sequence') { return true; // An empty array matches both element schemas. @@ -644,8 +673,15 @@ function sharesRequiredBinding( ): boolean { const a = left[leftIndex]!; const b = right[rightIndex]!; - if (a.optional || b.optional || a.varArgs || b.varArgs || a.kwArgs || b.kwArgs || - a.name !== b.name) { + if ( + a.optional || + b.optional || + a.varArgs || + b.varArgs || + a.kwArgs || + b.kwArgs || + a.name !== b.name + ) { return false; } if (a.keywordOnly && b.keywordOnly) { @@ -665,7 +701,8 @@ function overloadsMayOverlap( rightParameters: readonly Parameter[] ): boolean { const countRange = (parameters: readonly Parameter[]): [number, number] => [ - parameters.filter(parameter => !parameter.optional && !parameter.varArgs && !parameter.kwArgs).length, + parameters.filter(parameter => !parameter.optional && !parameter.varArgs && !parameter.kwArgs) + .length, parameters.some(parameter => parameter.varArgs || parameter.kwArgs) ? Number.POSITIVE_INFINITY : parameters.length, @@ -676,25 +713,30 @@ function overloadsMayOverlap( return false; } for (let leftIndex = 0; leftIndex < leftParameters.length; leftIndex += 1) { - if (leftParameters.filter(parameter => - parameter.name === leftParameters[leftIndex]!.name - ).length !== 1) { + if ( + leftParameters.filter(parameter => parameter.name === leftParameters[leftIndex]!.name) + .length !== 1 + ) { continue; } - const rightIndex = rightParameters.findIndex(parameter => - parameter.name === leftParameters[leftIndex]!.name + const rightIndex = rightParameters.findIndex( + parameter => parameter.name === leftParameters[leftIndex]!.name ); - if (rightIndex < 0 || rightParameters.filter(parameter => - parameter.name === leftParameters[leftIndex]!.name - ).length !== 1 || !sharesRequiredBinding( - leftParameters, leftIndex, rightParameters, rightIndex - )) { + if ( + rightIndex < 0 || + rightParameters.filter(parameter => parameter.name === leftParameters[leftIndex]!.name) + .length !== 1 || + !sharesRequiredBinding(leftParameters, leftIndex, rightParameters, rightIndex) + ) { continue; } const a = left[leftIndex]?.resolution; const b = right[rightIndex]?.resolution; - if (a?.status === 'supported' && b?.status === 'supported' && - !valuesMayOverlap(a.value, b.value)) { + if ( + a?.status === 'supported' && + b?.status === 'supported' && + !valuesMayOverlap(a.value, b.value) + ) { return false; } } @@ -764,12 +806,14 @@ function resolveCallable( } for (let index = 0; index < overloadParameters.length; index += 1) { for (let earlier = 0; earlier < index; earlier += 1) { - if (overloadsMayOverlap( - visibleOverloadParameters[earlier]!, - visibleOverloadParameters[index]!, - visibleOverloadSignatures[earlier]!, - visibleOverloadSignatures[index]! - )) { + if ( + overloadsMayOverlap( + visibleOverloadParameters[earlier]!, + visibleOverloadParameters[index]!, + visibleOverloadSignatures[earlier]!, + visibleOverloadSignatures[index]! + ) + ) { diagnostics.push({ severity: 'warning', code: 'overload-ambiguous', @@ -846,8 +890,7 @@ function resolveCallable( const resolvedParameters: Parameter[] = parameters.map((parameter, index) => ({ ...func.parameters[index]!, - type: - parameter.resolution.status === 'unsupported' ? UNKNOWN_TYPE : parameter.logicalType, + type: parameter.resolution.status === 'unsupported' ? UNKNOWN_TYPE : parameter.logicalType, })); const resolvedOverloads = (func.overloads ?? []).map((overload, index) => ({ ...overload, @@ -859,7 +902,9 @@ function resolveCallable( : parameter.type, })), returnType: - overloadResults[index]?.resolution.status === 'unsupported' ? UNKNOWN_TYPE : overload.returnType, + overloadResults[index]?.resolution.status === 'unsupported' + ? UNKNOWN_TYPE + : overload.returnType, })); const supportedValue = (value: ResolvedCallableValue): ValueContract | undefined => value.resolution.status === 'supported' ? value.resolution.value : undefined; @@ -905,7 +950,9 @@ export function compileContract( options: CompileContractOptions ): CompiledContract { if (ir.module !== options.module.name) { - throw new Error(`Compiled module ${options.module.name} does not match IR module ${ir.module}.`); + throw new Error( + `Compiled module ${options.module.name} does not match IR module ${ir.module}.` + ); } const canonical = transformIrToTsModel(ir); const select = ( @@ -931,10 +978,10 @@ export function compileContract( canonical.typeAliases ?? [], options.module.typeAliases ?? canonical.typeAliases ?? [] ); - const capabilities = new Map(options.capabilities.map(capability => [capability.name, capability])); - const dataclassNames = new Set( - ir.classes.filter(cls => cls.is_dataclass).map(cls => cls.name) + const capabilities = new Map( + options.capabilities.map(capability => [capability.name, capability]) ); + const dataclassNames = new Set(ir.classes.filter(cls => cls.is_dataclass).map(cls => cls.name)); const diagnostics: CallableCompilationDiagnostic[] = []; const callables: ResolvedCallable[] = []; const compileFunction = (func: PythonFunction, path: string): PythonFunction => { @@ -952,7 +999,10 @@ export function compileContract( const module: PythonModule = { ...canonical, functions: selectedFunctions.map(func => - compileFunction(func, `$.functions[${ir.functions.findIndex(entry => entry.name === func.name)}]`) + compileFunction( + func, + `$.functions[${ir.functions.findIndex(entry => entry.name === func.name)}]` + ) ), classes: selectedClasses.map(cls => ({ ...cls, diff --git a/src/core/generator.ts b/src/core/generator.ts index ff112b6c..3605b5dc 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -147,8 +147,10 @@ export class CodeGenerator { } private assertRuntimeGetterIdentifier(identifier: string): void { - if (identifier !== 'getRuntimeBridge' && - !/^__tywrapRuntimeProvider(?:[1-9][0-9]*)?$/.test(identifier)) { + if ( + identifier !== 'getRuntimeBridge' && + !/^__tywrapRuntimeProvider(?:[1-9][0-9]*)?$/.test(identifier) + ) { throw new Error(`Invalid runtime getter identifier: ${identifier}`); } } @@ -176,7 +178,9 @@ export class CodeGenerator { ]); const includeCallable = (func: PythonFunction, owner = ''): void => { occupied.add(this.escapeIdentifier(func.name)); - occupied.add(`__validate${owner}${this.escapeIdentifier(func.name, { preserveCase: true })}Result`); + occupied.add( + `__validate${owner}${this.escapeIdentifier(func.name, { preserveCase: true })}Result` + ); for (const parameter of [ ...func.parameters, ...(func.overloads ?? []).flatMap(overload => overload.parameters), @@ -674,14 +678,14 @@ export class CodeGenerator { .map(({ parameter, parameterIndex }) => ({ name: parameter.name, kind: parameter.varArgs - ? 'var-positional' as const + ? ('var-positional' as const) : parameter.kwArgs - ? 'var-keyword' as const + ? ('var-keyword' as const) : parameter.keywordOnly - ? 'keyword-only' as const + ? ('keyword-only' as const) : parameter.positionalOnly - ? 'positional-only' as const - : 'positional-or-keyword' as const, + ? ('positional-only' as const) + : ('positional-or-keyword' as const), optional: parameter.optional, value: contract?.parameterValues[parameterIndex] ? valueContractToReturnSchema(contract.parameterValues[parameterIndex]) @@ -887,9 +891,7 @@ export class CodeGenerator { return `${this.escapeIdentifier(overloadVarArgs.name)}${forceRequired ? '' : '?'}: unknown[]`; }; const renderKwargs = (forceRequired = false): string | null => - overloadNeedsKwargs - ? `kwargs${forceRequired ? '' : '?'}: ${overloadKwargsType}` - : null; + overloadNeedsKwargs ? `kwargs${forceRequired ? '' : '?'}: ${overloadKwargsType}` : null; const overloadReturnType = this.resolvedReturnType( func.callableContract?.overloads[overloadIndex]?.returnValue, overload.returnType, @@ -1035,9 +1037,10 @@ export class CodeGenerator { const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; const callPreludeLines = emitCallPrelude(callDescriptor, this.callEmitHelpers()); const callPrelude = callPreludeLines.length > 0 ? `${callPreludeLines.join('\n')}\n` : ''; - const overloadValidator = (func.overloads?.length ?? 0) > 0 - ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(func, returnDefinitions))}, __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n` - : ''; + const overloadValidator = + (func.overloads?.length ?? 0) > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(func, returnDefinitions))}, __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n` + : ''; const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${hasDeclaredOverloads ? '' : typeParamDecl}(${paramDecl}): Promise<${implementationReturnType}> { @@ -1254,25 +1257,28 @@ ${callPrelude}${guards}${overloadValidator} return ${runtimeGetterIdentifier}() const validatorName = `__validate${this.escapeIdentifier(cls.name, { preserveCase: true })}${this.escapeIdentifier(method.name, { preserveCase: true })}Result`; const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(method, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; - const declaredOverloads = (method.overloads?.length ?? 0) > 0 - ? this.generateFunctionWrapper( - { - ...method, - parameters: fparams, - overloads: method.overloads?.map(overload => ({ - ...overload, - parameters: overload.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'), - })), - }, - moduleName, - false, - moduleDeclaredNames, - returnDefinitions, - runtimeGetterIdentifier - ).declaration.match(/^export function .*;$/gm)?.map(line => - ` static ${line.slice('export function '.length)}` - ) ?? [] - : []; + const declaredOverloads = + (method.overloads?.length ?? 0) > 0 + ? (this.generateFunctionWrapper( + { + ...method, + parameters: fparams, + overloads: method.overloads?.map(overload => ({ + ...overload, + parameters: overload.parameters.filter( + p => p.name !== 'self' && p.name !== 'cls' + ), + })), + }, + moduleName, + false, + moduleDeclaredNames, + returnDefinitions, + runtimeGetterIdentifier + ) + .declaration.match(/^export function .*;$/gm) + ?.map(line => ` static ${line.slice('export function '.length)}`) ?? []) + : []; const overloads: string[] = [...declaredOverloads]; if (declaredOverloads.length === 0 && needsKwargsParam && requiredKwOnlyNames.length > 0) { const firstOptionalIndex = positionalParams.findIndex(p => p.optional); @@ -1318,10 +1324,13 @@ ${callPrelude}${guards}${overloadValidator} return ${runtimeGetterIdentifier}() const guardLines = emitArgGuards(callDescriptor); const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; - const overloadValidator = declaredOverloads.length > 0 - ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(method, returnDefinitions))}, __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n` - : ''; - const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; + const overloadValidator = + declaredOverloads.length > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(method, returnDefinitions))}, __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n` + : ''; + const selectedValidatorName = overloadValidator + ? '__selectedReturnValidator' + : validatorName; const callExpr = `${runtimeGetterIdentifier}().call<${implementationReturnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${hasDeclaredOverloads ? '' : methodTypeParamDecl}(${paramsDecl}): Promise<${implementationReturnType}> { @@ -1468,27 +1477,30 @@ ${migrationNote}${declarationMethodsSection} const needsRuntime = module.functions.length > 0 || hasRuntimeClasses; const hasOverloads = module.functions.some(func => (func.overloads?.length ?? 0) > 0) || - module.classes.some(cls => - cls.methods.some(method => (method.overloads?.length ?? 0) > 0) - ); + module.classes.some(cls => cls.methods.some(method => (method.overloads?.length ?? 0) > 0)); const emittedCallables = [ ...module.functions, - ...module.classes.flatMap(cls => cls.methods.filter(method => - method.name !== '__init__' && - (method.methodKind === 'class' || method.methodKind === 'static') - )), + ...module.classes.flatMap(cls => + cls.methods.filter( + method => + method.name !== '__init__' && + (method.methodKind === 'class' || method.methodKind === 'static') + ) + ), ]; const resolvedReturns = emittedCallables.flatMap(func => [ func.callableContract?.returnValue, ...(func.callableContract?.overloads.map(overload => overload.returnValue) ?? []), ]); const needsTorchFloat16 = resolvedReturns.some(value => value?.kind === 'torch-float16'); - const needsFloat16 = needsTorchFloat16 || - resolvedReturns.some(value => value?.kind === 'ndarray-float16'); + const needsFloat16 = + needsTorchFloat16 || resolvedReturns.some(value => value?.kind === 'ndarray-float16'); const scientificTypes = needsFloat16 - ? `type __tywrapFloat16Value = number | __tywrapFloat16Value[];\n${needsTorchFloat16 - ? `type __tywrapFloat16Tensor = { data: __tywrapFloat16Value; shape: number[]; dtype: 'torch.float16'; device?: string; sourceDtype?: string; sourceDevice?: string };\n` - : ''}\n` + ? `type __tywrapFloat16Value = number | __tywrapFloat16Value[];\n${ + needsTorchFloat16 + ? `type __tywrapFloat16Tensor = { data: __tywrapFloat16Value; shape: number[]; dtype: 'torch.float16'; device?: string; sourceDtype?: string; sourceDevice?: string };\n` + : '' + }\n` : ''; const bridgeDecl = needsRuntime ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}${importRuntimeGetter ? 'getRuntimeBridge, ' : ''}type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts index f54b8395..9ff2ded2 100644 --- a/src/core/ir-contract.ts +++ b/src/core/ir-contract.ts @@ -205,7 +205,11 @@ function visitArray( } } -function recordAt(value: unknown, path: string, state: ValidationState): Record | null { +function recordAt( + value: unknown, + path: string, + state: ValidationState +): Record | null { if (!isRecord(value)) { state.invalid(path, `${path} must be an object.`); return null; @@ -358,7 +362,8 @@ function validateFunction(value: unknown, path: string, state: ValidationState): const parameters = requiredArray(callable, 'parameters', path, state); if (parameters) { visitArray(parameters, `${path}.parameters`, state, (parameter, entryPath) => - validateParameter(parameter, entryPath, state)); + validateParameter(parameter, entryPath, state) + ); } requiredNullableString(callable, 'returns', path, state); requiredBoolean(callable, 'is_async', path, state); @@ -366,7 +371,8 @@ function validateFunction(value: unknown, path: string, state: ValidationState): const typeParameters = requiredArray(callable, 'type_params', path, state); if (typeParameters) { visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + validateTypeParameter(parameter, entryPath, state) + ); } const methodKind = requiredString(callable, 'method_kind', path, state); if (methodKind !== null && !METHOD_KINDS.has(methodKind)) { @@ -381,8 +387,12 @@ function validateFunction(value: unknown, path: string, state: ValidationState): } const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); if (overloadParameters) { - visitArray(overloadParameters, `${overloadPath}.parameters`, state, - (parameter, entryPath) => validateParameter(parameter, entryPath, state)); + visitArray( + overloadParameters, + `${overloadPath}.parameters`, + state, + (parameter, entryPath) => validateParameter(parameter, entryPath, state) + ); } requiredNullableString(signature, 'returns', overloadPath, state); }); @@ -416,7 +426,8 @@ function validateClass(value: unknown, path: string, state: ValidationState): vo const methods = requiredArray(cls, 'methods', path, state); if (methods) { visitArray(methods, `${path}.methods`, state, (method, entryPath) => - validateFunction(method, entryPath, state)); + validateFunction(method, entryPath, state) + ); } requiredBoolean(cls, 'typed_dict', path, state); const total = requiredValue(cls, 'total', path, state); @@ -426,7 +437,8 @@ function validateClass(value: unknown, path: string, state: ValidationState): vo const fields = requiredArray(cls, 'fields', path, state); if (fields) { visitArray(fields, `${path}.fields`, state, (field, entryPath) => - validateParameter(field, entryPath, state, FIELD_KINDS)); + validateParameter(field, entryPath, state, FIELD_KINDS) + ); } requiredBoolean(cls, 'is_protocol', path, state); requiredBoolean(cls, 'is_namedtuple', path, state); @@ -435,12 +447,14 @@ function validateClass(value: unknown, path: string, state: ValidationState): vo const typeParameters = requiredArray(cls, 'type_params', path, state); if (typeParameters) { visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + validateTypeParameter(parameter, entryPath, state) + ); } const accessors = requiredArray(cls, 'accessors', path, state); if (accessors) { visitArray(accessors, `${path}.accessors`, state, (accessor, entryPath) => - validateAccessor(accessor, entryPath, state)); + validateAccessor(accessor, entryPath, state) + ); } } @@ -466,7 +480,8 @@ function validateTypeAlias(value: unknown, path: string, state: ValidationState) const typeParameters = requiredArray(alias, 'type_params', path, state); if (typeParameters) { visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => - validateTypeParameter(parameter, entryPath, state)); + validateTypeParameter(parameter, entryPath, state) + ); } } @@ -495,22 +510,26 @@ export function validateIrContract( const functions = requiredArray(contract, 'functions', '$', state); if (functions) { visitArray(functions, '$.functions', state, (functionValue, entryPath) => - validateFunction(functionValue, entryPath, state)); + validateFunction(functionValue, entryPath, state) + ); } const classes = requiredArray(contract, 'classes', '$', state); if (classes) { visitArray(classes, '$.classes', state, (classValue, entryPath) => - validateClass(classValue, entryPath, state)); + validateClass(classValue, entryPath, state) + ); } const constants = requiredArray(contract, 'constants', '$', state); if (constants) { visitArray(constants, '$.constants', state, (constant, entryPath) => - validateConstant(constant, entryPath, state)); + validateConstant(constant, entryPath, state) + ); } const aliases = requiredArray(contract, 'type_aliases', '$', state); if (aliases) { visitArray(aliases, '$.type_aliases', state, (alias, entryPath) => - validateTypeAlias(alias, entryPath, state)); + validateTypeAlias(alias, entryPath, state) + ); } const metadata = contract.metadata; if (metadata === undefined && !options.allowOmittedMetadata) { diff --git a/src/core/ir-model.ts b/src/core/ir-model.ts index 7ef07590..b4526b12 100644 --- a/src/core/ir-model.ts +++ b/src/core/ir-model.ts @@ -154,9 +154,8 @@ export function transformIrToTsModel( return { name: ir.module, path: undefined, - version: typeof ir.metadata.package_version === 'string' - ? ir.metadata.package_version - : undefined, + version: + typeof ir.metadata.package_version === 'string' ? ir.metadata.package_version : undefined, functions: ir.functions.map(func => mapFunc(func)), classes: ir.classes.map(mapClass), typeAliases: ir.type_aliases.map(mapTypeAlias), diff --git a/src/runtime/validators.ts b/src/runtime/validators.ts index dd07557a..cabbafda 100644 --- a/src/runtime/validators.ts +++ b/src/runtime/validators.ts @@ -51,7 +51,12 @@ export type ReturnValidator = (result: T, provenance?: DecodedProve /** One Python parameter used to select a declared overload after call binding. */ export interface OverloadParameterSchema { name: string; - kind: 'positional-only' | 'positional-or-keyword' | 'var-positional' | 'keyword-only' | 'var-keyword'; + kind: + | 'positional-only' + | 'positional-or-keyword' + | 'var-positional' + | 'keyword-only' + | 'var-keyword'; optional: boolean; value: ReturnSchema; } @@ -191,14 +196,19 @@ function check( case 'literal': return Object.is(value, schema.value); case 'array': - return Array.isArray(value) && value.every((item, index) => - check(schema.element, item, state, { parent: value, key: index })); + return ( + Array.isArray(value) && + value.every((item, index) => + check(schema.element, item, state, { parent: value, key: index }) + ) + ); case 'tuple': return ( Array.isArray(value) && value.length === schema.elements.length && schema.elements.every((entry, index) => - check(entry, value[index], state, { parent: value, key: index })) + check(entry, value[index], state, { parent: value, key: index }) + ) ); case 'record': { if (!isPlainObject(value)) { @@ -220,7 +230,8 @@ function check( return ( !schema.values || Object.entries(value).every(([key, item]) => - check(schema.values as ReturnSchema, item, state, { parent: value, key })) + check(schema.values as ReturnSchema, item, state, { parent: value, key }) + ) ); } case 'union': @@ -253,7 +264,9 @@ export function createReturnValidator( ): ReturnValidator { const declaredType = renderSchema(schema); return (result: T, provenance?: DecodedProvenance): T => { - if (!check(schema, result, { definitions, pairs: new WeakMap>(), provenance })) { + if ( + !check(schema, result, { definitions, pairs: new WeakMap>(), provenance }) + ) { throw new BridgeValidationError({ declaredType, receivedShape: describeReceivedShape(result), @@ -336,9 +349,7 @@ export function selectOverloadReturnValidator( definitions: Readonly> = {} ): ReturnValidator { const selected = overloads.find(overload => matchesOverload(overload, args, kwargs)); - return selected - ? createReturnValidator(selected.result, callSite, definitions) - : fallback; + return selected ? createReturnValidator(selected.result, callSite, definitions) : fallback; } // ═══════════════════════════════════════════════════════════════════════════ diff --git a/src/tywrap.ts b/src/tywrap.ts index 3731dc33..6a99c5f4 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -472,27 +472,32 @@ export async function generate( const classIndex = ir.classes.findIndex(cls => cls.methods.some(method => method.qualname === qualname) ); - const methodIndex = classIndex < 0 ? -1 : ir.classes[classIndex]!.methods.findIndex( - method => method.qualname === qualname - ); - const path = functionIndex >= 0 - ? `$.functions[${functionIndex}]` - : methodIndex >= 0 - ? `$.classes[${classIndex}].methods[${methodIndex}]` - : ''; + const methodIndex = + classIndex < 0 + ? -1 + : ir.classes[classIndex]!.methods.findIndex(method => method.qualname === qualname); + const path = + functionIndex >= 0 + ? `$.functions[${functionIndex}]` + : methodIndex >= 0 + ? `$.classes[${classIndex}].methods[${methodIndex}]` + : ''; const callableResult = compiled.callables.find(callable => callable.path === path)?.result; const logicalType = callableResult?.logicalType; - const outerName = logicalType && + const outerName = + logicalType && (logicalType.kind === 'custom' || logicalType.kind === 'generic') && logicalType.module - ? `${logicalType.module}.${logicalType.name}` - : undefined; + ? `${logicalType.module}.${logicalType.name}` + : undefined; const resolution = callableResult?.resolution; - if (resolution?.status === 'supported' && candidate === outerName && ( - (resolution.value.kind === 'ndarray-float16' && + if ( + resolution?.status === 'supported' && + candidate === outerName && + ((resolution.value.kind === 'ndarray-float16' && (candidate === 'numpy.ndarray' || candidate === 'numpy.typing.NDArray')) || - (resolution.value.kind === 'torch-float16' && candidate === 'torch.Tensor') - )) { + (resolution.value.kind === 'torch-float16' && candidate === 'torch.Tensor')) + ) { continue; } } diff --git a/test/annotation-parser.test.ts b/test/annotation-parser.test.ts index daad8a50..f4fb0c99 100644 --- a/test/annotation-parser.test.ts +++ b/test/annotation-parser.test.ts @@ -76,11 +76,17 @@ describe('annotation parser', () => { { onUnknownTypeName: name => unknown.push(name) } ); expect(type).toMatchObject({ - kind: 'generic', name: 'ndarray', module: 'numpy', + kind: 'generic', + name: 'ndarray', + module: 'numpy', typeArgs: [ { kind: 'collection', name: 'tuple' }, - { kind: 'generic', name: 'dtype', module: 'numpy', - typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }] }, + { + kind: 'generic', + name: 'dtype', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }, ], }); expect(unknown).toEqual([]); diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 5bd900f6..4ed497fe 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -118,7 +118,12 @@ const moduleModel: PythonModule = { functions: [ { name: 'select', - signature: { parameters: [], returnType: { kind: 'custom', name: 'Any' }, isAsync: false, isGenerator: false }, + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Any' }, + isAsync: false, + isGenerator: false, + }, decorators: [], isAsync: false, isGenerator: false, @@ -173,7 +178,12 @@ const moduleModel: PythonModule = { }, { name: 'nested', - signature: { parameters: [], returnType: { kind: 'custom', name: 'Any' }, isAsync: false, isGenerator: false }, + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Any' }, + isAsync: false, + isGenerator: false, + }, decorators: [], isAsync: false, isGenerator: false, @@ -196,7 +206,12 @@ const moduleModel: PythonModule = { }, { name: 'async_value', - signature: { parameters: [], returnType: { kind: 'primitive', name: 'str' }, isAsync: true, isGenerator: false }, + signature: { + parameters: [], + returnType: { kind: 'primitive', name: 'str' }, + isAsync: true, + isGenerator: false, + }, decorators: [], isAsync: true, isGenerator: false, @@ -205,7 +220,12 @@ const moduleModel: PythonModule = { }, { name: 'point', - signature: { parameters: [], returnType: { kind: 'custom', name: 'Point' }, isAsync: false, isGenerator: false }, + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Point' }, + isAsync: false, + isGenerator: false, + }, decorators: [], isAsync: false, isGenerator: false, @@ -327,20 +347,28 @@ describe('compileContract', () => { const source = rawIr.functions[0]!; const key = source.overloads[0]!.parameters[0]!; const optionalBase = { - name: 'base', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: true, + name: 'base', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: 'int', + default: true, }; - const ir = validateIrContract({ - ...rawIr, - functions: [{ - ...source, - parameters: [key, optionalBase], - overloads: [ - { parameters: [key], returns: 'str' }, - { parameters: [key, optionalBase], returns: 'int' }, + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + parameters: [key, optionalBase], + overloads: [ + { parameters: [key], returns: 'str' }, + { parameters: [key, optionalBase], returns: 'int' }, + ], + }, ], - }], - classes: [], - }, 'optional overload contract'); + classes: [], + }, + 'optional overload contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; @@ -356,14 +384,16 @@ describe('compileContract', () => { const model: PythonModule = { ...moduleModel, classes: [], - functions: [{ - ...original, - parameters: [keyModel, baseModel], - overloads: [ - { parameters: [keyModel], returnType: { kind: 'primitive', name: 'str' } }, - { parameters: [keyModel, baseModel], returnType: { kind: 'primitive', name: 'int' } }, - ], - }], + functions: [ + { + ...original, + parameters: [keyModel, baseModel], + overloads: [ + { parameters: [keyModel], returnType: { kind: 'primitive', name: 'str' } }, + { parameters: [keyModel, baseModel], returnType: { kind: 'primitive', name: 'int' } }, + ], + }, + ], }; const compiled = compileContract(ir.contract, { module: model, @@ -371,28 +401,39 @@ describe('compileContract', () => { conversion: DEFAULT_VALUE_CONVERSION, capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); - expect(compiled.diagnostics).toEqual(expect.arrayContaining([ - expect.objectContaining({ code: 'overload-ambiguous', path: '$.functions[0].overloads[1]' }), - ])); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'overload-ambiguous', + path: '$.functions[0].overloads[1]', + }), + ]) + ); }); it('warns when keyword-only overloads overlap despite declaration order', () => { const source = rawIr.functions[0]!; const x = { name: 'x', kind: 'KEYWORD_ONLY', annotation: 'str', default: false }; const y = { name: 'y', kind: 'KEYWORD_ONLY', annotation: 'int', default: false }; - const ir = validateIrContract({ - ...rawIr, - functions: [{ - ...source, - name: 'choose', qualname: 'fixture.choose', - parameters: [x, y], - overloads: [ - { parameters: [x, y], returns: 'str' }, - { parameters: [y, x], returns: 'int' }, + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'choose', + qualname: 'fixture.choose', + parameters: [x, y], + overloads: [ + { parameters: [x, y], returns: 'str' }, + { parameters: [y, x], returns: 'int' }, + ], + }, ], - }], - classes: [], - }, 'keyword overload contract'); + classes: [], + }, + 'keyword overload contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; @@ -407,23 +448,36 @@ describe('compileContract', () => { conversion: DEFAULT_VALUE_CONVERSION, capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); - expect(compiled.diagnostics).toEqual(expect.arrayContaining([ - expect.objectContaining({ code: 'overload-ambiguous', path: '$.functions[0].overloads[1]' }), - ])); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'overload-ambiguous', + path: '$.functions[0].overloads[1]', + }), + ]) + ); }); it('takes callable types from validated IR, not from the export selection model', () => { const source = rawIr.functions[1]!; - const ir = validateIrContract({ - ...rawIr, - functions: [{ - ...source, - name: 'echo', qualname: 'fixture.echo', - parameters: [{ name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }], - returns: 'str', - }], - classes: [], - }, 'IR authority contract'); + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'echo', + qualname: 'fixture.echo', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + ], + classes: [], + }, + 'IR authority contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; @@ -431,10 +485,15 @@ describe('compileContract', () => { const forged = { ...moduleModel.functions[1]!, name: 'echo', - parameters: [{ - name: 'value', type: { kind: 'primitive', name: 'int' } as PythonType, - optional: false, varArgs: false, kwArgs: false, - }], + parameters: [ + { + name: 'value', + type: { kind: 'primitive', name: 'int' } as PythonType, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], returnType: { kind: 'primitive', name: 'int' } as PythonType, }; const compiled = compileContract(ir.contract, { @@ -443,9 +502,13 @@ describe('compileContract', () => { conversion: DEFAULT_VALUE_CONVERSION, capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); - expect(compiled.module.functions[0]?.parameters[0]?.type).toEqual({ kind: 'primitive', name: 'str' }); + expect(compiled.module.functions[0]?.parameters[0]?.type).toEqual({ + kind: 'primitive', + name: 'str', + }); expect(compiled.callables[0]?.result.resolution).toMatchObject({ - status: 'supported', value: { kind: 'string' }, + status: 'supported', + value: { kind: 'string' }, }); expect(compiled.generated.declaration).toContain('echo(value: string): Promise'); expect(compiled.generated.declaration).not.toContain('echo(value: number)'); @@ -454,7 +517,10 @@ describe('compileContract', () => { it('selects a class method overload without validating its implicit cls receiver', async () => { const source = rawIr.functions[0]!; const receiver = { - name: 'cls', kind: 'POSITIONAL_OR_KEYWORD', annotation: null, default: false, + name: 'cls', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: null, + default: false, }; const stringInput = source.overloads[0]!.parameters[0]!; const integerInput = source.overloads[1]!.parameters[0]!; @@ -469,18 +535,23 @@ describe('compileContract', () => { { parameters: [receiver, integerInput], returns: 'int' }, ], }; - const ir = validateIrContract({ - ...rawIr, - functions: [], - classes: [{ - ...rawIr.classes[0]!, - name: 'Converter', - qualname: 'fixture.Converter', - methods: [methodIr], - fields: [], - is_dataclass: false, - }], - }, 'class overload contract'); + const ir = validateIrContract( + { + ...rawIr, + functions: [], + classes: [ + { + ...rawIr.classes[0]!, + name: 'Converter', + qualname: 'fixture.Converter', + methods: [methodIr], + fields: [], + is_dataclass: false, + }, + ], + }, + 'class overload contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; @@ -504,13 +575,15 @@ describe('compileContract', () => { const model: PythonModule = { ...moduleModel, functions: [], - classes: [{ - ...moduleModel.classes[0]!, - name: 'Converter', - kind: 'class', - methods: [method], - properties: [], - }], + classes: [ + { + ...moduleModel.classes[0]!, + name: 'Converter', + kind: 'class', + methods: [method], + properties: [], + }, + ], }; const compiled = compileContract(ir.contract, { module: model, @@ -520,7 +593,9 @@ describe('compileContract', () => { }); expect(compiled.diagnostics.some(item => item.path.endsWith('.parameters[0]'))).toBe(false); expect(compiled.generated.typescript).toContain('"selectable":true'); - expect(compiled.generated.declaration).toContain('static convert(key: string): Promise;'); + expect(compiled.generated.declaration).toContain( + 'static convert(key: string): Promise;' + ); expect(compiled.generated.typescript).toContain( 'static async convert(key: unknown): Promise' ); @@ -559,30 +634,43 @@ describe('compileContract', () => { it('validates a generated scalar float16 wrapper from the HTTP codec proof', async () => { const source = rawIr.functions[1]!; - const ir = validateIrContract({ - ...rawIr, - functions: [{ ...source, name: 'scalar_value', qualname: 'fixture.scalar_value', - returns: 'numpy.typing.NDArray[numpy.float16]' }], - classes: [], - }, 'scalar contract'); + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'scalar_value', + qualname: 'fixture.scalar_value', + returns: 'numpy.typing.NDArray[numpy.float16]', + }, + ], + classes: [], + }, + 'scalar contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; } const scalarType: PythonType = { - kind: 'generic', name: 'NDArray', module: 'numpy.typing', + kind: 'generic', + name: 'NDArray', + module: 'numpy.typing', typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], }; const original = moduleModel.functions[1]!; const model: PythonModule = { ...moduleModel, classes: [], - functions: [{ - ...original, - name: 'scalar_value', - returnType: scalarType, - signature: { ...original.signature, returnType: scalarType }, - }], + functions: [ + { + ...original, + name: 'scalar_value', + returnType: scalarType, + signature: { ...original.signature, returnType: scalarType }, + }, + ], }; const compiled = compileContract(ir.contract, { module: model, @@ -600,13 +688,19 @@ describe('compileContract', () => { const server = createServer((request, response) => { request.resume(); response.setHeader('content-type', 'application/json'); - response.end(JSON.stringify({ - id: ++requestId, - result: { - __tywrap__: 'ndarray', codecVersion: 1, encoding: 'json', - shape: [], dtype, data: 1.5, - }, - })); + response.end( + JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + shape: [], + dtype, + data: 1.5, + }, + }) + ); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const address = server.address() as AddressInfo; @@ -684,30 +778,48 @@ describe('compileContract', () => { expect(resolve({ kind: 'custom', name: 'Tensor', module: 'torch' })).toMatchObject({ status: 'unresolved', }); - expect(resolve({ - kind: 'generic', - name: 'NDArray', - module: 'numpy', - typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], - })).toMatchObject({ + expect( + resolve({ + kind: 'generic', + name: 'NDArray', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }) + ).toMatchObject({ status: 'supported', value: { kind: 'ndarray-float16', dtype: 'float16' }, }); - expect(resolve(parseAnnotationToPythonType( - 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]' - ))).toMatchObject({ + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]' + ) + ) + ).toMatchObject({ status: 'supported', value: { kind: 'ndarray-float16', dtype: 'float16' }, }); - expect(resolve(parseAnnotationToPythonType( - 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float32]]' - ))).toMatchObject({ status: 'unresolved' }); - expect(resolve(parseAnnotationToPythonType( - 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]' - ))).toMatchObject({ status: 'unresolved' }); - expect(resolve(parseAnnotationToPythonType( - 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[torch.float16]]' - ))).toMatchObject({ status: 'unresolved' }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float32]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[torch.float16]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); }); it('keeps the existing bytes RPC shape in the value contract', () => { @@ -730,7 +842,9 @@ describe('compileContract', () => { typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], }; const resolution = DEFAULT_VALUE_CONVERSION.resolve({ - direction: 'output', path: '$.functions[0].returns', logicalType: tensorType, + direction: 'output', + path: '$.functions[0].returns', + logicalType: tensorType, }); expect(resolution.status).toBe('supported'); if (resolution.status !== 'supported') { @@ -739,54 +853,72 @@ describe('compileContract', () => { const generated = new CodeGenerator().generateModuleDefinition({ ...moduleModel, classes: [], - functions: [{ - ...moduleModel.functions[1]!, - name: 'tensorValue', - returnType: tensorType, - callableContract: { - parameterValues: [], returnValue: resolution.value, overloads: [], + functions: [ + { + ...moduleModel.functions[1]!, + name: 'tensorValue', + returnType: tensorType, + callableContract: { + parameterValues: [], + returnValue: resolution.value, + overloads: [], + }, }, - }], + ], }); expect(generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); - expect(generated.declaration).toContain( - 'tensorValue(): Promise<__tywrapFloat16Tensor>' - ); - expect(DEFAULT_VALUE_CONVERSION.resolve({ - direction: 'output', path: '$.functions[0].returns', - logicalType: { - ...tensorType, - typeArgs: [{ kind: 'custom', name: 'float16', module: 'external' }], - }, - })).toMatchObject({ status: 'unresolved' }); + expect(generated.declaration).toContain('tensorValue(): Promise<__tywrapFloat16Tensor>'); + expect( + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { + ...tensorType, + typeArgs: [{ kind: 'custom', name: 'float16', module: 'external' }], + }, + }) + ).toMatchObject({ status: 'unresolved' }); }); it('rejects a Torch float16 response with a mismatched nested dtype', async () => { const source = rawIr.functions[1]!; - const ir = validateIrContract({ - ...rawIr, - functions: [{ ...source, name: 'tensor_value', qualname: 'fixture.tensor_value', - returns: 'torch.Tensor[torch.float16]' }], - classes: [], - }, 'Torch contract'); + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'tensor_value', + qualname: 'fixture.tensor_value', + returns: 'torch.Tensor[torch.float16]', + }, + ], + classes: [], + }, + 'Torch contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; } const tensorType: PythonType = { - kind: 'generic', name: 'Tensor', module: 'torch', + kind: 'generic', + name: 'Tensor', + module: 'torch', typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], }; const original = moduleModel.functions[1]!; const model: PythonModule = { ...moduleModel, classes: [], - functions: [{ - ...original, - name: 'tensor_value', - returnType: tensorType, - signature: { ...original.signature, returnType: tensorType }, - }], + functions: [ + { + ...original, + name: 'tensor_value', + returnType: tensorType, + signature: { ...original.signature, returnType: tensorType }, + }, + ], }; const compiled = compileContract(ir.contract, { module: model, @@ -794,24 +926,36 @@ describe('compileContract', () => { conversion: DEFAULT_VALUE_CONVERSION, capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); - expect(compiled.generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); + expect(compiled.generated.typescript).toContain( + '"marker":"torch.tensor","dtype":"torch.float16"' + ); let nestedDtype = 'float16'; let requestId = 0; const server = createServer((request, response) => { request.resume(); response.setHeader('content-type', 'application/json'); - response.end(JSON.stringify({ - id: ++requestId, - result: { - __tywrap__: 'torch.tensor', codecVersion: 1, encoding: 'ndarray', - shape: [], dtype: 'torch.float16', device: 'cpu', - value: { - __tywrap__: 'ndarray', codecVersion: 1, encoding: 'json', - shape: [], dtype: nestedDtype, data: 1.5, + response.end( + JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + shape: [], + dtype: 'torch.float16', + device: 'cpu', + value: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + shape: [], + dtype: nestedDtype, + data: 1.5, + }, }, - }, - })); + }) + ); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const address = server.address() as AddressInfo; @@ -828,7 +972,8 @@ describe('compileContract', () => { tensorValue: () => Promise<{ data: number; dtype: string }>; }; await expect(generated.tensorValue()).resolves.toMatchObject({ - data: 1.5, dtype: 'torch.float16', + data: 1.5, + dtype: 'torch.float16', }); nestedDtype = 'float32'; await expect(generated.tensorValue()).rejects.toThrow(/value\.dtype.*must be "float16"/); @@ -876,14 +1021,19 @@ describe('compileContract', () => { it('requires returned dataclass fields even when their constructor has defaults', () => { const point = rawIr.classes[0]!; - const ir = validateIrContract({ - ...rawIr, - functions: [], - classes: [{ - ...point, - fields: [{ ...point.fields[0]!, default: true }, point.fields[1]!], - }], - }, 'Point default contract'); + const ir = validateIrContract( + { + ...rawIr, + functions: [], + classes: [ + { + ...point, + fields: [{ ...point.fields[0]!, default: true }, point.fields[1]!], + }, + ], + }, + 'Point default contract' + ); expect(ir.ok).toBe(true); if (!ir.ok) { return; diff --git a/test/generator.test.ts b/test/generator.test.ts index 5f7ccdc3..571ec3af 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -62,36 +62,71 @@ describe('CodeGenerator', () => { it('uses one opt-in runtime provider for free and static wrappers', () => { const ping = { name: 'ping', - signature: { parameters: [], returnType: { kind: 'primitive', name: 'int' }, - isAsync: false, isGenerator: false }, - decorators: [], isAsync: false, isGenerator: false, - returnType: { kind: 'primitive', name: 'int' }, parameters: [], + signature: { + parameters: [], + returnType: { kind: 'primitive', name: 'int' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType: { kind: 'primitive', name: 'int' }, + parameters: [], } as any; const module = { - name: 'binding_fixture', functions: [ping], - classes: [{ - name: 'Box', kind: 'class', bases: [], properties: [], decorators: [], - methods: [{ ...ping, name: 'pong', methodKind: 'static' }], - }], - typeAliases: [], imports: [], exports: [], + name: 'binding_fixture', + functions: [ping], + classes: [ + { + name: 'Box', + kind: 'class', + bases: [], + properties: [], + decorators: [], + methods: [{ ...ping, name: 'pong', methodKind: 'static' }], + }, + ], + typeAliases: [], + imports: [], + exports: [], } as any; const normal = gen.generateModuleDefinition(module); const bound = gen.generateModuleBindingTemplate(module); - expect(normal.typescript).toContain("getRuntimeBridge().call('binding_fixture', 'ping'"); - expect(normal.typescript).toContain("getRuntimeBridge().call('binding_fixture', 'Box.pong'"); - expect(bound.typescript).toContain("__tywrapRuntimeProvider().call('binding_fixture', 'ping'"); - expect(bound.typescript).toContain("__tywrapRuntimeProvider().call('binding_fixture', 'Box.pong'"); + expect(normal.typescript).toContain( + "getRuntimeBridge().call('binding_fixture', 'ping'" + ); + expect(normal.typescript).toContain( + "getRuntimeBridge().call('binding_fixture', 'Box.pong'" + ); + expect(bound.typescript).toContain( + "__tywrapRuntimeProvider().call('binding_fixture', 'ping'" + ); + expect(bound.typescript).toContain( + "__tywrapRuntimeProvider().call('binding_fixture', 'Box.pong'" + ); expect(bound.typescript).not.toContain('getRuntimeBridge'); expect(bound.declaration).toBe(normal.declaration); for (const identifier of [ - 'ping', 'provider', 'createReturnValidator', '__tywrapReturnDefinitions', - '__args', 'for', 'super', 'this', 'arguments', 'undefined', 'provider;inject', + 'ping', + 'provider', + 'createReturnValidator', + '__tywrapReturnDefinitions', + '__args', + 'for', + 'super', + 'this', + 'arguments', + 'undefined', + 'provider;inject', ]) { - expect(() => gen.generateModuleBindingTemplate(module, false, identifier)) - .toThrow(/Invalid runtime getter identifier/); + expect(() => gen.generateModuleBindingTemplate(module, false, identifier)).toThrow( + /Invalid runtime getter identifier/ + ); } - expect(gen.generateModuleBindingTemplate(module, false, '__tywrapRuntimeProvider1').typescript) - .toContain('__tywrapRuntimeProvider1().call'); + expect( + gen.generateModuleBindingTemplate(module, false, '__tywrapRuntimeProvider1').typescript + ).toContain('__tywrapRuntimeProvider1().call'); }); it('emits a null union member for X | None returns, not an accept-everything any', () => { @@ -739,9 +774,7 @@ describe('CodeGenerator', () => { }, 'fixture' ); - expect(classCode.declaration).toContain( - 'static getValue(key: string): Promise;' - ); + expect(classCode.declaration).toContain('static getValue(key: string): Promise;'); expect(classCode.declaration).not.toContain('fallback'); }); diff --git a/test/integration.test.ts b/test/integration.test.ts index ccc8b599..4bb70bc8 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -315,7 +315,9 @@ void wrong; try { const importDir = join(tempDir, 'py'); await mkdir(importDir, { recursive: true }); - await writeFile(join(importDir, 'generic_overload.py'), `from typing import TypeVar, overload + await writeFile( + join(importDir, 'generic_overload.py'), + `from typing import TypeVar, overload T = TypeVar("T") @@ -327,7 +329,9 @@ def choose(value: None) -> None: ... def choose(value: object) -> object: return value -`, 'utf8'); +`, + 'utf8' + ); const outDir = join(tempDir, 'generated'); const result = await generate({ pythonModules: { generic_overload: { runtime: 'node', typeHints: 'strict' } }, @@ -340,19 +344,39 @@ def choose(value: object) -> object: const declaration = await fsUtils.readFile(join(outDir, 'generic_overload.generated.d.ts')); expect(declaration).toContain('export function choose(value: T): Promise;'); const consumerPath = join(tempDir, 'consumer.ts'); - await writeFile(consumerPath, `import { choose } from './generated/generic_overload.generated.js'; + await writeFile( + consumerPath, + `import { choose } from './generated/generic_overload.generated.js'; const text: Promise = choose('key'); const integer: Promise = choose(2); void text; void integer; -`, 'utf8'); +`, + 'utf8' + ); const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); - const compile = await processUtils.exec(process.execPath, [ - tscPath, '--ignoreConfig', '--noEmit', '--pretty', 'false', '--target', 'ES2022', - '--lib', 'ES2022,DOM,DOM.Iterable', '--module', 'ESNext', '--moduleResolution', - 'bundler', '--skipLibCheck', consumerPath, - ], { cwd: process.cwd(), timeoutMs: 30_000 }); + const compile = await processUtils.exec( + process.execPath, + [ + tscPath, + '--ignoreConfig', + '--noEmit', + '--pretty', + 'false', + '--target', + 'ES2022', + '--lib', + 'ES2022,DOM,DOM.Iterable', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck', + consumerPath, + ], + { cwd: process.cwd(), timeoutMs: 30_000 } + ); expect(compile.code, `${compile.stdout}\n${compile.stderr}`).toBe(0); expect(compile.stderr).toBe(''); } finally { diff --git a/test/ir-contract-validation.test.ts b/test/ir-contract-validation.test.ts index 7c165d1c..2913b432 100644 --- a/test/ir-contract-validation.test.ts +++ b/test/ir-contract-validation.test.ts @@ -120,9 +120,11 @@ describe('validateIrContract', () => { const result = validateIrContract(oversized, 'large contract'); expect(result).toMatchObject({ ok: false }); if (!result.ok) { - expect(result.diagnostics.some(diagnostic => - diagnostic.message.includes('100000 entry validation limit') - )).toBe(true); + expect( + result.diagnostics.some(diagnostic => + diagnostic.message.includes('100000 entry validation limit') + ) + ).toBe(true); } }); }); diff --git a/test/ir_contract.test.ts b/test/ir_contract.test.ts index 48d81678..0f005bdf 100644 --- a/test/ir_contract.test.ts +++ b/test/ir_contract.test.ts @@ -100,20 +100,35 @@ describe('pinned IR contracts', () => { const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-ir-float16-')); try { const contractPath = join(tempDir, 'scientific.contract.json'); - await writeFile(contractPath, JSON.stringify({ - ir_version: '0.4.0', module: 'scientific_fixture', - functions: [{ - name: 'value', qualname: 'scientific_fixture.value', docstring: null, - parameters: [], - returns: 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', - is_async: false, is_generator: false, type_params: [], - method_kind: 'instance', overloads: [], - }], - classes: [], constants: [], type_aliases: [], metadata: {}, - warnings: [ - 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.ndarray.', - ], - }), 'utf8'); + await writeFile( + contractPath, + JSON.stringify({ + ir_version: '0.4.0', + module: 'scientific_fixture', + functions: [ + { + name: 'value', + qualname: 'scientific_fixture.value', + docstring: null, + parameters: [], + returns: 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + ], + classes: [], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [ + 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.ndarray.', + ], + }), + 'utf8' + ); const configured = { ...options(join(tempDir, 'generated')), pythonModules: { scientific_fixture: { typeHints: 'strict' as const } }, @@ -122,7 +137,10 @@ describe('pinned IR contracts', () => { const first = await generate(configured); expect(first.failures).toEqual([]); expect(first.warnings).toEqual([]); - const emitted = await readFile(join(tempDir, 'generated', 'scientific_fixture.generated.ts'), 'utf8'); + const emitted = await readFile( + join(tempDir, 'generated', 'scientific_fixture.generated.ts'), + 'utf8' + ); expect(emitted).toContain('value(): Promise<__tywrapFloat16Value>'); expect(emitted).toContain('"marker":"ndarray","dtype":"float16"'); const check = await generate(configured, { check: true }); @@ -138,12 +156,16 @@ describe('pinned IR contracts', () => { 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]'; await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); const rejected = await generate(configured); - expect(rejected.warnings.some(warning => - warning.includes('no value conversion is defined for numpy.ndarray') - )).toBe(true); - expect(rejected.warnings.some(warning => - warning.includes('resolves outside analyzed module: numpy.ndarray') - )).toBe(true); + expect( + rejected.warnings.some(warning => + warning.includes('no value conversion is defined for numpy.ndarray') + ) + ).toBe(true); + expect( + rejected.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.ndarray') + ) + ).toBe(true); foreign.functions[0]!.returns = 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]'; @@ -151,9 +173,11 @@ describe('pinned IR contracts', () => { 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.Opaque.'; await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); const unrelated = await generate(configured); - expect(unrelated.warnings.some(warning => - warning.includes('resolves outside analyzed module: numpy.Opaque') - )).toBe(true); + expect( + unrelated.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.Opaque') + ) + ).toBe(true); } finally { await rm(tempDir, { recursive: true, force: true }); } diff --git a/test/menagerie/gen.test.ts b/test/menagerie/gen.test.ts index e7e27b3b..d40160ca 100644 --- a/test/menagerie/gen.test.ts +++ b/test/menagerie/gen.test.ts @@ -79,18 +79,26 @@ describe('menagerie generation gate', () => { expect(generated[1]).toContain('export function overloaded(value: number)'); expect(generated[1]).toContain('export function overloaded(value: string)'); } else { - expect(generated[0]).toContain('export async function overloaded(value: number | string)'); + expect(generated[0]).toContain( + 'export async function overloaded(value: number | string)' + ); expect(generated[1]).toContain('export function overloaded(value: number | string)'); await compileGeneratedFile(generatedTs as string); } generated = [ generated[0] - .replace('createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge', - 'createReturnValidator, getRuntimeBridge') - .replace(/^(?:export function overloaded[^\n]*\n)*export async function overloaded[^\n]*\n[\s\S]*?^\}/m, - '/* overload implementation checked separately */'), - generated[1].replace(/^export function overloaded[^\n]*(?:\nexport function overloaded[^\n]*)*/m, - '/* overload declarations checked separately */'), + .replace( + 'createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge', + 'createReturnValidator, getRuntimeBridge' + ) + .replace( + /^(?:export function overloaded[^\n]*\n)*export async function overloaded[^\n]*\n[\s\S]*?^\}/m, + '/* overload implementation checked separately */' + ), + generated[1].replace( + /^export function overloaded[^\n]*(?:\nexport function overloaded[^\n]*)*/m, + '/* overload declarations checked separately */' + ), ]; } expect(generated).toMatchSnapshot(); diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 8004b203..ca35a82c 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -38,9 +38,7 @@ describe('generated return validators', () => { ); expect(validator({ count: 42 })).toEqual({ count: 42 }); expect(() => validator({ count: 1.5 })).toThrow(BridgeValidationError); - expect(() => validator({ count: Number.MAX_SAFE_INTEGER + 1 })).toThrow( - BridgeValidationError - ); + expect(() => validator({ count: Number.MAX_SAFE_INTEGER + 1 })).toThrow(BridgeValidationError); }); it('requires per-call provenance for a scalar float16 ndarray', () => { @@ -69,13 +67,16 @@ describe('generated return validators', () => { const proof = new DecodedProvenance(); proof.recordChild(rows[0]!, 0, { marker: 'ndarray', dims: 0, dtype: 'float16' }); proof.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); - const validate = createReturnValidator({ - kind: 'record', - fields: { - rows: { schema: { kind: 'array', element: { kind: 'tuple', elements: [scalar] } } }, - scalar: { schema: scalar }, + const validate = createReturnValidator( + { + kind: 'record', + fields: { + rows: { schema: { kind: 'array', element: { kind: 'tuple', elements: [scalar] } } }, + scalar: { schema: scalar }, + }, }, - }, 'fixture.nestedScalar'); + 'fixture.nestedScalar' + ); expect(validate(result, proof)).toBe(result); expect(() => validate(result)).toThrow(BridgeValidationError); @@ -114,8 +115,20 @@ describe('generated return validators', () => { }, ] as const; const fallback = createReturnValidator({ kind: 'any' }, 'fixture.ambiguous'); - const oneArg = selectOverloadReturnValidator(overloads, ['key'], undefined, fallback, 'fixture.ambiguous'); - const twoArgs = selectOverloadReturnValidator(overloads, ['key', 2], undefined, fallback, 'fixture.ambiguous'); + const oneArg = selectOverloadReturnValidator( + overloads, + ['key'], + undefined, + fallback, + 'fixture.ambiguous' + ); + const twoArgs = selectOverloadReturnValidator( + overloads, + ['key', 2], + undefined, + fallback, + 'fixture.ambiguous' + ); expect(() => oneArg(4)).toThrow(BridgeValidationError); expect(oneArg('text')).toBe('text'); expect(twoArgs(4)).toBe(4); From fcfb82980cbdee7aaa1b291de60eb7fe7aa51c5d Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:03:35 -0700 Subject: [PATCH 39/50] docs: exclude internal contract notes from llms bundle --- docs/public/llms-full.txt | 16 ++++++++-------- scripts/generate-llms-full.mjs | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/public/llms-full.txt b/docs/public/llms-full.txt index 47a36b49..6138750e 100644 --- a/docs/public/llms-full.txt +++ b/docs/public/llms-full.txt @@ -380,7 +380,7 @@ and `tywrap:check` scripts to `package.json` (disable with `--no-scripts`). 1. Set `"caching": true` to reuse generation IR across rebuilds. 2. Limit `functions` and `classes` when a wrapper needs only part of a module. -`batching` and `compression` are accepted config fields in 0.9.0, but no current +`batching` and `compression` are accepted config fields in 0.10.0, but no current generator or runtime code applies them. ```json @@ -773,7 +773,7 @@ types: { ``` Accepted presets are `numpy`, `pandas`, `pydantic`, `stdlib`, `scipy`, `torch`, -and `sklearn`. `numpy` is accepted as a no-op in 0.9.0. The other presets map +and `sklearn`. `numpy` is accepted as a no-op in 0.10.0. The other presets map the library types implemented by the generator, such as `DataFrame`, sparse matrix classes, `Tensor`, and `BaseEstimator`. @@ -2439,7 +2439,7 @@ Generates TypeScript shaped like: export type Pair = [T, T]; export type Transform

= (...args: P) => T; export class Container { - // NOTE: Instance members are not generated in v0.9; migrate this API to value-returning module functions. + // NOTE: Instance members are not generated in v0.10; migrate this API to value-returning module functions. } ``` @@ -2747,7 +2747,7 @@ so a payload can exceed the JSONL line ceiling. The capability is statically [Transport framing](https://bbopen.github.io/tywrap/transport-framing.md) for the wire format. `supportsStreaming` (incremental results for a single request) is `false` on -every backend. It is not implemented as of 0.9.0. +every backend. It is not implemented as of 0.10.0. ### `maxFrameBytes` @@ -2801,7 +2801,7 @@ through a `PooledTransport` lease reassembles correctly (see ## Scope: subprocess only -`tywrap-frame/1` is **subprocess-only** as of 0.9.0. The subprocess transport is +`tywrap-frame/1` is **subprocess-only** as of 0.10.0. The subprocess transport is the only backend with a real frame ceiling, the JSONL line-length limit. The other backends stay single-frame and keep `supportsChunking: false`: @@ -2811,7 +2811,7 @@ other backends stay single-frame and keep `supportsChunking: false`: - **Pyodide** is in-memory (`maxFrameBytes: Number.POSITIVE_INFINITY`, JSON-only). There is no wire to fragment. -`supportsStreaming` stays `false` on every backend as of 0.9.0. +`supportsStreaming` stays `false` on every backend as of 0.10.0. ## Layering @@ -2890,7 +2890,7 @@ construction. The codepoint-boundary rule is a sender obligation: a frame's `data` MUST NOT split a multi-byte UTF-8 sequence, so the receiver can decode each frame as valid UTF-8 and concatenate without re-aligning bytes across frame edges. `utf8-base64` remains a defined alternative in the wire schema for future -use (e.g. a non-text payload), but tywrap does not emit it as of 0.9.0. +use (e.g. a non-text payload), but tywrap does not emit it as of 0.10.0. > Implementation note for later workstreams: `total` and the per-frame split > points are computed over the message's UTF-8 **byte** length against @@ -3555,7 +3555,7 @@ console.log(await sin(pi / 4)); ## Value-returning APIs -v0.9 generated wrappers do not keep live Python class instances. Expose an +v0.10 generated wrappers do not keep live Python class instances. Expose an operation as a value-returning module function instead: ```ts diff --git a/scripts/generate-llms-full.mjs b/scripts/generate-llms-full.mjs index 23859f82..0da1aad8 100644 --- a/scripts/generate-llms-full.mjs +++ b/scripts/generate-llms-full.mjs @@ -30,6 +30,8 @@ const orderedDocs = [ const excludedDocs = new Set([ 'docs/perf-baselines.md', 'docs/release.md', + 'docs/maintainers/callable-contracts.md', + 'docs/maintainers/value-contracts.md', ]); async function collectDocs(dir) { From 1d76041b258662fbbd97ff93b305692bdd3596fb Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:06:58 -0700 Subject: [PATCH 40/50] test(node): assert generated coroutine return contract --- test/runtime_coroutines.test.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/runtime_coroutines.test.ts b/test/runtime_coroutines.test.ts index d130694b..ea35dd58 100644 --- a/test/runtime_coroutines.test.ts +++ b/test/runtime_coroutines.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { delimiter, join } from 'node:path'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import { EventEmitter } from 'node:events'; import type { ChildProcess } from 'node:child_process'; @@ -54,15 +54,24 @@ nodeSuite('Coroutine RPC through Node', () => { ); const outputDir = join(tempDir, 'generated'); const generated = await generate({ - pythonModules: { coroutine_fixture: { runtime: 'node', typeHints: 'strict' } }, + pythonModules: { coroutine_fixture: { typeHints: 'strict' } }, pythonImportPath: [tempDir], - output: { dir: outputDir, format: 'esm', declaration: false, sourceMap: false }, + output: { dir: outputDir, format: 'esm', declaration: true, sourceMap: false }, runtime: { node: { pythonPath: PYTHON ?? 'python3' } }, performance: { caching: false, batching: false, compression: 'none' }, } as never); expect(generated.failures).toEqual([]); + expect(generated.warnings).toEqual([]); const generatedPath = generated.written.find(path => path.endsWith('.generated.ts')); expect(generatedPath).toBeDefined(); + const declarationPath = generated.written.find(path => path.endsWith('.generated.d.ts')); + expect(declarationPath).toBeDefined(); + expect(await readFile(generatedPath as string, 'utf8')).toMatch( + /^export async function asyncText\(\): Promise \{$/m + ); + expect(await readFile(declarationPath as string, 'utf8')).toMatch( + /^export function asyncText\(\): Promise;$/m + ); bridge = new NodeBridge({ scriptPath: 'runtime/python_bridge.py', From 3fefce2b5ef47674583e2f9538947ab149eb4b40 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:06:58 -0700 Subject: [PATCH 41/50] test(pyodide): assert generated coroutine return contract --- test/runtime_pyodide_real.conformance.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/runtime_pyodide_real.conformance.ts b/test/runtime_pyodide_real.conformance.ts index d9d679e5..514a66c0 100644 --- a/test/runtime_pyodide_real.conformance.ts +++ b/test/runtime_pyodide_real.conformance.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -47,14 +47,24 @@ describe('real PyodideBridge', () => { 'utf8' ); const generated = await generate({ - pythonModules: { tywrap_async_text: { runtime: 'pyodide', typeHints: 'strict' } }, + pythonModules: { tywrap_async_text: { typeHints: 'strict' } }, pythonImportPath: [tempDir], - output: { dir: join(tempDir, 'generated'), format: 'esm', declaration: false }, + output: { dir: join(tempDir, 'generated'), format: 'esm', declaration: true }, + runtime: { pyodide: { indexURL } }, performance: { caching: false, batching: false, compression: 'none' }, } as never); expect(generated.failures).toEqual([]); + expect(generated.warnings).toEqual([]); const generatedPath = generated.written.find(path => path.endsWith('.generated.ts')); expect(generatedPath).toBeDefined(); + const declarationPath = generated.written.find(path => path.endsWith('.generated.d.ts')); + expect(declarationPath).toBeDefined(); + expect(readFileSync(generatedPath as string, 'utf8')).toMatch( + /^export async function asyncText\(\): Promise \{$/m + ); + expect(readFileSync(declarationPath as string, 'utf8')).toMatch( + /^export function asyncText\(\): Promise;$/m + ); globals.loadPyodide = async options => { const py = await loadPyodide(options); From 1515267d66bef49ac26bb1d037908f865fa916e2 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:07:51 -0700 Subject: [PATCH 42/50] test(pyodide): exercise default coroutine capability --- test/runtime_pyodide_real.conformance.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtime_pyodide_real.conformance.ts b/test/runtime_pyodide_real.conformance.ts index 514a66c0..699e7055 100644 --- a/test/runtime_pyodide_real.conformance.ts +++ b/test/runtime_pyodide_real.conformance.ts @@ -50,7 +50,6 @@ describe('real PyodideBridge', () => { pythonModules: { tywrap_async_text: { typeHints: 'strict' } }, pythonImportPath: [tempDir], output: { dir: join(tempDir, 'generated'), format: 'esm', declaration: true }, - runtime: { pyodide: { indexURL } }, performance: { caching: false, batching: false, compression: 'none' }, } as never); expect(generated.failures).toEqual([]); From 37ee8dbd7ab2536c2654ade88775e61bb7d5f3d2 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:11:31 -0700 Subject: [PATCH 43/50] Integrate coroutine capability and safe object fallback --- docs/guide/runtimes/http.md | 5 + docs/maintainers/callable-contracts.md | 114 +++++++++----- docs/public/llms-full.txt | 5 + src/core/callable-compiler.ts | 77 +++++++--- test/callable-compiler.test.ts | 140 +++++++++++++++++- test/menagerie/__snapshots__/gen.test.ts.snap | 16 +- 6 files changed, 287 insertions(+), 70 deletions(-) diff --git a/docs/guide/runtimes/http.md b/docs/guide/runtimes/http.md index c5cca83f..e91f556d 100644 --- a/docs/guide/runtimes/http.md +++ b/docs/guide/runtimes/http.md @@ -49,6 +49,11 @@ setRuntimeBridge( payloads. You must provide a compatible server endpoint. Each call is an independent POST request because the protocol is stateless. +Generated async callables have typed results when their return annotations have +supported value conversions. The in-repo Node and Pyodide bridges await Python +coroutines. For `HttpBridge`, your server must await coroutines before it sends +the result. `HttpBridge` cannot verify a remote server's coroutine support. + tywrap does not include an HTTP server. See the [API reference](/reference/api/) for the expected request and response format. diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 1e3393f4..d5d6fe50 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -1,53 +1,99 @@ # Callable compilation contracts -`tywrap_ir` emits IR 0.4. `validateIrContract()` checks required fields in unknown JSON. Each diagnostic has a JSON path. Validation stops after 100,000 array entries. +`tywrap_ir` emits IR 0.4. `validateIrContract()` checks required fields in +unknown JSON. Each diagnostic has a JSON path. Validation stops after 100,000 +array entries. -`compileContract()` is an internal pure step. It accepts validated IR, exported-name selection, value conversions, capabilities, and a generator. It maps callable types from the validated IR. The selection input cannot change those types. It returns the resolved module, callable records, diagnostics, and TypeScript and declaration content. +`compileContract()` is an internal pure step. It accepts validated IR, +exported-name selection, value conversions, capabilities, and a generator. It +maps callable types from the validated IR. The selection input cannot change +those types. It returns the resolved module, callable records, diagnostics, and +TypeScript and declaration content. -The compiler does not read files. It does not start Python. It does not read a cache. It does not write files. It does not access a runtime registry. +The compiler does not read files. It does not start Python. It does not read a +cache. It does not write files. It does not access a runtime registry. -`generate()` owns Python discovery, cache reads, output comparison, and writes. It validates IR and maps annotations, then calls `compileContract()` before emission. Offline contract files omit extractor metadata for stable bytes. The reader restores an empty metadata object. +`generate()` owns Python discovery, cache reads, output comparison, and writes. +It validates IR and maps annotations, then calls `compileContract()` before +emission. Offline contract files omit extractor metadata for stable bytes. The +reader restores an empty metadata object. ## Resolved callable data -The compiler records each parameter and result, including overloads. Each record has a Python type, direction, source path, and conversion result. One resolved module model serves declarations, call wrappers, and return validators. - -Each value conversion uses revision 2 of [value-contracts.v2.json](value-contracts.v2.json). Do not infer a wire rule from the TypeScript type. - -| Case | Result | -| --- | --- | -| `int` | JSON number, JavaScript number, safe-integer constraint | -| `bytes` | Existing base64 envelope, decoded as `Uint8Array` | -| nested `dict[str, T]` | JSON record with the child conversion at every value path | -| `str | int` | Selected union alternative with one child contract per option | -| `tuple[int, str]` | Exact JSON array with a contract for each index | -| float16 ndarray | Existing Arrow or JSON-fallback envelope, decoded as a numeric scalar or nested array | -| float16 Torch tensor | Outer `torch.float16` envelope with a nested `float16` ndarray | - -The compiler requires an explicit float16 dtype. An untyped ndarray or Torch tensor stays unresolved. - -NumPy can spell `NDArray[np.float16]` as `numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]`. The compiler reads the dtype argument, not the shape wildcard. Only a supported scientific result clears its matching external-annotation IR warning. Other external annotations remain warnings. - -The codec records scalar ndarray dtype and rank for one call. A generated return validator accepts a scalar only with matching proof. Nested proof follows its parent and array index or record key. Array index `0` and `"0"` refer to the same child. Equal numbers from separate calls do not share proof. Array and tensor objects keep their identity-based marker proof. - -The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. +The compiler records each parameter and result, including overloads. Each record +has a Python type, direction, source path, and conversion result. One resolved +module model serves declarations, call wrappers, and return validators. + +Each value conversion uses revision 2 of +[value-contracts.v2.json](value-contracts.v2.json). Do not infer a wire rule +from the TypeScript type. + +| Case | Result | +| --------------------- | ------------------------------------------------------------------------------------- | +| `int` | JSON number, JavaScript number, safe-integer constraint | +| `bytes` | Existing base64 envelope, decoded as `Uint8Array` | +| nested `dict[str, T]` | JSON record with the child conversion at every value path | +| `str \| int` | Selected union alternative with one child contract per option | +| `tuple[int, str]` | Exact JSON array with a contract for each index | +| float16 ndarray | Existing Arrow or JSON-fallback envelope, decoded as a numeric scalar or nested array | +| float16 Torch tensor | Outer `torch.float16` envelope with a nested `float16` ndarray | + +The compiler requires an explicit float16 dtype. An untyped ndarray or Torch +tensor stays unresolved. + +NumPy can spell `NDArray[np.float16]` as +`numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]`. The compiler reads +the dtype argument, not the shape wildcard. Only a supported scientific result +clears its matching external-annotation IR warning. Other external annotations +remain warnings. + +The codec records scalar ndarray dtype and rank for one call. A generated return +validator accepts a scalar only with matching proof. Nested proof follows its +parent and array index or record key. Array index `0` and `"0"` refer to the +same child. Equal numbers from separate calls do not share proof. Array and +tensor objects keep their identity-based marker proof. + +The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow +int64 values can decode as `bigint`. ## Diagnostics and fallback -Known unsupported cases create error diagnostics. The compiler changes the affected generated type to `unknown`. `generate()` returns each diagnostic as a warning, so `--fail-on-warn` rejects strict builds. - -- Coroutines require the capability owned by #338. -- Dataclasses require an adapter owned by #339. The default capability stays unavailable. An explicit output record conversion can use the adapter capability when enabled. +Known unsupported cases create error diagnostics. The compiler changes the +affected generated type to `unknown`. `generate()` returns each diagnostic as a +warning, so `--fail-on-warn` rejects strict builds. + +- The default compiler capability covers coroutine execution in the in-repo Node + and Pyodide bridges. A caller can mark this capability unavailable when it + uses an older or custom bridge. The compiler then emits `unknown` and an error + diagnostic for async results. +- Dataclasses require an adapter owned by #339. The default capability stays + unavailable. An explicit output record conversion can use the adapter + capability when enabled. - Non-string record keys require an explicit conversion. -Unresolved annotations create warning diagnostics. They retain the generator's current fallback and do not claim a supported conversion. +Unresolved annotations create warning diagnostics. They retain the generator's +current fallback and do not claim a supported conversion. An unresolved `object` +output, including an output that contains `object`, emits `unknown`. Python +`object` can hold a primitive value, so a TypeScript `object` return would be +too narrow. -IR overload signatures stay separate from the implementation signature. The extractor collects type variables from overload annotations. Generated declarations retain each input-to-result relation. The hidden TypeScript implementation accepts `unknown` values so it can cover every declared overload. +IR overload signatures stay separate from the implementation signature. The +extractor collects type variables from overload annotations. Generated +declarations retain each input-to-result relation. The hidden TypeScript +implementation accepts `unknown` values so it can cover every declared overload. -At runtime, the wrapper selects the first supported overload that matches bound arguments. This follows TypeScript declaration order. The compiler warns when overload inputs may overlap, including an omitted optional argument. Python 3.10 cannot expose registered overloads through `typing.get_overloads`; it keeps the implementation signature and emits an IR warning. +At runtime, the wrapper selects the first supported overload that matches bound +arguments. This follows TypeScript declaration order. The compiler warns when +overload inputs may overlap, including an omitted optional argument. Python 3.10 +cannot expose registered overloads through `typing.get_overloads`; it keeps the +implementation signature and emits an IR warning. ## Deferred work -Revision 2 does not describe references or recursive aliases. The compiler reports these cases as unresolved. A later value-contract revision can give them a supported conversion. +Revision 2 does not describe references or recursive aliases. The compiler +reports these cases as unresolved. A later value-contract revision can give them +a supported conversion. -The client-binding work may consume the resolved callable model later. It must remain opt-in. It must not add files to default generation or mutate the global runtime registry. +The client-binding work may consume the resolved callable model later. It must +remain opt-in. It must not add files to default generation or mutate the global +runtime registry. diff --git a/docs/public/llms-full.txt b/docs/public/llms-full.txt index 6138750e..3bfa3758 100644 --- a/docs/public/llms-full.txt +++ b/docs/public/llms-full.txt @@ -1852,6 +1852,11 @@ setRuntimeBridge( payloads. You must provide a compatible server endpoint. Each call is an independent POST request because the protocol is stateless. +Generated async callables have typed results when their return annotations have +supported value conversions. The in-repo Node and Pyodide bridges await Python +coroutines. For `HttpBridge`, your server must await coroutines before it sends +the result. `HttpBridge` cannot verify a remote server's coroutine support. + tywrap does not include an HTTP server. See the [API reference](https://bbopen.github.io/tywrap/reference/api/) for the expected request and response format. diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index fb6db356..f90eaadc 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -153,8 +153,8 @@ export const DEFAULT_CALLABLE_CAPABILITIES: readonly CapabilityDescription[] = [ }, { name: 'coroutine-execution', - available: false, - guidance: 'Coroutine execution belongs to #338. Expose a value-returning adapter until then.', + available: true, + guidance: 'Use a runtime bridge that awaits Python coroutine results.', }, { name: 'dataclass-adapter', @@ -603,6 +603,38 @@ function diagnosticForResolution( }; } +function containsUnconstrainedObject(type: PythonType): boolean { + switch (type.kind) { + case 'custom': + return type.name === 'object' && (type.module === undefined || type.module === 'builtins'); + case 'collection': + return type.itemTypes.some(containsUnconstrainedObject); + case 'generic': + return type.typeArgs.some(containsUnconstrainedObject); + case 'union': + return type.types.some(containsUnconstrainedObject); + case 'optional': + case 'final': + case 'classvar': + case 'unpack': + return containsUnconstrainedObject(type.type); + case 'annotated': + return containsUnconstrainedObject(type.base); + default: + return false; + } +} + +function outputType(value: ResolvedCallableValue): PythonType { + if ( + value.resolution.status === 'unsupported' || + (value.resolution.status === 'unresolved' && containsUnconstrainedObject(value.logicalType)) + ) { + return UNKNOWN_TYPE; + } + return value.logicalType; +} + function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { if (left.kind === 'unsupported' || right.kind === 'unsupported') { return true; @@ -848,22 +880,22 @@ function resolveCallable( guidance: 'Use a TypedDict or an explicit record adapter.', }; } - if ( - value.direction === 'output' && - func.isAsync && - capabilities.get('coroutine-execution')?.available !== true - ) { - diagnostics.push({ - severity: 'error', - code: 'coroutine-unsupported', - path: value.path, - message: `${value.path}: this callable requires coroutine execution, which #338 has not implemented.`, - }); - value.resolution = { - status: 'unsupported', - reason: 'Coroutine execution is unavailable.', - guidance: 'Expose a synchronous value-returning adapter until #338 lands.', - }; + if (value.direction === 'output' && func.isAsync) { + if (capabilities.get('coroutine-execution')?.available !== true) { + diagnostics.push({ + severity: 'error', + code: 'coroutine-unsupported', + path: value.path, + message: `${value.path}: this callable requires a runtime bridge that awaits Python coroutine results.`, + }); + value.resolution = { + status: 'unsupported', + reason: 'Coroutine execution is unavailable.', + guidance: 'Use a coroutine-capable runtime bridge or a synchronous adapter.', + }; + } else { + requiredCapabilities.add('coroutine-execution'); + } } if (value.resolution.status === 'supported') { const needed = capabilityNamesFor(value.resolution.value); @@ -901,10 +933,7 @@ function resolveCallable( ? UNKNOWN_TYPE : parameter.type, })), - returnType: - overloadResults[index]?.resolution.status === 'unsupported' - ? UNKNOWN_TYPE - : overload.returnType, + returnType: overloadResults[index] ? outputType(overloadResults[index]) : overload.returnType, })); const supportedValue = (value: ResolvedCallableValue): ValueContract | undefined => value.resolution.status === 'supported' ? value.resolution.value : undefined; @@ -914,9 +943,9 @@ function resolveCallable( signature: { ...func.signature, parameters: resolvedParameters, - returnType: result.resolution.status === 'unsupported' ? UNKNOWN_TYPE : func.returnType, + returnType: outputType(result), }, - returnType: result.resolution.status === 'unsupported' ? UNKNOWN_TYPE : func.returnType, + returnType: outputType(result), overloads: resolvedOverloads, callableContract: { parameterValues: parameters.map(supportedValue), diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 4ed497fe..414bd2e2 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -987,7 +987,7 @@ describe('compileContract', () => { } }); - it('degrades unimplemented dataclass and coroutine outputs with local diagnostics', () => { + it('keeps supported coroutine results and degrades unadapted dataclass outputs', () => { const validation = validateIrContract(rawIr, 'fixture contract'); expect(validation.ok).toBe(true); if (!validation.ok) { @@ -1003,20 +1003,152 @@ describe('compileContract', () => { expect(compiled.diagnostics).toEqual( expect.arrayContaining([ - expect.objectContaining({ code: 'coroutine-unsupported', path: '$.functions[2].returns' }), expect.objectContaining({ code: 'dataclass-unsupported', path: '$.functions[3].returns' }), ]) ); - expect(compiled.module.functions[2]?.returnType).toEqual({ + expect( + compiled.diagnostics.some(diagnostic => diagnostic.code === 'coroutine-unsupported') + ).toBe(false); + expect(compiled.module.functions[2]?.returnType).toEqual({ kind: 'primitive', name: 'str' }); + expect(compiled.generated.declaration).toContain('asyncValue(): Promise'); + expect(compiled.callables[2]?.requiredCapabilities).toContain('coroutine-execution'); + expect(compiled.module.functions[3]?.returnType).toEqual({ kind: 'custom', name: 'Any', module: 'typing', }); - expect(compiled.module.functions[3]?.returnType).toEqual({ + }); + + it('degrades coroutine results when the execution capability is unavailable', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES.map(capability => + capability.name === 'coroutine-execution' ? { ...capability, available: false } : capability + ), + }); + + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'coroutine-unsupported', path: '$.functions[2].returns' }), + ]) + ); + expect(compiled.module.functions[2]?.returnType).toEqual({ kind: 'custom', name: 'Any', module: 'typing', }); + expect(compiled.generated.declaration).toContain('asyncValue(): Promise'); + }); + + it('does not promise a TypeScript object for an unconstrained Python object result', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'object_value', + qualname: 'fixture.object_value', + returns: 'object', + }, + ], + classes: [], + }, + 'object result contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: [{ ...moduleModel.functions[1]!, name: 'object_value' }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[0].returns' }), + ]) + ); + expect(compiled.generated.declaration).toContain('objectValue(): Promise'); + expect(compiled.generated.typescript).toContain('createReturnValidator({"kind":"any"}'); + + for (const annotation of ['list[object]', 'dict[str, object]']) { + const nested = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'object_value', + qualname: 'fixture.object_value', + returns: annotation, + }, + ], + classes: [], + }, + 'nested object result contract' + ); + expect(nested.ok).toBe(true); + if (!nested.ok) { + continue; + } + const nestedCompiled = compileContract(nested.contract, { + module: { + ...moduleModel, + classes: [], + functions: [{ ...moduleModel.functions[1]!, name: 'object_value' }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(nestedCompiled.generated.declaration).toContain('objectValue(): Promise'); + } + + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ id: ++requestId, result: 'a valid Python object' })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-object-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + objectValue: () => Promise; + }; + await expect(generated.objectValue()).resolves.toBe('a valid Python object'); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } }); it('requires returned dataclass fields even when their constructor has defaults', () => { diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 76ecb63e..dd291a82 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -17,11 +17,11 @@ export async function annotatedEcho(value: number): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'annotated_echo', __args, undefined, __validateannotated_echoResult); } -const __validateasync_textResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.async_text", __tywrapReturnDefinitions); +const __validateasync_textResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.typing_torture.async_text", __tywrapReturnDefinitions); -export async function asyncText(): Promise { +export async function asyncText(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args, undefined, __validateasync_textResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args, undefined, __validateasync_textResult); } const __validatebytes_echoResult = createReturnValidator({"kind":"primitive","type":"Uint8Array"}, "fixtures.typing_torture.bytes_echo", __tywrapReturnDefinitions); @@ -186,7 +186,7 @@ export type UserId = number export function annotatedEcho(value: number): Promise; -export function asyncText(): Promise; +export function asyncText(): Promise; export function bytesEcho(value: Uint8Array): Promise; @@ -274,11 +274,11 @@ export async function complexValue(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'complex_value', __args, undefined, __validatecomplex_valueResult); } -const __validatecoroutine_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.coroutine_value", __tywrapReturnDefinitions); +const __validatecoroutine_valueResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.values_torture.coroutine_value", __tywrapReturnDefinitions); -export async function coroutineValue(): Promise { +export async function coroutineValue(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'coroutine_value', __args, undefined, __validatecoroutine_valueResult); + return getRuntimeBridge().call('fixtures.values_torture', 'coroutine_value', __args, undefined, __validatecoroutine_valueResult); } const __validatedataclass_instanceResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.dataclass_instance", __tywrapReturnDefinitions); @@ -477,7 +477,7 @@ export function bytesEcho(value: Uint8Array): Promise; export function complexValue(): Promise; -export function coroutineValue(): Promise; +export function coroutineValue(): Promise; export function dataclassInstance(): Promise; From 67f030e8ce5fe2743de128663c74821ebfd5ebd9 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:13:31 -0700 Subject: [PATCH 44/50] Update object return snapshots to unknown --- test/menagerie/__snapshots__/gen.test.ts.snap | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index dd291a82..9f07d690 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -295,11 +295,11 @@ export async function decimalValues(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args, undefined, __validatedecimal_valuesResult); } -const __validatedeeply_nestedResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.deeply_nested", __tywrapReturnDefinitions); +const __validatedeeply_nestedResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.deeply_nested", __tywrapReturnDefinitions); -export async function deeplyNested(): Promise { +export async function deeplyNested(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'deeply_nested', __args, undefined, __validatedeeply_nestedResult); + return getRuntimeBridge().call('fixtures.values_torture', 'deeply_nested', __args, undefined, __validatedeeply_nestedResult); } const __validateecho_intResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.echo_int", __tywrapReturnDefinitions); @@ -309,11 +309,11 @@ export async function echoInt(value: number): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'echo_int', __args, undefined, __validateecho_intResult); } -const __validateempty_valuesResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"tuple","elements":[{"kind":"any"}]},{"kind":"array","element":{"kind":"any"}},{"kind":"record","values":{"kind":"any"}},{"kind":"array","element":{"kind":"any"}}]}, "fixtures.values_torture.empty_values", __tywrapReturnDefinitions); +const __validateempty_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.empty_values", __tywrapReturnDefinitions); -export async function emptyValues(): Promise<[[unknown], object[], Record, object[]]> { +export async function emptyValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call<[[unknown], object[], Record, object[]]>('fixtures.values_torture', 'empty_values', __args, undefined, __validateempty_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'empty_values', __args, undefined, __validateempty_valuesResult); } const __validateenum_memberResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.enum_member", __tywrapReturnDefinitions); @@ -332,9 +332,9 @@ export async function finiteFloatEdges(): Promise { const __validategenerator_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.generator_value", __tywrapReturnDefinitions); -export async function generatorValue(): Promise { +export async function generatorValue(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'generator_value', __args, undefined, __validategenerator_valueResult); + return getRuntimeBridge().call('fixtures.values_torture', 'generator_value', __args, undefined, __validategenerator_valueResult); } const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.int_key_dict", __tywrapReturnDefinitions); @@ -426,11 +426,11 @@ export async function specialFloats(include?: boolean): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'special_floats', __args, undefined, __validatespecial_floatsResult); } -const __validatetemporal_valuesResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.temporal_values", __tywrapReturnDefinitions); +const __validatetemporal_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.temporal_values", __tywrapReturnDefinitions); -export async function temporalValues(): Promise> { +export async function temporalValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'temporal_values', __args, undefined, __validatetemporal_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'temporal_values', __args, undefined, __validatetemporal_valuesResult); } const __validatetuple_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.tuple_key_dict", __tywrapReturnDefinitions); @@ -447,11 +447,11 @@ export async function unicodeText(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'unicode_text', __args, undefined, __validateunicode_textResult); } -const __validateuuid_and_pathResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.uuid_and_path", __tywrapReturnDefinitions); +const __validateuuid_and_pathResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.uuid_and_path", __tywrapReturnDefinitions); -export async function uuidAndPath(): Promise> { +export async function uuidAndPath(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'uuid_and_path', __args, undefined, __validateuuid_and_pathResult); + return getRuntimeBridge().call('fixtures.values_torture', 'uuid_and_path', __args, undefined, __validateuuid_and_pathResult); } /** @@ -483,17 +483,17 @@ export function dataclassInstance(): Promise; export function decimalValues(): Promise; -export function deeplyNested(): Promise; +export function deeplyNested(): Promise; export function echoInt(value: number): Promise; -export function emptyValues(): Promise<[[unknown], object[], Record, object[]]>; +export function emptyValues(): Promise; export function enumMember(): Promise; export function finiteFloatEdges(): Promise; -export function generatorValue(): Promise; +export function generatorValue(): Promise; export function intKeyDict(): Promise; @@ -520,13 +520,13 @@ export function setAndFrozenset(): Promise<[number[], string[]]>; export function specialFloats(): Promise; export function specialFloats(include?: boolean): Promise; -export function temporalValues(): Promise>; +export function temporalValues(): Promise; export function tupleKeyDict(): Promise; export function unicodeText(): Promise; -export function uuidAndPath(): Promise>; +export function uuidAndPath(): Promise; /** * Point(x: 'int', y: 'int') From 57cdbc7ba8a1a8a19f4e883ca18adc9e6aebdaac Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:16:48 -0700 Subject: [PATCH 45/50] Fail closed on unresolved returns and recognize torch HalfTensor --- docs/maintainers/callable-contracts.md | 12 ++- docs/public/llms-full.txt | 6 ++ docs/reference/api/index.md | 6 ++ src/core/callable-compiler.ts | 77 ++++++-------- src/tywrap.ts | 3 +- test/callable-compiler.test.ts | 134 +++++++++++++++++++++++++ 6 files changed, 183 insertions(+), 55 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index d5d6fe50..51da45a3 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -41,6 +41,9 @@ from the TypeScript type. The compiler requires an explicit float16 dtype. An untyped ndarray or Torch tensor stays unresolved. +For an evaluated PyTorch annotation, use `torch.HalfTensor`. The compiler +accepts this exact class and leaves other Torch tensor classes unresolved. + NumPy can spell `NDArray[np.float16]` as `numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]`. The compiler reads the dtype argument, not the shape wildcard. Only a supported scientific result @@ -71,11 +74,10 @@ warning, so `--fail-on-warn` rejects strict builds. capability when enabled. - Non-string record keys require an explicit conversion. -Unresolved annotations create warning diagnostics. They retain the generator's -current fallback and do not claim a supported conversion. An unresolved `object` -output, including an output that contains `object`, emits `unknown`. Python -`object` can hold a primitive value, so a TypeScript `object` return would be -too narrow. +Unresolved annotations create warning diagnostics. Their result type is +`unknown`, including an output that contains `object` or a type variable. Python +`object` can hold a primitive value. A type variable has no validated conversion +to the decoded result. Neither annotation justifies a precise TypeScript return. IR overload signatures stay separate from the implementation signature. The extractor collects type variables from overload annotations. Generated diff --git a/docs/public/llms-full.txt b/docs/public/llms-full.txt index 3bfa3758..ad493d75 100644 --- a/docs/public/llms-full.txt +++ b/docs/public/llms-full.txt @@ -3077,6 +3077,12 @@ When `output.declaration` is enabled, `generate()` writes matching code. Simple `TypeVar` and callable `ParamSpec` declarations are preserved when tywrap can represent them safely. +Unresolved return annotations now emit `Promise` and a located warning. +This includes `TypeVar` and `ParamSpec` results without a supported value +conversion. Input generics remain in the declaration. Earlier versions could +emit `Promise` without validating the decoded value as `T`; callers must +narrow the result or use a supported concrete overload. + `generate()` returns: ```ts diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md index 26a5e3b5..2d4a6976 100644 --- a/docs/reference/api/index.md +++ b/docs/reference/api/index.md @@ -59,6 +59,12 @@ When `output.declaration` is enabled, `generate()` writes matching code. Simple `TypeVar` and callable `ParamSpec` declarations are preserved when tywrap can represent them safely. +Unresolved return annotations now emit `Promise` and a located warning. +This includes `TypeVar` and `ParamSpec` results without a supported value +conversion. Input generics remain in the declaration. Earlier versions could +emit `Promise` without validating the decoded value as `T`; callers must +narrow the result or use a supported concrete overload. + `generate()` returns: ```ts diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index f90eaadc..734785e1 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -358,6 +358,28 @@ function resolveNdarray(type: PythonType): ValueResolution { }; } +function resolveTorchFloat16(): ValueResolution { + const ndarray = resolveNdarray({ + kind: 'generic', + name: 'ndarray', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }); + if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { + return ndarray; + } + return { + status: 'supported', + value: { + kind: 'torch-float16', + wire: 'ndarray-envelope', + decodedAs: 'tensor-record', + dtype: 'torch.float16', + value: ndarray.value, + }, + }; +} + /** The conversion set that #335 may use with the frozen value-contract revision. */ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { revision: VALUE_CONTRACT_REVISION, @@ -467,25 +489,7 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { ) { return { status: 'unresolved', annotation: annotationName(type) }; } - const ndarray = resolveNdarray({ - kind: 'generic', - name: 'ndarray', - module: 'numpy', - typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], - }); - if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { - return ndarray; - } - return { - status: 'supported', - value: { - kind: 'torch-float16', - wire: 'ndarray-envelope', - decodedAs: 'tensor-record', - dtype: 'torch.float16', - value: ndarray.value, - }, - }; + return resolveTorchFloat16(); } if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { return resolveSequence( @@ -509,6 +513,9 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { } case 'custom': { const leaf = leafName(type); + if (type.name === 'HalfTensor' && type.module === 'torch') { + return resolveTorchFloat16(); + } if (leaf === 'ndarray' || leaf === 'NDArray') { return resolveNdarray(type); } @@ -592,7 +599,7 @@ function diagnosticForResolution( severity: 'warning', code: 'conversion-unresolved', path, - message: `${path}: no value conversion is defined for ${resolution.annotation}; the generated type keeps its existing explicit fallback.`, + message: `${path}: no value conversion is defined for ${resolution.annotation}; the generated result type is unknown.`, }; } return { @@ -603,36 +610,8 @@ function diagnosticForResolution( }; } -function containsUnconstrainedObject(type: PythonType): boolean { - switch (type.kind) { - case 'custom': - return type.name === 'object' && (type.module === undefined || type.module === 'builtins'); - case 'collection': - return type.itemTypes.some(containsUnconstrainedObject); - case 'generic': - return type.typeArgs.some(containsUnconstrainedObject); - case 'union': - return type.types.some(containsUnconstrainedObject); - case 'optional': - case 'final': - case 'classvar': - case 'unpack': - return containsUnconstrainedObject(type.type); - case 'annotated': - return containsUnconstrainedObject(type.base); - default: - return false; - } -} - function outputType(value: ResolvedCallableValue): PythonType { - if ( - value.resolution.status === 'unsupported' || - (value.resolution.status === 'unresolved' && containsUnconstrainedObject(value.logicalType)) - ) { - return UNKNOWN_TYPE; - } - return value.logicalType; + return value.resolution.status === 'supported' ? value.logicalType : UNKNOWN_TYPE; } function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { diff --git a/src/tywrap.ts b/src/tywrap.ts index 6a99c5f4..f31deeee 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -496,7 +496,8 @@ export async function generate( candidate === outerName && ((resolution.value.kind === 'ndarray-float16' && (candidate === 'numpy.ndarray' || candidate === 'numpy.typing.NDArray')) || - (resolution.value.kind === 'torch-float16' && candidate === 'torch.Tensor')) + (resolution.value.kind === 'torch-float16' && + (candidate === 'torch.Tensor' || candidate === 'torch.HalfTensor'))) ) { continue; } diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 414bd2e2..01f2cb5b 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -880,6 +880,26 @@ describe('compileContract', () => { ).toMatchObject({ status: 'unresolved' }); }); + it('accepts only the evaluated torch.HalfTensor class as an exact float16 annotation', () => { + const resolve = (annotation: string) => + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: parseAnnotationToPythonType(annotation), + }); + expect(resolve('torch.HalfTensor')).toMatchObject({ + status: 'supported', + value: { + kind: 'torch-float16', + dtype: 'torch.float16', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }, + }); + expect(resolve('torch.FloatTensor')).toMatchObject({ status: 'unresolved' }); + expect(resolve('foreign.HalfTensor')).toMatchObject({ status: 'unresolved' }); + expect(resolve('HalfTensor')).toMatchObject({ status: 'unresolved' }); + }); + it('rejects a Torch float16 response with a mismatched nested dtype', async () => { const source = rawIr.functions[1]!; const ir = validateIrContract( @@ -1151,6 +1171,120 @@ describe('compileContract', () => { } }); + it('keeps generic inputs but uses unknown for unvalidated type-variable results', async () => { + const source = rawIr.functions[1]!; + const typeVar = { + name: 'T', + kind: 'typevar' as const, + bound: null, + constraints: null, + variance: null, + }; + const generic = { + ...source, + name: 'generic_identity', + qualname: 'fixture.generic_identity', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: '~T', default: false }, + ], + returns: '~T', + type_params: [typeVar], + }; + const nested = { + ...generic, + name: 'generic_nested', + qualname: 'fixture.generic_nested', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'list[~T]', default: false }, + ], + returns: 'list[~T]', + }; + const paramspec = { + ...generic, + name: 'paramspec_apply', + qualname: 'fixture.paramspec_apply', + parameters: [ + { + name: 'callback', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: 'typing.Callable[~P, ~T]', + default: false, + }, + ], + type_params: [ + { name: 'P', kind: 'paramspec' as const, bound: null, constraints: null, variance: null }, + typeVar, + ], + }; + const ir = validateIrContract( + { ...rawIr, functions: [generic, nested, paramspec], classes: [] }, + 'generic result contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: [generic, nested, paramspec].map(func => ({ + ...moduleModel.functions[1]!, + name: func.name, + })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[0].returns' }), + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[1].returns' }), + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[2].returns' }), + ]) + ); + expect(compiled.generated.declaration).toContain( + 'genericIdentity(value: T): Promise' + ); + expect(compiled.generated.declaration).toContain( + 'genericNested(value: T[]): Promise' + ); + expect(compiled.generated.declaration).toMatch( + /paramspecApply<[^>]*>\([^\n]*\): Promise/ + ); + + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ id: ++requestId, result: 'not the numeric input' })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-generic-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + genericIdentity: (value: T) => Promise; + }; + await expect(generated.genericIdentity(42)).resolves.toBe('not the numeric input'); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('requires returned dataclass fields even when their constructor has defaults', () => { const point = rawIr.classes[0]!; const ir = validateIrContract( From 4f7c8e50a32981f559eef2421a5f3ecd25794d83 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:23:07 -0700 Subject: [PATCH 46/50] Keep validation checks when widening unresolved results --- docs/maintainers/callable-contracts.md | 4 + src/core/callable-compiler.ts | 25 +++--- src/core/generator.ts | 10 ++- src/types/index.ts | 2 + test/callable-compiler.test.ts | 112 +++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 12 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 51da45a3..1fc7e459 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -79,6 +79,10 @@ Unresolved annotations create warning diagnostics. Their result type is `object` can hold a primitive value. A type variable has no validated conversion to the decoded result. Neither annotation justifies a precise TypeScript return. +The compiler keeps an existing best-effort return check when it widens a +declaration. For example, an unresolved `list[object]` still rejects a non-array +result, and a `DataFrame` annotation still requires a decoded marker. + IR overload signatures stay separate from the implementation signature. The extractor collects type variables from overload annotations. Generated declarations retain each input-to-result relation. The hidden TypeScript diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 734785e1..22d4107e 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -929,9 +929,11 @@ function resolveCallable( callableContract: { parameterValues: parameters.map(supportedValue), returnValue: supportedValue(result), + returnValidationType: func.returnType, overloads: overloadResults.map((overload, index) => ({ parameterValues: overloadParameters[index]!.map(supportedValue), returnValue: supportedValue(overload), + returnValidationType: func.overloads![index]!.returnType, })), }, }; @@ -1014,16 +1016,19 @@ export function compileContract( ), classes: selectedClasses.map(cls => ({ ...cls, - methods: cls.methods.map(method => { - const sourceClassIndex = ir.classes.findIndex(entry => entry.name === cls.name); - const sourceMethodIndex = ir.classes[sourceClassIndex]!.methods.findIndex( - entry => entry.name === method.name - ); - return compileFunction( - method, - `$.classes[${sourceClassIndex}].methods[${sourceMethodIndex}]` - ); - }), + methods: + cls.kind === 'protocol' + ? cls.methods + : cls.methods.map(method => { + const sourceClassIndex = ir.classes.findIndex(entry => entry.name === cls.name); + const sourceMethodIndex = ir.classes[sourceClassIndex]!.methods.findIndex( + entry => entry.name === method.name + ); + return compileFunction( + method, + `$.classes[${sourceClassIndex}].methods[${sourceMethodIndex}]` + ); + }), })), typeAliases: selectedAliases, }; diff --git a/src/core/generator.ts b/src/core/generator.ts index 3605b5dc..41033419 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -662,7 +662,10 @@ export class CodeGenerator { const value = func.callableContract?.returnValue; return value ? valueContractToReturnSchema(value) - : (this.returnSchema(func.returnType, definitions) as ReturnSchema); + : (this.returnSchema( + func.callableContract?.returnValidationType ?? func.returnType, + definitions + ) as ReturnSchema); } private overloadReturnSchemas( @@ -693,7 +696,10 @@ export class CodeGenerator { })), result: contract?.returnValue ? valueContractToReturnSchema(contract.returnValue) - : (this.returnSchema(overload.returnType, definitions) as ReturnSchema), + : (this.returnSchema( + contract?.returnValidationType ?? overload.returnType, + definitions + ) as ReturnSchema), selectable: contract !== undefined && contract.returnValue !== undefined && diff --git a/src/types/index.ts b/src/types/index.ts index d7bf807d..90546f36 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -74,6 +74,8 @@ export interface PythonFunctionOverload { export interface ResolvedCallableSignatureContract { parameterValues: readonly (ValueContract | undefined)[]; returnValue?: ValueContract; + /** Original annotation used for best-effort validation when no value contract resolves. */ + returnValidationType?: PythonType; } /** Internal compiler data consumed by generated validators. */ diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 01f2cb5b..45b0355a 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1107,6 +1107,7 @@ describe('compileContract', () => { expect(compiled.generated.declaration).toContain('objectValue(): Promise'); expect(compiled.generated.typescript).toContain('createReturnValidator({"kind":"any"}'); + let nestedList: ReturnType | undefined; for (const annotation of ['list[object]', 'dict[str, object]']) { const nested = validateIrContract( { @@ -1138,6 +1139,14 @@ describe('compileContract', () => { capabilities: DEFAULT_CALLABLE_CAPABILITIES, }); expect(nestedCompiled.generated.declaration).toContain('objectValue(): Promise'); + expect(nestedCompiled.generated.typescript).toContain( + annotation.startsWith('list') + ? 'createReturnValidator({"kind":"array","element":{"kind":"any"}}' + : 'createReturnValidator({"kind":"record","values":{"kind":"any"}}' + ); + if (annotation.startsWith('list')) { + nestedList = nestedCompiled; + } } let requestId = 0; @@ -1161,6 +1170,20 @@ describe('compileContract', () => { objectValue: () => Promise; }; await expect(generated.objectValue()).resolves.toBe('a valid Python object'); + if (nestedList) { + const nestedPath = join(temporary, 'nested.generated.mjs'); + await writeFile( + nestedPath, + ts.transpileModule(nestedList.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText, + 'utf8' + ); + const nestedGenerated = (await import(pathToFileURL(nestedPath).href)) as { + objectValue: () => Promise; + }; + await expect(nestedGenerated.objectValue()).rejects.toThrow(BridgeValidationError); + } } finally { clearRuntimeBridge(); await bridge.dispose(); @@ -1270,6 +1293,24 @@ describe('compileContract', () => { compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, }).outputText; await writeFile(outputPath, javascript, 'utf8'); + const declarationPath = join(temporary, 'fixture.generated.d.ts'); + const consumerPath = join(temporary, 'consumer.ts'); + await writeFile(declarationPath, compiled.generated.declaration, 'utf8'); + await writeFile( + consumerPath, + "import { genericIdentity } from './fixture.generated.js';\nconst numeric: Promise = genericIdentity(42);\nvoid numeric;\n", + 'utf8' + ); + const program = ts.createProgram([consumerPath], { + noEmit: true, + strict: true, + skipLibCheck: true, + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + types: [], + }); + expect(ts.getPreEmitDiagnostics(program).map(diagnostic => diagnostic.code)).toContain(2322); setRuntimeBridge(bridge); const generated = (await import(pathToFileURL(outputPath).href)) as { genericIdentity: (value: T) => Promise; @@ -1285,6 +1326,77 @@ describe('compileContract', () => { } }); + it('keeps existing marker and literal checks when the declaration falls back to unknown', async () => { + const source = rawIr.functions[1]!; + const annotations = [ + { name: 'dataframe_value', returns: 'pandas.DataFrame' }, + { name: 'literal_value', returns: "typing.Literal['x']" }, + ]; + const ir = validateIrContract( + { + ...rawIr, + functions: annotations.map(({ name, returns }) => ({ + ...source, + name, + qualname: `fixture.${name}`, + returns, + })), + classes: [], + }, + 'unresolved validation contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: annotations.map(({ name }) => ({ ...moduleModel.functions[1]!, name })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.declaration).toContain('dataframeValue(): Promise'); + expect(compiled.generated.declaration).toContain('literalValue(): Promise'); + expect(compiled.generated.typescript).toContain('"marker":"dataframe"'); + expect(compiled.generated.typescript).toContain('"kind":"literal","value":"x"'); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-fallback-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = (functionName === 'literal_value' ? 'y' : {}) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + dataframeValue: () => Promise; + literalValue: () => Promise; + }; + await expect(generated.dataframeValue()).rejects.toThrow(BridgeValidationError); + await expect(generated.literalValue()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('requires returned dataclass fields even when their constructor has defaults', () => { const point = rawIr.classes[0]!; const ir = validateIrContract( From a289bdcb253d70c59e09ca4ee8955812f12e63da Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:25:50 -0700 Subject: [PATCH 47/50] Resolve selected local records and simple aliases --- docs/maintainers/callable-contracts.md | 8 ++ src/core/callable-compiler.ts | 63 ++++++++++++++- test/callable-compiler.test.ts | 103 +++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 2 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 1fc7e459..9c54a8c5 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -59,6 +59,10 @@ tensor objects keep their identity-based marker proof. The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. +Selected local non-generic `TypedDict` classes use revision-2 record contracts +when every field has a supported conversion. Selected simple aliases use their +underlying supported contract. Foreign and recursive names remain unresolved. + ## Diagnostics and fallback Known unsupported cases create error diagnostics. The compiler changes the @@ -83,6 +87,10 @@ The compiler keeps an existing best-effort return check when it widens a declaration. For example, an unresolved `list[object]` still rejects a non-array result, and a `DataFrame` annotation still requires a decoded marker. +`Literal` returns keep their exact-value validator, but stay `unknown` until a +literal value contract exists. Unresolved literal overloads cannot select a +precise return at runtime. + IR overload signatures stay separate from the implementation signature. The extractor collects type variables from overload annotations. Generated declarations retain each input-to-result relation. The hidden TypeScript diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 22d4107e..6b438508 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -5,7 +5,11 @@ * deterministic step between validated IR and generated wrapper files. */ -import { VALUE_CONTRACT_REVISION, type ValueContract } from '../contracts/value-contract.js'; +import { + VALUE_CONTRACT_REVISION, + type ValueContract, + type ValueContractField, +} from '../contracts/value-contract.js'; import type { GeneratedCode, Parameter, @@ -988,6 +992,61 @@ export function compileContract( canonical.typeAliases ?? [], options.module.typeAliases ?? canonical.typeAliases ?? [] ); + const localTypedDicts = new Map( + selectedClasses + .filter(cls => cls.kind === 'typed_dict' && !cls.typeParameters?.length) + .map(cls => [cls.name, cls] as const) + ); + const localAliases = new Map( + selectedAliases + .filter(alias => !alias.typeParameters?.length) + .map(alias => [alias.name, alias] as const) + ); + const conversion: ValueConversionDescription = { + revision: options.conversion.revision, + resolve(request): ValueResolution { + const type = request.logicalType; + if (type.kind !== 'custom' || (type.module !== undefined && type.module !== ir.module)) { + return options.conversion.resolve(request); + } + const typedDict = localTypedDicts.get(type.name); + if (typedDict) { + if ((request.depth ?? 0) >= 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } + const fields: ValueContractField[] = []; + for (const property of typedDict.properties) { + const field = options.conversion.resolve({ + ...request, + logicalType: property.type, + path: `${request.path}.${property.name}`, + depth: (request.depth ?? 0) + 1, + }); + if (field.status !== 'supported') { + return field; + } + fields.push({ name: property.name, value: field.value, required: !property.optional }); + } + return { + status: 'supported', + value: { kind: 'record', wire: 'json', decodedAs: 'object', fields }, + }; + } + const alias = localAliases.get(type.name); + if (alias) { + return options.conversion.resolve({ + ...request, + logicalType: alias.type, + depth: (request.depth ?? 0) + 1, + }); + } + return options.conversion.resolve(request); + }, + }; const capabilities = new Map( options.capabilities.map(capability => [capability.name, capability]) ); @@ -998,7 +1057,7 @@ export function compileContract( const result = resolveCallable( func, path, - options.conversion, + conversion, capabilities, dataclassNames, diagnostics diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index 45b0355a..b0a2c48a 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1397,6 +1397,109 @@ describe('compileContract', () => { } }); + it('resolves selected local TypedDict fields and simple aliases through revision 2', async () => { + const source = rawIr.functions[1]!; + const point = rawIr.classes[0]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'payload_value', + qualname: 'fixture.payload_value', + returns: 'Payload', + }, + { ...source, name: 'user_id', qualname: 'fixture.user_id', returns: 'UserId' }, + ], + classes: [ + { + ...point, + name: 'Payload', + qualname: 'fixture.Payload', + typed_dict: true, + is_dataclass: false, + fields: [ + { name: 'x', kind: 'FIELD', annotation: 'int', default: false }, + { name: 'label', kind: 'FIELD', annotation: 'str', default: true }, + ], + }, + ], + type_aliases: [{ name: 'UserId', definition: 'int', is_generic: false, type_params: [] }], + }, + 'selected named values contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: ['payload_value', 'user_id'].map(name => ({ + ...moduleModel.functions[1]!, + name, + })), + classes: [{ ...moduleModel.classes[0]!, name: 'Payload' }], + typeAliases: [{ name: 'UserId', type: { kind: 'primitive', name: 'int' } }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual([]); + expect(compiled.generated.declaration).toContain('payloadValue(): Promise'); + expect(compiled.generated.declaration).toContain('userId(): Promise'); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'record', + fields: [ + { name: 'x', required: true, value: { kind: 'integer' } }, + { name: 'label', required: false, value: { kind: 'string' } }, + ], + }, + }); + expect(compiled.callables[1]?.result.resolution).toMatchObject({ + status: 'supported', + value: { kind: 'integer', constraint: 'safe-integer' }, + }); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-named-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = ( + functionName === 'user_id' ? Number.MAX_SAFE_INTEGER + 1 : { x: 'bad' } + ) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + payloadValue: () => Promise<{ x: number; label?: string }>; + userId: () => Promise; + }; + await expect(generated.payloadValue()).rejects.toThrow(BridgeValidationError); + await expect(generated.userId()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + it('requires returned dataclass fields even when their constructor has defaults', () => { const point = rawIr.classes[0]!; const ir = validateIrContract( From ebfa17e9ef5e1bea21860c654a03e91ef38eb49f Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:30:03 -0700 Subject: [PATCH 48/50] Reconcile generic safety tests and generated snapshots --- src/core/callable-compiler.ts | 7 ++- test/integration.test.ts | 22 ++++++---- test/menagerie/__snapshots__/gen.test.ts.snap | 44 +++++++++---------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 6b438508..d8b3f3d1 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -933,11 +933,14 @@ function resolveCallable( callableContract: { parameterValues: parameters.map(supportedValue), returnValue: supportedValue(result), - returnValidationType: func.returnType, + returnValidationType: result.resolution.status === 'unresolved' ? func.returnType : undefined, overloads: overloadResults.map((overload, index) => ({ parameterValues: overloadParameters[index]!.map(supportedValue), returnValue: supportedValue(overload), - returnValidationType: func.overloads![index]!.returnType, + returnValidationType: + overload.resolution.status === 'unresolved' + ? func.overloads![index]!.returnType + : undefined, })), }, }; diff --git a/test/integration.test.ts b/test/integration.test.ts index 4bb70bc8..e97d85e6 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -342,16 +342,19 @@ def choose(value: object) -> object: } as any); expect(result.failures).toEqual([]); const declaration = await fsUtils.readFile(join(outDir, 'generic_overload.generated.d.ts')); - expect(declaration).toContain('export function choose(value: T): Promise;'); + expect(declaration).toContain('export function choose(value: T): Promise;'); const consumerPath = join(tempDir, 'consumer.ts'); await writeFile( consumerPath, `import { choose } from './generated/generic_overload.generated.js'; -const text: Promise = choose('key'); -const integer: Promise = choose(2); +const text: Promise = choose('key'); +const integer: Promise = choose(2); +// @ts-expect-error An unresolved TypeVar does not validate the decoded result. +const unsafe: Promise = choose('key'); void text; void integer; +void unsafe; `, 'utf8' ); @@ -455,9 +458,9 @@ class Container(Generic[T]): const typescript = await fsUtils.readFile(join(outDir, 'generic_module.generated.ts')); const declaration = await fsUtils.readFile(join(outDir, 'generic_module.generated.d.ts')); - expect(typescript).toContain('export function identity(x: T): Promise;'); + expect(typescript).toContain('export function identity(x: T): Promise;'); expect(typescript).toContain( - 'export async function forward(container: Container): Promise>' + 'export async function forward(container: Container): Promise' ); expect(typescript).toContain( 'export async function acceptTransform

(' @@ -507,9 +510,11 @@ export declare function createReturnValidator(schema: ReturnSchema, const pair: Pair = ['a', 'b']; const transform: Transform<[number], string> = (...args) => String(args[0]); const container = {} as Container; -const accepted: Promise> = acceptTransform<[number], string>(transform); -const forwarded: Promise> = forward(container); -const resolved: Promise = identity(1); +const accepted: Promise = acceptTransform<[number], string>(transform); +const forwarded: Promise = forward(container); +const resolved: Promise = identity(1); +// @ts-expect-error The decoded generic result has no supported value contract. +const unsafe: Promise = identity(1); const passthroughResult: Promise = passthrough([1, 2], { flag: true }); void pair; @@ -517,6 +522,7 @@ void transform; void accepted; void forwarded; void resolved; +void unsafe; void passthroughResult; `, 'utf-8' diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 9f07d690..42230ec6 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -47,9 +47,9 @@ export async function generatorValues(): Promise { const __validategeneric_identityResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.generic_identity", __tywrapReturnDefinitions); -export async function genericIdentity(value: T): Promise { +export async function genericIdentity(value: T): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'generic_identity', __args, undefined, __validategeneric_identityResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'generic_identity', __args, undefined, __validategeneric_identityResult); } const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.int_key_dict", __tywrapReturnDefinitions); @@ -68,9 +68,9 @@ export async function iteratorValues(): Promise { const __validateliteral_echoResult = createReturnValidator({"kind":"union","options":[{"kind":"literal","value":"red"},{"kind":"literal","value":"blue"},{"kind":"literal","value":7}]}, "fixtures.typing_torture.literal_echo", __tywrapReturnDefinitions); -export async function literalEcho(value: "red" | "blue" | 7): Promise<"red" | "blue" | 7> { +export async function literalEcho(value: "red" | "blue" | 7): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call<"red" | "blue" | 7>('fixtures.typing_torture', 'literal_echo', __args, undefined, __validateliteral_echoResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'literal_echo', __args, undefined, __validateliteral_echoResult); } const __validatenested_containersResult = createReturnValidator({"kind":"array","element":{"kind":"record","fields":{},"values":{"kind":"array","element":{"kind":"tuple","elements":[{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"string"}]}}}}, "fixtures.typing_torture.nested_containers", __tywrapReturnDefinitions); @@ -100,7 +100,7 @@ const __validateoverloadedResult = createReturnValidator({"kind":"union","option const __validateparamspec_applyResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.paramspec_apply", __tywrapReturnDefinitions); -export async function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise { +export async function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise { let __kwargs = kwargs; const __args: unknown[] = [callback]; let __varargs: unknown[] = []; @@ -114,17 +114,17 @@ export async function paramspecApply

(callback: (...args: } } __args.push(...__varargs); - return getRuntimeBridge().call('fixtures.typing_torture', 'paramspec_apply', __args, __kwargs, __validateparamspec_applyResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'paramspec_apply', __args, __kwargs, __validateparamspec_applyResult); } const __validateprotocol_identityResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.protocol_identity", __tywrapReturnDefinitions); -export async function protocolIdentity(value: SupportsClose): Promise { +export async function protocolIdentity(value: SupportsClose): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'protocol_identity', __args, undefined, __validateprotocol_identityResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'protocol_identity', __args, undefined, __validateprotocol_identityResult); } -const __validaterated_movieResult = createReturnValidator({"kind":"ref","name":"RatedMovie"}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); +const __validaterated_movieResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":true}}}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); export async function ratedMovie(): Promise { const __args: unknown[] = []; @@ -145,14 +145,14 @@ export async function tupleKeyDict(value: unknown): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); } -const __validatetyped_dict_echoResult = createReturnValidator({"kind":"ref","name":"Movie"}, "fixtures.typing_torture.typed_dict_echo", __tywrapReturnDefinitions); +const __validatetyped_dict_echoResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false}}}, "fixtures.typing_torture.typed_dict_echo", __tywrapReturnDefinitions); export async function typedDictEcho(value: Movie): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'typed_dict_echo', __args, undefined, __validatetyped_dict_echoResult); } -const __validateuser_id_echoResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.user_id_echo", __tywrapReturnDefinitions); +const __validateuser_id_echoResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.typing_torture.user_id_echo", __tywrapReturnDefinitions); export async function userIdEcho(value: UserId): Promise { const __args: unknown[] = [value]; @@ -194,13 +194,13 @@ export function frozenValues(value: string[]): Promise; export function generatorValues(): Promise; -export function genericIdentity(value: T): Promise; +export function genericIdentity(value: T): Promise; export function intKeyDict(value: unknown): Promise; export function iteratorValues(): Promise; -export function literalEcho(value: "red" | "blue" | 7): Promise<"red" | "blue" | 7>; +export function literalEcho(value: "red" | "blue" | 7): Promise; export function nestedContainers(value: Record[]): Promise[]>; @@ -210,9 +210,9 @@ export function optionalUnion(value: string | number | null): Promise(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; +export function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; -export function protocolIdentity(value: SupportsClose): Promise; +export function protocolIdentity(value: SupportsClose): Promise; export function ratedMovie(): Promise; @@ -290,12 +290,12 @@ export async function dataclassInstance(): Promise { const __validatedecimal_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.decimal_values", __tywrapReturnDefinitions); -export async function decimalValues(): Promise { +export async function decimalValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args, undefined, __validatedecimal_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args, undefined, __validatedecimal_valuesResult); } -const __validatedeeply_nestedResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.deeply_nested", __tywrapReturnDefinitions); +const __validatedeeply_nestedResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.deeply_nested", __tywrapReturnDefinitions); export async function deeplyNested(): Promise { const __args: unknown[] = []; @@ -309,7 +309,7 @@ export async function echoInt(value: number): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'echo_int', __args, undefined, __validateecho_intResult); } -const __validateempty_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.empty_values", __tywrapReturnDefinitions); +const __validateempty_valuesResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"tuple","elements":[{"kind":"any"}]},{"kind":"array","element":{"kind":"any"}},{"kind":"record","values":{"kind":"any"}},{"kind":"array","element":{"kind":"any"}}]}, "fixtures.values_torture.empty_values", __tywrapReturnDefinitions); export async function emptyValues(): Promise { const __args: unknown[] = []; @@ -426,7 +426,7 @@ export async function specialFloats(include?: boolean): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'special_floats', __args, undefined, __validatespecial_floatsResult); } -const __validatetemporal_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.temporal_values", __tywrapReturnDefinitions); +const __validatetemporal_valuesResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.temporal_values", __tywrapReturnDefinitions); export async function temporalValues(): Promise { const __args: unknown[] = []; @@ -447,7 +447,7 @@ export async function unicodeText(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'unicode_text', __args, undefined, __validateunicode_textResult); } -const __validateuuid_and_pathResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.uuid_and_path", __tywrapReturnDefinitions); +const __validateuuid_and_pathResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.uuid_and_path", __tywrapReturnDefinitions); export async function uuidAndPath(): Promise { const __args: unknown[] = []; @@ -481,7 +481,7 @@ export function coroutineValue(): Promise; export function dataclassInstance(): Promise; -export function decimalValues(): Promise; +export function decimalValues(): Promise; export function deeplyNested(): Promise; From ec2dbb682c7e85956354ecf899f3f810e9c25a98 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:36:42 -0700 Subject: [PATCH 49/50] Compose local value contracts and guard foreign validation --- docs/maintainers/callable-contracts.md | 5 +- src/core/callable-compiler.ts | 64 ++++---- src/core/generator.ts | 15 +- test/callable-compiler.test.ts | 150 ++++++++++++++++++ test/menagerie/__snapshots__/gen.test.ts.snap | 4 +- 5 files changed, 203 insertions(+), 35 deletions(-) diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md index 9c54a8c5..8de1ea45 100644 --- a/docs/maintainers/callable-contracts.md +++ b/docs/maintainers/callable-contracts.md @@ -60,8 +60,9 @@ The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow int64 values can decode as `bigint`. Selected local non-generic `TypedDict` classes use revision-2 record contracts -when every field has a supported conversion. Selected simple aliases use their -underlying supported contract. Foreign and recursive names remain unresolved. +when every field has a supported conversion. Nested records and selected simple +aliases compose through the same resolver. Foreign and recursive names remain +unresolved. ## Diagnostics and fallback diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index d8b3f3d1..33d7953f 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -43,6 +43,10 @@ export interface ValueConversionRequest { path: string; /** Internal recursion counter. Revision 2 rejects contracts deeper than 64 nodes. */ depth?: number; + /** Compile-local resolver for nested named values. */ + resolveNested?: (request: ValueConversionRequest) => ValueResolution; + /** Exact local names already being expanded, used to stop recursive aliases. */ + activeNames?: readonly string[]; } export interface SupportedValueResolution { @@ -255,7 +259,7 @@ function resolveTuple( }, }; } - if (items.length === 2 && leafName(items[1]!) === '...') { + if (items.length === 2 && items[1]?.kind === 'custom' && items[1].name === '...') { return resolveSequence(items[0]!, request, conversion); } const entries = items.map((item, index) => @@ -389,6 +393,9 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { revision: VALUE_CONTRACT_REVISION, resolve(request): ValueResolution { const { logicalType: type } = request; + const nestedConversion: ValueConversionDescription = request.resolveNested + ? { revision: VALUE_CONTRACT_REVISION, resolve: request.resolveNested } + : DEFAULT_VALUE_CONVERSION; if ((request.depth ?? 0) > 64) { return { status: 'unsupported', @@ -442,18 +449,18 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { guidance: 'Use a supported primitive or an explicit value adapter.', }; case 'annotated': - return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.base }); + return nestedConversion.resolve({ ...request, logicalType: type.base }); case 'final': case 'classvar': - return DEFAULT_VALUE_CONVERSION.resolve({ ...request, logicalType: type.type }); + return nestedConversion.resolve({ ...request, logicalType: type.type }); case 'optional': return resolveUnion( [type.type, { kind: 'primitive', name: 'None' }], request, - DEFAULT_VALUE_CONVERSION + nestedConversion ); case 'union': - return resolveUnion(type.types, request, DEFAULT_VALUE_CONVERSION); + return resolveUnion(type.types, request, nestedConversion); case 'collection': if (type.name === 'dict') { const key = type.itemTypes[0]; @@ -464,20 +471,12 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { guidance: 'Convert keys to strings before crossing the value RPC boundary.', }; } - return resolveRecord( - type.itemTypes[1] ?? UNKNOWN_TYPE, - request, - DEFAULT_VALUE_CONVERSION - ); + return resolveRecord(type.itemTypes[1] ?? UNKNOWN_TYPE, request, nestedConversion); } if (type.name === 'tuple') { - return resolveTuple(type.itemTypes, request, DEFAULT_VALUE_CONVERSION); + return resolveTuple(type.itemTypes, request, nestedConversion); } - return resolveSequence( - type.itemTypes[0] ?? UNKNOWN_TYPE, - request, - DEFAULT_VALUE_CONVERSION - ); + return resolveSequence(type.itemTypes[0] ?? UNKNOWN_TYPE, request, nestedConversion); case 'generic': { const leaf = leafName(type); if (leaf === 'NDArray' || leaf === 'ndarray') { @@ -496,11 +495,7 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { return resolveTorchFloat16(); } if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { - return resolveSequence( - type.typeArgs[0] ?? UNKNOWN_TYPE, - request, - DEFAULT_VALUE_CONVERSION - ); + return resolveSequence(type.typeArgs[0] ?? UNKNOWN_TYPE, request, nestedConversion); } if (['dict', 'Dict', 'Mapping', 'MutableMapping'].includes(leaf ?? '')) { const key = type.typeArgs[0]; @@ -511,7 +506,7 @@ export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { guidance: 'Convert keys to strings before crossing the value RPC boundary.', }; } - return resolveRecord(type.typeArgs[1] ?? UNKNOWN_TYPE, request, DEFAULT_VALUE_CONVERSION); + return resolveRecord(type.typeArgs[1] ?? UNKNOWN_TYPE, request, nestedConversion); } return { status: 'unresolved', annotation: annotationName(type) }; } @@ -1009,11 +1004,20 @@ export function compileContract( revision: options.conversion.revision, resolve(request): ValueResolution { const type = request.logicalType; - if (type.kind !== 'custom' || (type.module !== undefined && type.module !== ir.module)) { - return options.conversion.resolve(request); + const direct = options.conversion.resolve({ ...request, resolveNested: conversion.resolve }); + if ( + direct.status !== 'unresolved' || + type.kind !== 'custom' || + (type.module !== undefined && type.module !== ir.module) + ) { + return direct; } const typedDict = localTypedDicts.get(type.name); if (typedDict) { + const key = `typed-dict:${type.name}`; + if (request.activeNames?.includes(key)) { + return { status: 'unresolved', annotation: annotationName(type) }; + } if ((request.depth ?? 0) >= 64) { return { status: 'unsupported', @@ -1023,11 +1027,12 @@ export function compileContract( } const fields: ValueContractField[] = []; for (const property of typedDict.properties) { - const field = options.conversion.resolve({ + const field = conversion.resolve({ ...request, logicalType: property.type, path: `${request.path}.${property.name}`, depth: (request.depth ?? 0) + 1, + activeNames: [...(request.activeNames ?? []), key], }); if (field.status !== 'supported') { return field; @@ -1041,13 +1046,18 @@ export function compileContract( } const alias = localAliases.get(type.name); if (alias) { - return options.conversion.resolve({ + const key = `alias:${type.name}`; + if (request.activeNames?.includes(key)) { + return { status: 'unresolved', annotation: annotationName(type) }; + } + return conversion.resolve({ ...request, logicalType: alias.type, depth: (request.depth ?? 0) + 1, + activeNames: [...(request.activeNames ?? []), key], }); } - return options.conversion.resolve(request); + return direct; }, }; const capabilities = new Map( diff --git a/src/core/generator.ts b/src/core/generator.ts index 41033419..41cddf4e 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -39,7 +39,7 @@ interface GenericRenderContext { emittedParamSpecs: Set; } -type ReturnDefinitionNames = ReadonlySet; +type ReturnDefinitionNames = ReadonlySet & { readonly moduleName?: string }; function valueContractToReturnSchema(value: ValueContract): ReturnSchema { switch (value.kind) { @@ -605,7 +605,10 @@ export class CodeGenerator { values: schema(current.typeArgs[1] ?? { kind: 'custom', name: 'Any' }), }; } - return definitions.has(leaf) ? { kind: 'ref', name: leaf } : { kind: 'any' }; + return definitions.has(leaf) && + (current.module === undefined || current.module === definitions.moduleName) + ? { kind: 'ref', name: leaf } + : { kind: 'any' }; } case 'custom': { const leaf = current.name.split('.').at(-1) ?? current.name; @@ -640,7 +643,10 @@ export class CodeGenerator { ) { return { kind: 'marker', marker: 'sklearn.estimator' }; } - return definitions.has(leaf) ? { kind: 'ref', name: leaf } : { kind: 'any' }; + return definitions.has(leaf) && + (current.module === undefined || current.module === definitions.moduleName) + ? { kind: 'ref', name: leaf } + : { kind: 'any' }; } case 'typevar': case 'paramspec': @@ -714,11 +720,12 @@ export class CodeGenerator { } private returnDefinitions(module: PythonModule): ReturnDefinitionNames { - return new Set( + const names = new Set( module.classes .filter(cls => cls.kind === 'typed_dict' || cls.decorators.includes('__typed_dict__')) .map(cls => cls.name) ); + return Object.assign(names, { moduleName: module.name }); } private emitReturnDefinitions(module: PythonModule): string { diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index b0a2c48a..f8a20fc9 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1500,6 +1500,156 @@ describe('compileContract', () => { } }); + it('composes nested local records and aliases, but stops cycles and foreign names', () => { + const source = rawIr.functions[1]!; + const point = rawIr.classes[0]!; + const typedDict = (name: string, annotation: string) => ({ + ...point, + name, + qualname: `fixture.${name}`, + typed_dict: true, + is_dataclass: false, + fields: [{ name: 'value', kind: 'FIELD', annotation, default: false }], + }); + const returns = [ + ['nested_values', 'list[Outer]'], + ['alias_values', 'AliasAlias'], + ['cycle_values', 'Loop'], + ['foreign_values', 'foreign.Outer'], + ['self_values', 'SelfRef'], + ['qualified_values', 'fixture.Outer'], + ] as const; + const ir = validateIrContract( + { + ...rawIr, + functions: returns.map(([name, annotation]) => ({ + ...source, + name, + qualname: `fixture.${name}`, + returns: annotation, + })), + classes: [ + typedDict('Child', 'int'), + typedDict('Outer', 'Child'), + typedDict('SelfRef', 'SelfRef'), + ], + type_aliases: [ + { name: 'UserId', definition: 'int', is_generic: false, type_params: [] }, + { name: 'AliasAlias', definition: 'UserId', is_generic: false, type_params: [] }, + { name: 'Loop', definition: 'Loop', is_generic: false, type_params: [] }, + ], + }, + 'composed named values contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: returns.map(([name]) => ({ ...moduleModel.functions[1]!, name })), + classes: ['Child', 'Outer', 'SelfRef'].map(name => ({ + ...moduleModel.classes[0]!, + name, + })), + typeAliases: ['UserId', 'AliasAlias', 'Loop'].map(name => ({ + name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'sequence', + item: { + kind: 'record', + fields: [ + { + name: 'value', + value: { + kind: 'record', + fields: [{ name: 'value', value: { kind: 'integer' } }], + }, + }, + ], + }, + }, + }); + expect(compiled.callables[1]?.result.resolution).toMatchObject({ + status: 'supported', + value: { kind: 'integer', constraint: 'safe-integer' }, + }); + expect(compiled.callables[2]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[3]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[4]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[5]?.result.resolution.status).toBe('supported'); + expect(compiled.generated.declaration).toContain('nestedValues(): Promise'); + expect(compiled.generated.declaration).toContain('aliasValues(): Promise'); + expect(compiled.generated.declaration).toContain('cycleValues(): Promise'); + expect(compiled.generated.declaration).toContain('foreignValues(): Promise'); + expect(compiled.generated.declaration).toContain('selfValues(): Promise'); + expect(compiled.generated.declaration).toContain('qualifiedValues(): Promise'); + expect(compiled.generated.typescript).toContain( + 'const __validateforeign_valuesResult = createReturnValidator({"kind":"any"}' + ); + + const rejectIntegers: typeof DEFAULT_VALUE_CONVERSION = { + revision: DEFAULT_VALUE_CONVERSION.revision, + resolve(request) { + if (request.logicalType.kind === 'primitive' && request.logicalType.name === 'int') { + return { + status: 'unsupported', + reason: 'This configured conversion declines integers.', + guidance: 'Use a different conversion.', + }; + } + return DEFAULT_VALUE_CONVERSION.resolve(request); + }, + }; + const declined = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: returns.map(([name]) => ({ ...moduleModel.functions[1]!, name })), + classes: ['Child', 'Outer', 'SelfRef'].map(name => ({ + ...moduleModel.classes[0]!, + name, + })), + typeAliases: ['UserId', 'AliasAlias', 'Loop'].map(name => ({ + name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: rejectIntegers, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(declined.callables[0]?.result.resolution.status).toBe('unsupported'); + expect(declined.callables[1]?.result.resolution.status).toBe('unsupported'); + expect(declined.generated.declaration).toContain('nestedValues(): Promise'); + }); + + it('treats a variadic integer tuple as the supported sequence contract', () => { + const logicalType = parseAnnotationToPythonType('tuple[int, ...]'); + expect( + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType, + }) + ).toMatchObject({ + status: 'supported', + value: { + kind: 'sequence', + item: { kind: 'integer', constraint: 'safe-integer' }, + }, + }); + }); + it('requires returned dataclass fields even when their constructor has defaults', () => { const point = rawIr.classes[0]!; const ir = validateIrContract( diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 42230ec6..6b2979da 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -124,7 +124,7 @@ export async function protocolIdentity(value: SupportsClose): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'protocol_identity', __args, undefined, __validateprotocol_identityResult); } -const __validaterated_movieResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":true}}}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); +const __validaterated_movieResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number","constraint":"finite"},"optional":true}}}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); export async function ratedMovie(): Promise { const __args: unknown[] = []; @@ -159,7 +159,7 @@ export async function userIdEcho(value: UserId): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'user_id_echo', __args, undefined, __validateuser_id_echoResult); } -const __validatevariadic_tupleResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.typing_torture.variadic_tuple", __tywrapReturnDefinitions); +const __validatevariadic_tupleResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.typing_torture.variadic_tuple", __tywrapReturnDefinitions); export async function variadicTuple(value: number[]): Promise { const __args: unknown[] = [value]; From 42feeca0abd1314592be691faa6d608541dfc6d2 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 02:40:57 -0700 Subject: [PATCH 50/50] Bound compile-local alias expansion depth --- src/core/callable-compiler.ts | 7 +++++ test/callable-compiler.test.ts | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts index 33d7953f..9a0d6726 100644 --- a/src/core/callable-compiler.ts +++ b/src/core/callable-compiler.ts @@ -1050,6 +1050,13 @@ export function compileContract( if (request.activeNames?.includes(key)) { return { status: 'unresolved', annotation: annotationName(type) }; } + if ((request.depth ?? 0) >= 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } return conversion.resolve({ ...request, logicalType: alias.type, diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts index f8a20fc9..e154674f 100644 --- a/test/callable-compiler.test.ts +++ b/test/callable-compiler.test.ts @@ -1633,6 +1633,57 @@ describe('compileContract', () => { expect(declined.generated.declaration).toContain('nestedValues(): Promise'); }); + it('bounds local alias expansion when a converter leaves custom names unresolved', () => { + const aliases = Array.from({ length: 70 }, (_, index) => ({ + name: `Alias${index}`, + definition: index === 69 ? 'External' : `Alias${index + 1}`, + is_generic: false, + type_params: [], + })); + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { ...source, name: 'alias_chain', qualname: 'fixture.alias_chain', returns: 'Alias0' }, + ], + type_aliases: aliases, + }, + 'bounded alias chain contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const unresolvedCustom: typeof DEFAULT_VALUE_CONVERSION = { + revision: DEFAULT_VALUE_CONVERSION.revision, + resolve(request) { + if (request.logicalType.kind === 'custom') { + return { status: 'unresolved', annotation: request.logicalType.name }; + } + return DEFAULT_VALUE_CONVERSION.resolve(request); + }, + }; + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: [{ ...moduleModel.functions[1]!, name: 'alias_chain' }], + classes: [], + typeAliases: aliases.map(alias => ({ + name: alias.name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: unresolvedCustom, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + }); + }); + it('treats a variadic integer tuple as the supported sequence contract', () => { const logicalType = parseAnnotationToPythonType('tuple[int, ...]'); expect(