diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92445f5f..f81752d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: - name: Run Python codec and frame tests run: | python -m pip install pytest - python -m pytest test/python/test_bridge_codec.py test/python/test_frame_codec.py + python -m pytest test/python/test_bridge_codec.py test/python/test_bridge_core.py test/python/test_frame_codec.py python-suite-data: if: github.event_name != 'schedule' @@ -223,7 +223,7 @@ jobs: PIP_PROGRESS_BAR: 'off' run: | python -m pip install -e tywrap_ir - python -m pip install numpy==2.3.5 pandas==3.0.2 pyarrow==24.0.0 scipy==1.16.3 scikit-learn==1.8.0 + python -m pip install numpy==2.3.5 pandas==3.0.2 pyarrow==24.0.0 scipy==1.16.3 scikit-learn==1.8.0 pydantic==2.7.4 python -m pip install torch==2.10.0 --index-url https://download.pytorch.org/whl/cpu - name: Run pinned scientific menagerie env: diff --git a/docs/guide/runtimes/http.md b/docs/guide/runtimes/http.md index c5cca83f..e91f556d 100644 --- a/docs/guide/runtimes/http.md +++ b/docs/guide/runtimes/http.md @@ -49,6 +49,11 @@ setRuntimeBridge( payloads. You must provide a compatible server endpoint. Each call is an independent POST request because the protocol is stateless. +Generated async callables have typed results when their return annotations have +supported value conversions. The in-repo Node and Pyodide bridges await Python +coroutines. For `HttpBridge`, your server must await coroutines before it sends +the result. `HttpBridge` cannot verify a remote server's coroutine support. + tywrap does not include an HTTP server. See the [API reference](/reference/api/) for the expected request and response format. diff --git a/docs/maintainers/callable-contracts.md b/docs/maintainers/callable-contracts.md new file mode 100644 index 00000000..8de1ea45 --- /dev/null +++ b/docs/maintainers/callable-contracts.md @@ -0,0 +1,114 @@ +# Callable compilation contracts + +`tywrap_ir` emits IR 0.4. `validateIrContract()` checks required fields in +unknown JSON. Each diagnostic has a JSON path. Validation stops after 100,000 +array entries. + +`compileContract()` is an internal pure step. It accepts validated IR, +exported-name selection, value conversions, capabilities, and a generator. It +maps callable types from the validated IR. The selection input cannot change +those types. It returns the resolved module, callable records, diagnostics, and +TypeScript and declaration content. + +The compiler does not read files. It does not start Python. It does not read a +cache. It does not write files. It does not access a runtime registry. + +`generate()` owns Python discovery, cache reads, output comparison, and writes. +It validates IR and maps annotations, then calls `compileContract()` before +emission. Offline contract files omit extractor metadata for stable bytes. The +reader restores an empty metadata object. + +## Resolved callable data + +The compiler records each parameter and result, including overloads. Each record +has a Python type, direction, source path, and conversion result. One resolved +module model serves declarations, call wrappers, and return validators. + +Each value conversion uses revision 2 of +[value-contracts.v2.json](value-contracts.v2.json). Do not infer a wire rule +from the TypeScript type. + +| Case | Result | +| --------------------- | ------------------------------------------------------------------------------------- | +| `int` | JSON number, JavaScript number, safe-integer constraint | +| `bytes` | Existing base64 envelope, decoded as `Uint8Array` | +| nested `dict[str, T]` | JSON record with the child conversion at every value path | +| `str \| int` | Selected union alternative with one child contract per option | +| `tuple[int, str]` | Exact JSON array with a contract for each index | +| float16 ndarray | Existing Arrow or JSON-fallback envelope, decoded as a numeric scalar or nested array | +| float16 Torch tensor | Outer `torch.float16` envelope with a nested `float16` ndarray | + +The compiler requires an explicit float16 dtype. An untyped ndarray or Torch +tensor stays unresolved. + +For an evaluated PyTorch annotation, use `torch.HalfTensor`. The compiler +accepts this exact class and leaves other Torch tensor classes unresolved. + +NumPy can spell `NDArray[np.float16]` as +`numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]`. The compiler reads +the dtype argument, not the shape wildcard. Only a supported scientific result +clears its matching external-annotation IR warning. Other external annotations +remain warnings. + +The codec records scalar ndarray dtype and rank for one call. A generated return +validator accepts a scalar only with matching proof. Nested proof follows its +parent and array index or record key. Array index `0` and `"0"` refer to the +same child. Equal numbers from separate calls do not share proof. Array and +tensor objects keep their identity-based marker proof. + +The compiler keeps Arrow int64 behavior outside this prototype. Existing Arrow +int64 values can decode as `bigint`. + +Selected local non-generic `TypedDict` classes use revision-2 record contracts +when every field has a supported conversion. Nested records and selected simple +aliases compose through the same resolver. Foreign and recursive names remain +unresolved. + +## Diagnostics and fallback + +Known unsupported cases create error diagnostics. The compiler changes the +affected generated type to `unknown`. `generate()` returns each diagnostic as a +warning, so `--fail-on-warn` rejects strict builds. + +- The default compiler capability covers coroutine execution in the in-repo Node + and Pyodide bridges. A caller can mark this capability unavailable when it + uses an older or custom bridge. The compiler then emits `unknown` and an error + diagnostic for async results. +- Dataclasses require an adapter owned by #339. The default capability stays + unavailable. An explicit output record conversion can use the adapter + capability when enabled. +- Non-string record keys require an explicit conversion. + +Unresolved annotations create warning diagnostics. Their result type is +`unknown`, including an output that contains `object` or a type variable. Python +`object` can hold a primitive value. A type variable has no validated conversion +to the decoded result. Neither annotation justifies a precise TypeScript return. + +The compiler keeps an existing best-effort return check when it widens a +declaration. For example, an unresolved `list[object]` still rejects a non-array +result, and a `DataFrame` annotation still requires a decoded marker. + +`Literal` returns keep their exact-value validator, but stay `unknown` until a +literal value contract exists. Unresolved literal overloads cannot select a +precise return at runtime. + +IR overload signatures stay separate from the implementation signature. The +extractor collects type variables from overload annotations. Generated +declarations retain each input-to-result relation. The hidden TypeScript +implementation accepts `unknown` values so it can cover every declared overload. + +At runtime, the wrapper selects the first supported overload that matches bound +arguments. This follows TypeScript declaration order. The compiler warns when +overload inputs may overlap, including an omitted optional argument. Python 3.10 +cannot expose registered overloads through `typing.get_overloads`; it keeps the +implementation signature and emits an IR warning. + +## Deferred work + +Revision 2 does not describe references or recursive aliases. The compiler +reports these cases as unresolved. A later value-contract revision can give them +a supported conversion. + +The client-binding work may consume the resolved callable model later. It must +remain opt-in. It must not add files to default generation or mutate the global +runtime registry. diff --git a/docs/maintainers/value-contract-fixtures.v2.json b/docs/maintainers/value-contract-fixtures.v2.json new file mode 100644 index 00000000..78337545 --- /dev/null +++ b/docs/maintainers/value-contract-fixtures.v2.json @@ -0,0 +1,63 @@ +{ + "revision": 2, + "integerCases": [ + { "decimal": "-9007199254740991", "outcome": "number" }, + { "decimal": "9007199254740991", "outcome": "number" }, + { "decimal": "-9007199254740992", "outcome": "reject" }, + { "decimal": "9007199254740992", "outcome": "reject" }, + { "decimal": "9007199254740993", "outcome": "reject" }, + { "decimal": "-9223372036854775808", "outcome": "reject" }, + { "decimal": "9223372036854775807", "outcome": "reject" }, + { "decimal": "265252859812191058636308480000000", "outcome": "reject" } + ], + "bytesCases": [ + { + "logicalHex": "00ff80", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "AP+A" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "AP+A" }, + "decodedAs": "Uint8Array[0,255,128]" + }, + { + "logicalHex": "", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "" }, + "decodedAs": "Uint8Array[]" + } + ], + "binary16Cases": [ + { "wordHex": "0000", "outcome": "0" }, + { "wordHex": "8000", "outcome": "-0" }, + { "wordHex": "0001", "outcome": "5.960464477539063e-8" }, + { "wordHex": "03ff", "outcome": "0.00006097555160522461" }, + { "wordHex": "0400", "outcome": "0.00006103515625" }, + { "wordHex": "3e00", "outcome": "1.5" }, + { "wordHex": "c080", "outcome": "-2.25" }, + { "wordHex": "7bff", "outcome": "65504" }, + { "wordHex": "fbff", "outcome": "-65504" }, + { "wordHex": "7c00", "outcome": "reject" }, + { "wordHex": "fc00", "outcome": "reject" }, + { "wordHex": "7e00", "outcome": "reject" } + ], + "compositionCases": [ + { + "path": "result.outer[0].value", + "logicalValue": "9007199254740993", + "outcome": "reject" + }, + { + "path": "result.outer[0].tensor.value", + "logicalValue": "float16[1.5,-2.25]", + "outcome": "number[1.5,-2.25]" + } + ], + "unionCases": [ + { "contract": "str|int", "value": "label", "outcome": "string" }, + { "contract": "str|int", "value": 42, "outcome": "safe-integer number" }, + { "contract": "str|int", "value": 9007199254740992, "outcome": "reject" } + ], + "tupleCases": [ + { "contract": "tuple[str,int]", "value": ["label", 42], "outcome": "accept" }, + { "contract": "tuple[str,int]", "value": ["label"], "outcome": "reject-length" }, + { "contract": "tuple[str,int]", "value": ["label", 9007199254740992], "outcome": "reject-integer" } + ] +} diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md new file mode 100644 index 00000000..20c8293f --- /dev/null +++ b/docs/maintainers/value-contracts.md @@ -0,0 +1,113 @@ +# Value contracts + +The frozen internal prototype is +[value-contracts.v2.json](value-contracts.v2.json). That JSON file owns the +policy. The TypeScript interface mirrors it and has revision 2. + +Each contract names a logical value, its wire encoding, its decoded JavaScript +value, and its constraints. Sequences, fixed tuples, records, and unions compose +child contracts. A union has two to 32 alternatives. The compiler rejects a +union if one wire value could decode two ways. Contract depth stops at 64. Named +records can inline within that limit. Recursive aliases stay unsupported until a +reviewed reference graph exists. The prototype covers float16 ndarrays and Torch +tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 +may decode as `bigint`. Unsupported conversions carry a reason and guidance. + +The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow +envelopes, JSON fallback, and transports keep their shape. The producer rejects +unsafe Python integers before it writes JSON. A generated wrapper that consumes +this interface must keep `number` for Python `int` and add a safe-integer return +constraint. Regenerate wrappers after that compiler change. Do not return +`bigint` under an existing `Promise`. + +The bytes leaf keeps the existing base64 envelopes. Requests use +`__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to +`Uint8Array`. The Torch float16 contract names `torch.float16` on the outer +tensor and `float16` on its nested version 1 ndarray. The decoder rejects a +different nested dtype before it decodes the data. + +Migration for users with large Python integers: convert them to strings in +Python with an explicit return annotation, or keep integer columns in Arrow +tables. JSON fallback for ndarray and pandas values still rejects unsafe +integers. This is a behavior change from silent rounding to a call error. + +The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer +boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused +tests also compare every finite binary16 storage word with an independent +float32 bit reference. They test signed zero with `Object.is`. + +The Python serializer checks plain integers, NumPy scalar integers, and model +dumps during its existing value walk. It also checks terminal estimator +parameters and Series name metadata before JSON output. A JSON ndarray or sparse +matrix checks integer data before it creates an envelope. Sparse metadata checks +shapes and indices. The TypeScript decoder converts float16 Arrow storage during +extraction, before it reshapes the ndarray. Torch uses that same ndarray path. +These checks do not walk Arrow payloads again for schema validation. + +## Exact bigint extension for #337 + +This extension is a design sketch. The 0.11 policy still rejects unsafe plain +Python integers. + +Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map +Python `int` to TypeScript `bigint` for both parameters and returns. The return +validator must require `typeof value === 'bigint'`, including every record and +sequence member. Keep `bool` separate. Calls without the option keep `number` +and the safe range. + +Add a version 2 integer value envelope with a canonical signed decimal string. A +sample value is +`{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. +The request encoder and Python request decoder must validate the same grammar +and digit limit. The response encoder and TypeScript decoder must do the same. +Use `BigInt` only after validation. Keep Arrow int64 columns on their current +Arrow path. + +The bridge must advertise exact integer support during capability negotiation. A +generated bigint wrapper must reject an older bridge before its first call. The +new bridge must emit version 2 integer envelopes only when the call opts in. An +older client therefore continues to receive the 0.11 safe integer policy. +Unknown integer envelopes, mixed versions, oversized strings, and malformed +decimal values must fail with a path. Do not guess a conversion. + +Acceptance evidence still needed: a bounded prototype must round trip positive +and negative values beyond 64 bits through nested records and arrays. It must +test bigint inputs, malformed tags, payload limits, and both mixed-version +directions. An independent review must approve the wire grammar, capability +rule, generated types, and migration cost before #337 can close. + +## Dataclass return extension for #339 + +This extension is a design sketch. Dataclass inputs remain unsupported and need +a separate input contract. Users must pass a plain record through an explicitly +typed function if they need that behavior now. + +For an annotated dataclass return, compile a structural TypeScript record from +`dataclasses.fields`. Include each instance field in the output, including +fields with defaults or `init=False`. A default does not make a returned field +optional. A field annotated `Optional[T]` permits `null`. Compose nested +dataclasses, records, and sequences only when every field has a supported value +contract. Reject ordinary class instances and unsupported fields with their +result paths. + +Use a version 2 dataclass envelope with a declared type identity and a `fields` +record. For example, `Point(1, 2)` would carry +`{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. +The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must +check the type identity, exact field names, field values, and required fields. +It must reject extra fields unless the compiled contract allows them. The Python +encoder must use the current bounded value walk. It must detect cycles and +enforce depth, node, and payload limits without calling `dataclasses.asdict` as +a shortcut. + +The bridge must advertise dataclass value support. A generated dataclass wrapper +must reject an older bridge before calling it. A new bridge must send the +envelope only when the wrapper requests it. This leaves existing value RPC and +ordinary class behavior unchanged. + +Acceptance evidence still needed: a generated `Promise` wrapper must +return the exact plain record and reject missing, extra, or wrong typed fields. +Tests must cover optional and defaulted fields, nested values, cycles, +unsupported fields, depth, and payload limits. An independent review must +approve the type identity, version rule, and migration cost before #339 can +close. diff --git a/docs/maintainers/value-contracts.v2.json b/docs/maintainers/value-contracts.v2.json new file mode 100644 index 00000000..827a766c --- /dev/null +++ b/docs/maintainers/value-contracts.v2.json @@ -0,0 +1,85 @@ +{ + "revision": 2, + "status": "frozen-internal-prototype", + "transport": "tywrap/1 value RPC", + "codecVersion": 1, + "rules": { + "integer": { + "logicalKind": "Python int, excluding bool", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "wire": "JSON integer number", + "decodedAs": "JavaScript number", + "failure": "reject before JSON encoding with result path and explicit conversion guidance" + }, + "float": { + "logicalKind": "finite Python float", + "constraint": "finite", + "wire": "JSON number", + "decodedAs": "JavaScript number", + "failure": "reject NaN and Infinity" + }, + "bytes": { + "logicalKind": "Python bytes or bytearray", + "requestWire": "existing __tywrap_bytes__ base64 envelope", + "responseWire": "existing __type__: bytes base64 envelope", + "decodedAs": "JavaScript Uint8Array", + "failure": "reject malformed base64 or payloads above the existing byte limit" + }, + "sequence": { + "logicalKind": "list or variadic tuple of one contracted item kind", + "wire": "JSON array", + "decodedAs": "JavaScript array", + "composition": "apply each item rule at its result path" + }, + "tuple": { + "logicalKind": "fixed-length heterogeneous tuple of contracted values", + "wire": "JSON array", + "decodedAs": "JavaScript array with one contracted value per position", + "composition": "require exact length and apply each item rule at its result index; an empty tuple is valid" + }, + "union": { + "logicalKind": "one value matching a nonempty set of contracted alternatives", + "wire": "the selected alternative's existing wire encoding", + "decodedAs": "the selected alternative's JavaScript value", + "minimumAlternatives": 2, + "maximumAlternatives": 32, + "composition": "require at least one matching alternative; reject a contract if the same wire value can match alternatives with different decoded representations; order has no semantic effect", + "failure": "reject an empty or one-option union during contract construction; reject a value that matches no alternative with its path" + }, + "record": { + "logicalKind": "string-keyed record of contracted values", + "wire": "JSON object", + "decodedAs": "JavaScript object", + "composition": "apply each field rule at its result path" + }, + "ndarray-float16": { + "logicalKind": "NumPy float16 ndarray", + "wire": "codecVersion 1 ndarray Arrow envelope; JSON fallback stays explicit", + "decodedAs": "JavaScript numeric scalar or nested array", + "conversion": "decode each Arrow Uint16 storage word as IEEE 754 binary16 before reshape", + "failure": "reject non-finite converted values" + }, + "torch-float16": { + "logicalKind": "Torch float16 tensor", + "wire": "codecVersion 1 torch.tensor envelope containing ndarray envelope", + "decodedAs": "tensor record with numeric data", + "outerDtype": "torch.float16", + "nestedDtype": "float16", + "conversion": "reuse ndarray-float16 rule for nested value" + } + }, + "prototypeBoundary": "Only float16 scientific values use the new ndarray contract. Existing Arrow int64 values may decode as bigint and keep their current rules.", + "contractLimits": { + "maximumDepth": 64, + "maximumUnionAlternatives": 32, + "namedRecordAliases": "inline nonrecursive aliases within the depth limit", + "recursiveAliases": "unsupported until a reviewed reference-graph contract exists" + }, + "compatibility": { + "wireChange": false, + "unknownVersion": "reject", + "unsafeInteger": "previous clients could round; new producers reject", + "generatedWrappers": "regenerate when callable-contract code consumes revision 2" + } +} diff --git a/docs/public/llms-full.txt b/docs/public/llms-full.txt index 47a36b49..ad493d75 100644 --- a/docs/public/llms-full.txt +++ b/docs/public/llms-full.txt @@ -380,7 +380,7 @@ and `tywrap:check` scripts to `package.json` (disable with `--no-scripts`). 1. Set `"caching": true` to reuse generation IR across rebuilds. 2. Limit `functions` and `classes` when a wrapper needs only part of a module. -`batching` and `compression` are accepted config fields in 0.9.0, but no current +`batching` and `compression` are accepted config fields in 0.10.0, but no current generator or runtime code applies them. ```json @@ -773,7 +773,7 @@ types: { ``` Accepted presets are `numpy`, `pandas`, `pydantic`, `stdlib`, `scipy`, `torch`, -and `sklearn`. `numpy` is accepted as a no-op in 0.9.0. The other presets map +and `sklearn`. `numpy` is accepted as a no-op in 0.10.0. The other presets map the library types implemented by the generator, such as `DataFrame`, sparse matrix classes, `Tensor`, and `BaseEstimator`. @@ -1852,6 +1852,11 @@ setRuntimeBridge( payloads. You must provide a compatible server endpoint. Each call is an independent POST request because the protocol is stateless. +Generated async callables have typed results when their return annotations have +supported value conversions. The in-repo Node and Pyodide bridges await Python +coroutines. For `HttpBridge`, your server must await coroutines before it sends +the result. `HttpBridge` cannot verify a remote server's coroutine support. + tywrap does not include an HTTP server. See the [API reference](https://bbopen.github.io/tywrap/reference/api/) for the expected request and response format. @@ -2439,7 +2444,7 @@ Generates TypeScript shaped like: export type Pair = [T, T]; export type Transform

= (...args: P) => T; export class Container { - // NOTE: Instance members are not generated in v0.9; migrate this API to value-returning module functions. + // NOTE: Instance members are not generated in v0.10; migrate this API to value-returning module functions. } ``` @@ -2747,7 +2752,7 @@ so a payload can exceed the JSONL line ceiling. The capability is statically [Transport framing](https://bbopen.github.io/tywrap/transport-framing.md) for the wire format. `supportsStreaming` (incremental results for a single request) is `false` on -every backend. It is not implemented as of 0.9.0. +every backend. It is not implemented as of 0.10.0. ### `maxFrameBytes` @@ -2801,7 +2806,7 @@ through a `PooledTransport` lease reassembles correctly (see ## Scope: subprocess only -`tywrap-frame/1` is **subprocess-only** as of 0.9.0. The subprocess transport is +`tywrap-frame/1` is **subprocess-only** as of 0.10.0. The subprocess transport is the only backend with a real frame ceiling, the JSONL line-length limit. The other backends stay single-frame and keep `supportsChunking: false`: @@ -2811,7 +2816,7 @@ other backends stay single-frame and keep `supportsChunking: false`: - **Pyodide** is in-memory (`maxFrameBytes: Number.POSITIVE_INFINITY`, JSON-only). There is no wire to fragment. -`supportsStreaming` stays `false` on every backend as of 0.9.0. +`supportsStreaming` stays `false` on every backend as of 0.10.0. ## Layering @@ -2890,7 +2895,7 @@ construction. The codepoint-boundary rule is a sender obligation: a frame's `data` MUST NOT split a multi-byte UTF-8 sequence, so the receiver can decode each frame as valid UTF-8 and concatenate without re-aligning bytes across frame edges. `utf8-base64` remains a defined alternative in the wire schema for future -use (e.g. a non-text payload), but tywrap does not emit it as of 0.9.0. +use (e.g. a non-text payload), but tywrap does not emit it as of 0.10.0. > Implementation note for later workstreams: `total` and the per-frame split > points are computed over the message's UTF-8 **byte** length against @@ -3072,6 +3077,12 @@ When `output.declaration` is enabled, `generate()` writes matching code. Simple `TypeVar` and callable `ParamSpec` declarations are preserved when tywrap can represent them safely. +Unresolved return annotations now emit `Promise` and a located warning. +This includes `TypeVar` and `ParamSpec` results without a supported value +conversion. Input generics remain in the declaration. Earlier versions could +emit `Promise` without validating the decoded value as `T`; callers must +narrow the result or use a supported concrete overload. + `generate()` returns: ```ts @@ -3555,7 +3566,7 @@ console.log(await sin(pi / 4)); ## Value-returning APIs -v0.9 generated wrappers do not keep live Python class instances. Expose an +v0.10 generated wrappers do not keep live Python class instances. Expose an operation as a value-returning module function instead: ```ts diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md index 26a5e3b5..2d4a6976 100644 --- a/docs/reference/api/index.md +++ b/docs/reference/api/index.md @@ -59,6 +59,12 @@ When `output.declaration` is enabled, `generate()` writes matching code. Simple `TypeVar` and callable `ParamSpec` declarations are preserved when tywrap can represent them safely. +Unresolved return annotations now emit `Promise` and a located warning. +This includes `TypeVar` and `ParamSpec` results without a supported value +conversion. Input generics remain in the declaration. Earlier versions could +emit `Promise` without validating the decoded value as `T`; callers must +narrow the result or use a supported concrete overload. + `generate()` returns: ```ts diff --git a/runtime/tywrap_bridge_core.py b/runtime/tywrap_bridge_core.py index 90b5b2c9..4a6f62ac 100644 --- a/runtime/tywrap_bridge_core.py +++ b/runtime/tywrap_bridge_core.py @@ -565,13 +565,17 @@ def serialize_dataframe_json(obj): } -def serialize_series(obj, *, force_json_markers): +def serialize_series(obj, *, force_json_markers, path='result'): """ Encode a pandas Series as a single-column Arrow Table stream (the JS decoder contract is "table-like"); JSON when force_json_markers is set. """ if force_json_markers: - return serialize_series_json(obj) + return serialize_series_json(obj, path=path) + name = getattr(obj, 'name', None) + _check_terminal_json_integers( + encode_value(name, allow_nan=False), _serialize_path(path, 'name') + ) try: import pyarrow as pa # type: ignore except Exception as exc: @@ -591,13 +595,13 @@ def serialize_series(obj, *, force_json_markers): 'codecVersion': CODEC_VERSION, 'encoding': 'arrow', 'b64': b64, - 'name': getattr(obj, 'name', None), + 'name': name, } except Exception as exc: raise RuntimeError('Arrow encoding failed for pandas.Series') from exc -def serialize_series_json(obj): +def serialize_series_json(obj, *, path='result'): """JSON fallback for Series values that JavaScript can represent safely.""" import pandas as pd # type: ignore @@ -615,12 +619,16 @@ def serialize_series_json(obj): _normalize_pandas_json_scalar(value, f'Series value at position {position}', pd) for position, value in enumerate(data) ] + name = getattr(obj, 'name', None) + _check_terminal_json_integers( + encode_value(name, allow_nan=False), _serialize_path(path, 'name') + ) return { '__tywrap__': 'series', 'codecVersion': CODEC_VERSION, 'encoding': 'json', 'data': data, - 'name': getattr(obj, 'name', None), + 'name': name, } @@ -688,7 +696,7 @@ def _normalize_pandas_json_scalar(value, location, pd): ) -def serialize_sparse_matrix(obj): +def serialize_sparse_matrix(obj, *, path='result'): """ Serialize scipy sparse matrices into structured JSON envelopes (json-only; there is no Arrow path). Preserves sparsity; rejects unsupported formats and @@ -715,6 +723,32 @@ def serialize_sparse_matrix(obj): 'Complex scipy sparse matrices are not supported by the JSON codec; ' 'split into real/imag components explicitly before returning' ) + if getattr(obj.dtype, 'kind', None) in ('i', 'u') and obj.data.size and ( + (obj.data < -JS_SAFE_INTEGER_MAX).any() + or (obj.data > JS_SAFE_INTEGER_MAX).any() + ): + raise RuntimeError( + 'JSON scipy sparse encoding cannot safely represent data outside the ' + 'JavaScript safe integer range; cast or encode the values explicitly' + ) + + for index, dimension in enumerate(obj.shape): + if int.__int__(dimension) > JS_SAFE_INTEGER_MAX: + raise RuntimeError( + f'Unsafe Python integer at {_serialize_path(_serialize_path(path, "shape"), index)}: ' + f'{dimension} is outside the JavaScript safe integer range' + ) + index_fields = ('indices', 'indptr') if fmt in ('csr', 'csc') else ('row', 'col') + for field in index_fields: + values = getattr(obj, field) + if values.size and ( + (values < -JS_SAFE_INTEGER_MAX).any() + or (values > JS_SAFE_INTEGER_MAX).any() + ): + raise RuntimeError( + f'JSON scipy sparse encoding cannot safely represent {field} at ' + f'{_serialize_path(path, field)} outside the JavaScript safe integer range' + ) if fmt in ('csr', 'csc'): data = obj.data.tolist() @@ -845,7 +879,7 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False): return envelope -def serialize_sklearn_estimator(obj): +def serialize_sklearn_estimator(obj, *, path): """Serialize sklearn estimators as metadata only (json-only); no pickling.""" try: import sklearn # noqa: F401 @@ -861,7 +895,11 @@ def serialize_sklearn_estimator(obj): # params here for parity with the response codec. for key, value in params.items(): try: - json.dumps(value, allow_nan=False) + encoded = json.dumps(value, allow_nan=False) + param_path = _serialize_path(_serialize_path(path, 'params'), str(key)) + _check_terminal_json_integers(encoded, param_path) + except RuntimeError: + raise except (TypeError, ValueError) as exc: raise RuntimeError( f'scikit-learn estimator param {key!r} is not JSON-serializable ' @@ -881,6 +919,25 @@ def serialize_sklearn_estimator(obj): } +def _check_terminal_json_integers(encoded, path): + """Check integer tokens in a small JSON metadata value before emission.""" + stack = [(json.loads(encoded), path)] + while stack: + current, current_path = stack.pop() + if type(current) is int: + _serialize_leaf(current, current_path) + elif isinstance(current, dict): + stack.extend( + (item, _serialize_path(current_path, key)) + for key, item in current.items() + ) + elif isinstance(current, list): + stack.extend( + (item, _serialize_path(current_path, index)) + for index, item in enumerate(current) + ) + + _NO_PYDANTIC = object() @@ -954,11 +1011,11 @@ def _serialize_scientific(obj, *, force_json_markers, torch_allow_copy, depth, p return serialize_dataframe(obj, force_json_markers=force_json_markers) if is_pandas_series(obj): _check_serialize_depth(depth, path) - return serialize_series(obj, force_json_markers=force_json_markers) + return serialize_series(obj, force_json_markers=force_json_markers, path=path) elif package == 'scipy' and 'scipy.sparse' in sys.modules: if is_scipy_sparse(obj): _check_serialize_depth(depth, path) - return serialize_sparse_matrix(obj) + return serialize_sparse_matrix(obj, path=path) elif package == 'torch' and 'torch' in sys.modules: if is_torch_tensor(obj): _check_serialize_depth(depth, path) @@ -972,7 +1029,7 @@ def _serialize_scientific(obj, *, force_json_markers, torch_allow_copy, depth, p # estimators live outside the 'sklearn' package and must still get the # estimator serializer (and its param-naming errors). _check_serialize_depth(depth, path) - return serialize_sklearn_estimator(obj) + return serialize_sklearn_estimator(obj, path=path) return _NO_SCIENTIFIC @@ -995,21 +1052,42 @@ def _needs_serialize_visit(value): """Return whether value needs container or scientific traversal work.""" if type(value) in (type(None), bool, int, float, str): return False - if type(value) in (dict, list, tuple): + if isinstance(value, (dict, list, tuple, set, frozenset)): return True package = type(value).__module__.split('.', 1)[0] if package in ('numpy', 'pandas', 'scipy', 'torch'): return True - return 'sklearn.base' in sys.modules and is_sklearn_estimator(value) + return callable(getattr(value, 'model_dump', None)) or ( + 'sklearn.base' in sys.modules and is_sklearn_estimator(value) + ) -def _serialize_leaf(value): +def _serialize_leaf(value, path): """Apply non-container conversions without allocating a traversal frame.""" + integer_value = int.__int__(value) if isinstance(value, int) and not isinstance(value, bool) else None + if integer_value is not None and ( + integer_value < -JS_SAFE_INTEGER_MAX or integer_value > JS_SAFE_INTEGER_MAX + ): + raise RuntimeError( + f'Unsafe Python integer at {path}: {integer_value} is outside the JavaScript safe ' + 'integer range; return an explicit string or use an Arrow integer column' + ) if type(value) in (type(None), bool, int, float, str): return value + np = sys.modules.get('numpy') + if np is not None and isinstance(value, np.generic): + extracted = value.item() + if isinstance(extracted, np.generic): + raise TypeError( + f'NumPy scalar at {path} did not convert to a JSON value; ' + 'cast it explicitly before returning' + ) + return _serialize_leaf(extracted, path) pydantic_value = serialize_pydantic(value) if pydantic_value is not _NO_PYDANTIC: return pydantic_value + if isinstance(value, (set, frozenset)): + return list(value) stdlib_value = serialize_stdlib(value) if stdlib_value is not None: return stdlib_value @@ -1037,6 +1115,9 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): while stack: frame = stack.pop() action = frame[0] + if action == 'release-model': + active_ids.remove(frame[1]) + continue if action == 'dict': _, current, depth, path, parent, key, output, iterator = frame try: @@ -1057,22 +1138,23 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): if _needs_serialize_visit(item): stack.append(('visit', item, depth + 1, child_path, output, item_key)) else: - output[item_key] = _serialize_leaf(item) + output[item_key] = _serialize_leaf(item, child_path) continue if action == 'sequence': _, current, depth, path, parent, key, output, index = frame if index == len(output): active_ids.remove(id(current)) - parent[key] = output if type(current) is list else tuple(output) + parent[key] = output if isinstance(current, list) else tuple(output) continue stack.append(('sequence', current, depth, path, parent, key, output, index + 1)) item = current[index] + child_path = _serialize_path(path, index) if _needs_serialize_visit(item): stack.append( - ('visit', item, depth + 1, _serialize_path(path, index), output, index) + ('visit', item, depth + 1, child_path, output, index) ) else: - output[index] = _serialize_leaf(item) + output[index] = _serialize_leaf(item, child_path) continue _, current, depth, path, parent, key = frame @@ -1085,7 +1167,7 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): path=path, ) except Exception as exc: - if path == 'result': + if path == 'result' and 'safe integer range' not in str(exc): raise raise RuntimeError(f'Scientific value serialization failed at {path}: {exc}') from exc if scientific is not _NO_SCIENTIFIC: @@ -1106,8 +1188,25 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): parent[key] = scientific continue - container_type = type(current) - if container_type in (dict, list, tuple): + if callable(getattr(current, 'model_dump', None)): + _check_serialize_depth(depth, path) + visited_nodes += 1 + _check_serialize_nodes(visited_nodes, path) + current_id = id(current) + if current_id in active_ids: + raise RuntimeError(f'Circular model conversion detected at {path}') + active_ids.add(current_id) + normalized = _serialize_leaf(current, path) + if normalized is current: + raise RuntimeError(f'Circular model conversion detected at {path}') + stack.append(('release-model', current_id)) + if _needs_serialize_visit(normalized): + stack.append(('visit', normalized, depth + 1, path, parent, key)) + else: + parent[key] = _serialize_leaf(normalized, path) + continue + + if isinstance(current, (dict, list, tuple)): _check_serialize_depth(depth, path) visited_nodes += 1 _check_serialize_nodes(visited_nodes, path) @@ -1116,7 +1215,7 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): raise RuntimeError(f'Circular reference detected at {path}') active_ids.add(current_id) - if container_type is dict: + if isinstance(current, dict): output = {} parent[key] = output stack.append( @@ -1125,12 +1224,18 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False): continue output = [None] * len(current) - if container_type is list: + if isinstance(current, list): parent[key] = output stack.append(('sequence', current, depth, path, parent, key, output, 0)) continue - parent[key] = _serialize_leaf(current) + normalized = _serialize_leaf(current, path) + if normalized is not current and _needs_serialize_visit(normalized): + stack.append(('visit', normalized, depth, path, parent, key)) + elif normalized is not current: + parent[key] = _serialize_leaf(normalized, path) + else: + parent[key] = normalized return root[0] diff --git a/scripts/generate-llms-full.mjs b/scripts/generate-llms-full.mjs index 23859f82..0da1aad8 100644 --- a/scripts/generate-llms-full.mjs +++ b/scripts/generate-llms-full.mjs @@ -30,6 +30,8 @@ const orderedDocs = [ const excludedDocs = new Set([ 'docs/perf-baselines.md', 'docs/release.md', + 'docs/maintainers/callable-contracts.md', + 'docs/maintainers/value-contracts.md', ]); async function collectDocs(dir) { diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts new file mode 100644 index 00000000..d8bc033b --- /dev/null +++ b/src/contracts/value-contract.ts @@ -0,0 +1,80 @@ +/** Internal value policy consumed by codecs and callable-contract generation. */ +export const VALUE_CONTRACT_REVISION = 2 as const; + +/** JSON numbers can carry Python integers exactly only within this range. */ +export const MAX_SAFE_JSON_INTEGER = 2 ** 53 - 1; + +export type ValueContract = + | { readonly kind: 'null'; readonly wire: 'json'; readonly decodedAs: 'null' } + | { readonly kind: 'boolean'; readonly wire: 'json'; readonly decodedAs: 'boolean' } + | { + readonly kind: 'integer'; + readonly wire: 'json'; + readonly decodedAs: 'number'; + readonly constraint: 'safe-integer'; + } + | { + readonly kind: 'float'; + readonly wire: 'json'; + readonly decodedAs: 'number'; + readonly constraint: 'finite'; + } + | { readonly kind: 'string'; readonly wire: 'json'; readonly decodedAs: 'string' } + | { readonly kind: 'bytes'; readonly wire: 'base64-envelope'; readonly decodedAs: 'Uint8Array' } + | { + readonly kind: 'sequence'; + readonly wire: 'json'; + readonly decodedAs: 'array'; + readonly item: ValueContract; + } + | { + readonly kind: 'tuple'; + readonly wire: 'json'; + readonly decodedAs: 'array'; + readonly items: readonly ValueContract[]; + } + | { + readonly kind: 'union'; + readonly wire: 'selected-option'; + readonly decodedAs: 'selected-option'; + readonly options: readonly [ValueContract, ValueContract, ...ValueContract[]]; + } + | { + readonly kind: 'record'; + readonly wire: 'json'; + readonly decodedAs: 'object'; + readonly fields: readonly ValueContractField[]; + readonly additionalValues?: ValueContract; + } + | { + readonly kind: 'ndarray-float16'; + readonly wire: 'arrow' | 'json-fallback'; + readonly decodedAs: 'nested-array-or-scalar'; + readonly element: Extract; + readonly dtype: 'float16'; + readonly rank?: number; + } + | { + readonly kind: 'torch-float16'; + readonly wire: 'ndarray-envelope'; + readonly decodedAs: 'tensor-record'; + readonly dtype: 'torch.float16'; + readonly value: Extract; + } + | { + readonly kind: 'unsupported'; + readonly wire: 'none'; + readonly decodedAs: 'never'; + readonly reason: string; + readonly guidance: string; + }; + +export interface ValueContractField { + readonly name: string; + readonly value: ValueContract; + readonly required: boolean; +} + +export function isSafeJsonInteger(value: number): boolean { + return Number.isInteger(value) && Math.abs(value) <= MAX_SAFE_JSON_INTEGER; +} diff --git a/src/core/annotation-parser.ts b/src/core/annotation-parser.ts index 5dfc2864..9bf6543a 100644 --- a/src/core/annotation-parser.ts +++ b/src/core/annotation-parser.ts @@ -16,10 +16,14 @@ export function parseAnnotationToPythonType( (options.typeParameters ?? []).map(param => [param.name, param] as const) ); const modulePrefixes = ['', 'typing.', 'typing_extensions.', 'collections.abc.'] as const; + let numpyShapeDepth = 0; const unknownType = (): PythonType => ({ kind: 'custom', name: 'Any', module: 'typing' }); const recordUnknown = (name: string): void => { + if (numpyShapeDepth > 0) { + return; + } try { onUnknownTypeName?.(name); } catch { @@ -495,8 +499,22 @@ export function parseAnnotationToPythonType( const generic = splitGenericInvocation(raw); if (generic) { - const typeArgs = splitTopLevel(generic.inner, ',').map(part => parse(part.trim(), depth + 1)); const qualified = splitQualifiedName(generic.name); + const parts = splitTopLevel(generic.inner, ','); + const numpyArrayShape = + qualified.module === 'numpy' && qualified.name === 'ndarray' && parts.length === 2; + const typeArgs = parts.map((part, index) => { + if (!numpyArrayShape || index !== 0) { + return parse(part.trim(), depth + 1); + } + // NumPy's first ndarray argument describes shape, not decoded value type. + numpyShapeDepth += 1; + try { + return parse(part.trim(), depth + 1); + } finally { + numpyShapeDepth -= 1; + } + }); return { kind: 'generic', name: qualified.name, diff --git a/src/core/callable-compiler.ts b/src/core/callable-compiler.ts new file mode 100644 index 00000000..9a0d6726 --- /dev/null +++ b/src/core/callable-compiler.ts @@ -0,0 +1,1126 @@ +/** + * Pure callable compilation. + * + * `generate()` owns discovery, caching, and writes. This module owns the + * deterministic step between validated IR and generated wrapper files. + */ + +import { + VALUE_CONTRACT_REVISION, + type ValueContract, + type ValueContractField, +} from '../contracts/value-contract.js'; +import type { + GeneratedCode, + Parameter, + PythonFunction, + PythonModule, + PythonType, +} from '../types/index.js'; +import type { CodeGenerator } from './generator.js'; +import type { ValidatedIrContract } from './ir-contract.js'; +import { transformIrToTsModel } from './ir-model.js'; + +export type CallableDirection = 'input' | 'output'; + +export type CallableCapability = + | 'value-rpc' + | 'return-validation' + | 'scientific-ndarray' + | 'scientific-torch' + | 'coroutine-execution' + | 'dataclass-adapter'; + +export interface CapabilityDescription { + name: CallableCapability; + available: boolean; + guidance: string; +} + +export interface ValueConversionRequest { + direction: CallableDirection; + logicalType: PythonType; + path: string; + /** Internal recursion counter. Revision 2 rejects contracts deeper than 64 nodes. */ + depth?: number; + /** Compile-local resolver for nested named values. */ + resolveNested?: (request: ValueConversionRequest) => ValueResolution; + /** Exact local names already being expanded, used to stop recursive aliases. */ + activeNames?: readonly string[]; +} + +export interface SupportedValueResolution { + status: 'supported'; + value: ValueContract; +} + +export interface UnsupportedValueResolution { + status: 'unsupported'; + reason: string; + guidance: string; +} + +export interface UnresolvedValueResolution { + status: 'unresolved'; + annotation: string; +} + +export type ValueResolution = + | SupportedValueResolution + | UnsupportedValueResolution + | UnresolvedValueResolution; + +/** + * Maps an internal Python type to the frozen #343 value policy. + * + * The compiler does not infer wire behavior from a TypeScript spelling. + */ +export interface ValueConversionDescription { + revision: typeof VALUE_CONTRACT_REVISION; + resolve(request: ValueConversionRequest): ValueResolution; +} + +export interface CallableCompilationDiagnostic { + severity: 'warning' | 'error'; + code: + | 'conversion-unresolved' + | 'conversion-unsupported' + | 'capability-unavailable' + | 'coroutine-unsupported' + | 'dataclass-unsupported' + | 'overload-ambiguous'; + path: string; + message: string; +} + +export interface ResolvedCallableValue { + direction: CallableDirection; + path: string; + logicalType: PythonType; + resolution: ValueResolution; +} + +export interface ResolvedCallable { + name: string; + path: string; + parameters: readonly ResolvedCallableValue[]; + result: ResolvedCallableValue; + overloadParameters: readonly (readonly ResolvedCallableValue[])[]; + overloadResults: readonly ResolvedCallableValue[]; + requiredCapabilities: readonly CallableCapability[]; +} + +export interface CompiledFile { + kind: 'typescript' | 'declaration'; + content: string; +} + +export interface CompiledContract { + /** Validated IR identity. It is retained for diagnostics and reproducibility. */ + ir: Pick; + /** Resolved model consumed by declarations, wrappers, and return validators. */ + module: PythonModule; + callables: readonly ResolvedCallable[]; + diagnostics: readonly CallableCompilationDiagnostic[]; + files: readonly CompiledFile[]; + generated: GeneratedCode; +} + +export interface CompileContractOptions { + /** Select exported names only. The compiler derives all types from validated IR. */ + module: PythonModule; + /** The generator is synchronous and performs no filesystem or runtime work. */ + generator: Pick; + conversion: ValueConversionDescription; + capabilities: readonly CapabilityDescription[]; + annotatedJSDoc?: boolean; +} + +const UNKNOWN_TYPE: PythonType = { kind: 'custom', name: 'Any', module: 'typing' }; + +export const DEFAULT_CALLABLE_CAPABILITIES: readonly CapabilityDescription[] = [ + { + name: 'value-rpc', + available: true, + guidance: 'Use the existing tywrap/1 value RPC transport.', + }, + { + name: 'return-validation', + available: true, + guidance: 'Generate a return validator for supported decoded values.', + }, + { + name: 'scientific-ndarray', + available: true, + guidance: 'Keep the existing ndarray Arrow envelope and JSON fallback.', + }, + { + name: 'scientific-torch', + available: true, + guidance: 'Reuse the ndarray conversion inside the existing Torch envelope.', + }, + { + name: 'coroutine-execution', + available: true, + guidance: 'Use a runtime bridge that awaits Python coroutine results.', + }, + { + name: 'dataclass-adapter', + available: false, + guidance: + 'Use a TypedDict or an explicit record adapter until #339 defines dataclass conversion.', + }, +]; + +function leafName(type: PythonType): string | null { + if (type.kind !== 'custom' && type.kind !== 'generic') { + return null; + } + return type.name.split('.').at(-1) ?? null; +} + +function annotationName(type: PythonType): string { + switch (type.kind) { + case 'primitive': + return type.name; + case 'custom': + case 'generic': + return type.module ? `${type.module}.${type.name}` : type.name; + default: + return type.kind; + } +} + +function firstUnresolvedChild( + resolutions: readonly ValueResolution[] +): UnsupportedValueResolution | UnresolvedValueResolution | null { + return resolutions.find(resolution => resolution.status !== 'supported') ?? null; +} + +function resolveSequence( + item: PythonType, + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + const child = conversion.resolve({ + ...request, + logicalType: item, + path: `${request.path}[]`, + depth: (request.depth ?? 0) + 1, + }); + if (child.status !== 'supported') { + return child; + } + return { + status: 'supported', + value: { kind: 'sequence', wire: 'json', decodedAs: 'array', item: child.value }, + }; +} + +function resolveRecord( + valueType: PythonType, + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + const child = conversion.resolve({ + ...request, + logicalType: valueType, + path: `${request.path}{value}`, + depth: (request.depth ?? 0) + 1, + }); + if (child.status !== 'supported') { + return child; + } + return { + status: 'supported', + value: { + kind: 'record', + wire: 'json', + decodedAs: 'object', + fields: [], + additionalValues: child.value, + }, + }; +} + +function resolveTuple( + items: readonly PythonType[], + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + if (items.length === 0) { + return { + status: 'supported', + value: { + kind: 'tuple', + wire: 'json', + decodedAs: 'array', + items: [], + }, + }; + } + if (items.length === 2 && items[1]?.kind === 'custom' && items[1].name === '...') { + return resolveSequence(items[0]!, request, conversion); + } + const entries = items.map((item, index) => + conversion.resolve({ + ...request, + logicalType: item, + path: `${request.path}[${index}]`, + depth: (request.depth ?? 0) + 1, + }) + ); + const childFailure = firstUnresolvedChild(entries); + if (childFailure) { + return childFailure; + } + const values = entries as SupportedValueResolution[]; + return { + status: 'supported', + value: { + kind: 'tuple', + wire: 'json', + decodedAs: 'array', + items: values.map(entry => entry.value), + }, + }; +} + +function resolveUnion( + options: readonly PythonType[], + request: ValueConversionRequest, + conversion: ValueConversionDescription +): ValueResolution { + if (options.length < 2 || options.length > 32) { + return { + status: 'unsupported', + reason: 'Revision 2 accepts unions with two to 32 alternatives.', + guidance: 'Split a larger union or provide an explicit tagged record adapter.', + }; + } + const entries = options.map((option, index) => + conversion.resolve({ + ...request, + logicalType: option, + path: `${request.path}.options[${index}]`, + depth: (request.depth ?? 0) + 1, + }) + ); + const childFailure = firstUnresolvedChild(entries); + if (childFailure) { + return childFailure; + } + const values = entries as SupportedValueResolution[]; + const first = values[0]?.value; + const second = values[1]?.value; + if (!first || !second) { + return { + status: 'unsupported', + reason: 'Revision 2 requires at least two supported union alternatives.', + guidance: 'Provide two explicit value alternatives.', + }; + } + return { + status: 'supported', + value: { + kind: 'union', + wire: 'selected-option', + decodedAs: 'selected-option', + options: [first, second, ...values.slice(2).map(entry => entry.value)], + }, + }; +} + +function resolveNdarray(type: PythonType): ValueResolution { + const typeArgument = + type.kind === 'generic' && + type.typeArgs.length === 2 && + type.typeArgs[1]?.kind === 'generic' && + type.typeArgs[1].name === 'dtype' && + type.typeArgs[1].module === 'numpy' && + (type.typeArgs[0]?.kind === 'collection' || + (type.typeArgs[0]?.kind === 'custom' && type.typeArgs[0].name === 'Any')) + ? type.typeArgs[1].typeArgs[0] + : type.kind === 'generic' && type.typeArgs.length === 1 + ? type.typeArgs[0] + : undefined; + if ( + typeArgument?.kind !== 'custom' || + typeArgument.name !== 'float16' || + typeArgument.module !== 'numpy' + ) { + return { + status: 'unresolved', + annotation: annotationName(type), + }; + } + return { + status: 'supported', + value: { + kind: 'ndarray-float16', + wire: 'arrow', + decodedAs: 'nested-array-or-scalar', + element: { kind: 'float', wire: 'json', decodedAs: 'number', constraint: 'finite' }, + dtype: 'float16', + }, + }; +} + +function resolveTorchFloat16(): ValueResolution { + const ndarray = resolveNdarray({ + kind: 'generic', + name: 'ndarray', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }); + if (ndarray.status !== 'supported' || ndarray.value.kind !== 'ndarray-float16') { + return ndarray; + } + return { + status: 'supported', + value: { + kind: 'torch-float16', + wire: 'ndarray-envelope', + decodedAs: 'tensor-record', + dtype: 'torch.float16', + value: ndarray.value, + }, + }; +} + +/** The conversion set that #335 may use with the frozen value-contract revision. */ +export const DEFAULT_VALUE_CONVERSION: ValueConversionDescription = { + revision: VALUE_CONTRACT_REVISION, + resolve(request): ValueResolution { + const { logicalType: type } = request; + const nestedConversion: ValueConversionDescription = request.resolveNested + ? { revision: VALUE_CONTRACT_REVISION, resolve: request.resolveNested } + : DEFAULT_VALUE_CONVERSION; + if ((request.depth ?? 0) > 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } + switch (type.kind) { + case 'primitive': + if (type.name === 'None') { + return { status: 'supported', value: { kind: 'null', wire: 'json', decodedAs: 'null' } }; + } + if (type.name === 'bool') { + return { + status: 'supported', + value: { kind: 'boolean', wire: 'json', decodedAs: 'boolean' }, + }; + } + if (type.name === 'int') { + return { + status: 'supported', + value: { + kind: 'integer', + wire: 'json', + decodedAs: 'number', + constraint: 'safe-integer', + }, + }; + } + if (type.name === 'float') { + return { + status: 'supported', + value: { kind: 'float', wire: 'json', decodedAs: 'number', constraint: 'finite' }, + }; + } + if (type.name === 'str') { + return { + status: 'supported', + value: { kind: 'string', wire: 'json', decodedAs: 'string' }, + }; + } + if (type.name === 'bytes') { + return { + status: 'supported', + value: { kind: 'bytes', wire: 'base64-envelope', decodedAs: 'Uint8Array' }, + }; + } + return { + status: 'unsupported', + reason: 'The value contract does not specify this primitive conversion.', + guidance: 'Use a supported primitive or an explicit value adapter.', + }; + case 'annotated': + return nestedConversion.resolve({ ...request, logicalType: type.base }); + case 'final': + case 'classvar': + return nestedConversion.resolve({ ...request, logicalType: type.type }); + case 'optional': + return resolveUnion( + [type.type, { kind: 'primitive', name: 'None' }], + request, + nestedConversion + ); + case 'union': + return resolveUnion(type.types, request, nestedConversion); + case 'collection': + if (type.name === 'dict') { + const key = type.itemTypes[0]; + if (key?.kind !== 'primitive' || key.name !== 'str') { + return { + status: 'unsupported', + reason: 'The frozen record contract requires string keys.', + guidance: 'Convert keys to strings before crossing the value RPC boundary.', + }; + } + return resolveRecord(type.itemTypes[1] ?? UNKNOWN_TYPE, request, nestedConversion); + } + if (type.name === 'tuple') { + return resolveTuple(type.itemTypes, request, nestedConversion); + } + return resolveSequence(type.itemTypes[0] ?? UNKNOWN_TYPE, request, nestedConversion); + case 'generic': { + const leaf = leafName(type); + if (leaf === 'NDArray' || leaf === 'ndarray') { + return resolveNdarray(type); + } + if (leaf === 'Tensor' && type.module?.startsWith('torch')) { + const tensorDtype = type.typeArgs[0]; + if ( + type.typeArgs.length !== 1 || + tensorDtype?.kind !== 'custom' || + tensorDtype.name !== 'float16' || + tensorDtype.module !== 'torch' + ) { + return { status: 'unresolved', annotation: annotationName(type) }; + } + return resolveTorchFloat16(); + } + if (['list', 'List', 'Sequence', 'Iterable', 'set', 'frozenset'].includes(leaf ?? '')) { + return resolveSequence(type.typeArgs[0] ?? UNKNOWN_TYPE, request, nestedConversion); + } + if (['dict', 'Dict', 'Mapping', 'MutableMapping'].includes(leaf ?? '')) { + const key = type.typeArgs[0]; + if (key?.kind !== 'primitive' || key.name !== 'str') { + return { + status: 'unsupported', + reason: 'The frozen record contract requires string keys.', + guidance: 'Convert keys to strings before crossing the value RPC boundary.', + }; + } + return resolveRecord(type.typeArgs[1] ?? UNKNOWN_TYPE, request, nestedConversion); + } + return { status: 'unresolved', annotation: annotationName(type) }; + } + case 'custom': { + const leaf = leafName(type); + if (type.name === 'HalfTensor' && type.module === 'torch') { + return resolveTorchFloat16(); + } + if (leaf === 'ndarray' || leaf === 'NDArray') { + return resolveNdarray(type); + } + return { status: 'unresolved', annotation: annotationName(type) }; + } + default: + return { status: 'unresolved', annotation: annotationName(type) }; + } + }, +}; + +function capabilityNamesFor(value: ValueContract): CallableCapability[] { + const names = new Set(['value-rpc', 'return-validation']); + const collect = (current: ValueContract): void => { + switch (current.kind) { + case 'sequence': + collect(current.item); + break; + case 'tuple': + current.items.forEach(collect); + break; + case 'union': + current.options.forEach(collect); + break; + case 'record': + current.fields.forEach(field => collect(field.value)); + if (current.additionalValues) { + collect(current.additionalValues); + } + break; + case 'ndarray-float16': + names.add('scientific-ndarray'); + break; + case 'torch-float16': + names.add('scientific-torch'); + names.add('scientific-ndarray'); + break; + } + }; + collect(value); + return [...names]; +} + +function containsNamedType(type: PythonType, names: ReadonlySet): boolean { + if ((type.kind === 'custom' || type.kind === 'generic') && names.has(leafName(type) ?? '')) { + return true; + } + switch (type.kind) { + case 'collection': + return type.itemTypes.some(item => containsNamedType(item, names)); + case 'union': + return type.types.some(item => containsNamedType(item, names)); + case 'optional': + case 'final': + case 'classvar': + case 'unpack': + return containsNamedType(type.type, names); + case 'annotated': + return containsNamedType(type.base, names); + case 'generic': + return type.typeArgs.some(item => containsNamedType(item, names)); + case 'callable': + return ( + type.parameters.some(item => containsNamedType(item, names)) || + containsNamedType(type.returnType, names) + ); + default: + return false; + } +} + +function diagnosticForResolution( + resolution: ValueResolution, + path: string +): CallableCompilationDiagnostic | null { + if (resolution.status === 'supported') { + return null; + } + if (resolution.status === 'unresolved') { + return { + severity: 'warning', + code: 'conversion-unresolved', + path, + message: `${path}: no value conversion is defined for ${resolution.annotation}; the generated result type is unknown.`, + }; + } + return { + severity: 'error', + code: 'conversion-unsupported', + path, + message: `${path}: ${resolution.reason} ${resolution.guidance}`, + }; +} + +function outputType(value: ResolvedCallableValue): PythonType { + return value.resolution.status === 'supported' ? value.logicalType : UNKNOWN_TYPE; +} + +function valuesMayOverlap(left: ValueContract, right: ValueContract): boolean { + if (left.kind === 'unsupported' || right.kind === 'unsupported') { + return true; + } + if (left.kind === 'union') { + return left.options.some(option => valuesMayOverlap(option, right)); + } + if (right.kind === 'union') { + return right.options.some(option => valuesMayOverlap(left, option)); + } + if ( + (left.kind === 'integer' && right.kind === 'float') || + (left.kind === 'float' && right.kind === 'integer') + ) { + return true; + } + if ( + left.kind === 'ndarray-float16' || + right.kind === 'ndarray-float16' || + left.kind === 'torch-float16' || + right.kind === 'torch-float16' + ) { + return true; + } + if (left.kind === 'tuple' && right.kind === 'sequence') { + return left.items.every(item => valuesMayOverlap(item, right.item)); + } + if (left.kind === 'sequence' && right.kind === 'tuple') { + return right.items.every(item => valuesMayOverlap(left.item, item)); + } + if (left.kind !== right.kind) { + return false; + } + if (left.kind === 'tuple' && right.kind === 'tuple') { + return ( + left.items.length === right.items.length && + left.items.every((item, index) => valuesMayOverlap(item, right.items[index]!)) + ); + } + if (left.kind === 'sequence' && right.kind === 'sequence') { + return true; // An empty array matches both element schemas. + } + return true; +} + +function positionalSlot(parameters: readonly Parameter[], index: number): number | null { + const parameter = parameters[index]!; + if (parameter.keywordOnly || parameter.kwArgs || parameter.varArgs) { + return null; + } + let slot = 0; + for (const earlier of parameters.slice(0, index)) { + if (earlier.varArgs) { + return null; + } + if (!earlier.keywordOnly && !earlier.kwArgs) { + slot += 1; + } + } + return slot; +} + +function sharesRequiredBinding( + left: readonly Parameter[], + leftIndex: number, + right: readonly Parameter[], + rightIndex: number +): boolean { + const a = left[leftIndex]!; + const b = right[rightIndex]!; + if ( + a.optional || + b.optional || + a.varArgs || + b.varArgs || + a.kwArgs || + b.kwArgs || + a.name !== b.name + ) { + return false; + } + if (a.keywordOnly && b.keywordOnly) { + return true; + } + if (a.keywordOnly || b.keywordOnly) { + return !a.positionalOnly && !b.positionalOnly; + } + const leftSlot = positionalSlot(left, leftIndex); + return leftSlot !== null && leftSlot === positionalSlot(right, rightIndex); +} + +function overloadsMayOverlap( + left: readonly ResolvedCallableValue[], + right: readonly ResolvedCallableValue[], + leftParameters: readonly Parameter[], + rightParameters: readonly Parameter[] +): boolean { + const countRange = (parameters: readonly Parameter[]): [number, number] => [ + parameters.filter(parameter => !parameter.optional && !parameter.varArgs && !parameter.kwArgs) + .length, + parameters.some(parameter => parameter.varArgs || parameter.kwArgs) + ? Number.POSITIVE_INFINITY + : parameters.length, + ]; + const [leftMin, leftMax] = countRange(leftParameters); + const [rightMin, rightMax] = countRange(rightParameters); + if (leftMax < rightMin || rightMax < leftMin) { + return false; + } + for (let leftIndex = 0; leftIndex < leftParameters.length; leftIndex += 1) { + if ( + leftParameters.filter(parameter => parameter.name === leftParameters[leftIndex]!.name) + .length !== 1 + ) { + continue; + } + const rightIndex = rightParameters.findIndex( + parameter => parameter.name === leftParameters[leftIndex]!.name + ); + if ( + rightIndex < 0 || + rightParameters.filter(parameter => parameter.name === leftParameters[leftIndex]!.name) + .length !== 1 || + !sharesRequiredBinding(leftParameters, leftIndex, rightParameters, rightIndex) + ) { + continue; + } + const a = left[leftIndex]?.resolution; + const b = right[rightIndex]?.resolution; + if ( + a?.status === 'supported' && + b?.status === 'supported' && + !valuesMayOverlap(a.value, b.value) + ) { + return false; + } + } + return true; +} + +function resolveCallable( + func: PythonFunction, + path: string, + conversion: ValueConversionDescription, + capabilities: ReadonlyMap, + dataclassNames: ReadonlySet, + diagnostics: CallableCompilationDiagnostic[] +): { function: PythonFunction; callable: ResolvedCallable } { + const resolveValue = ( + direction: CallableDirection, + logicalType: PythonType, + valuePath: string + ): ResolvedCallableValue => ({ + direction, + path: valuePath, + logicalType, + resolution: conversion.resolve({ direction, logicalType, path: valuePath }), + }); + const parameters = func.parameters.map((parameter, index) => + resolveValue('input', parameter.type, `${path}.parameters[${index}]`) + ); + const result = resolveValue('output', func.returnType, `${path}.returns`); + const overloadParameters = (func.overloads ?? []).map((overload, overloadIndex) => + overload.parameters.map((parameter, parameterIndex) => + resolveValue( + 'input', + parameter.type, + `${path}.overloads[${overloadIndex}].parameters[${parameterIndex}]` + ) + ) + ); + const overloadResults = (func.overloads ?? []).map((overload, index) => + resolveValue('output', overload.returnType, `${path}.overloads[${index}].returns`) + ); + // Python binds these receivers before the wrapper sends RPC arguments. + const visibleParameters = parameters.filter( + (_, index) => func.parameters[index]?.name !== 'self' && func.parameters[index]?.name !== 'cls' + ); + const visibleOverloadParameters = overloadParameters.map((values, overloadIndex) => + values.filter( + (_, index) => + func.overloads?.[overloadIndex]?.parameters[index]?.name !== 'self' && + func.overloads?.[overloadIndex]?.parameters[index]?.name !== 'cls' + ) + ); + const visibleOverloadSignatures = (func.overloads ?? []).map(overload => + overload.parameters.filter(parameter => parameter.name !== 'self' && parameter.name !== 'cls') + ); + const values = [ + ...visibleParameters, + result, + ...visibleOverloadParameters.flat(), + ...overloadResults, + ]; + const requiredCapabilities = new Set(); + for (const value of values) { + const diagnostic = diagnosticForResolution(value.resolution, value.path); + if (diagnostic) { + diagnostics.push(diagnostic); + } + } + for (let index = 0; index < overloadParameters.length; index += 1) { + for (let earlier = 0; earlier < index; earlier += 1) { + if ( + overloadsMayOverlap( + visibleOverloadParameters[earlier]!, + visibleOverloadParameters[index]!, + visibleOverloadSignatures[earlier]!, + visibleOverloadSignatures[index]! + ) + ) { + diagnostics.push({ + severity: 'warning', + code: 'overload-ambiguous', + path: `${path}.overloads[${index}]`, + message: `${path}.overloads[${index}]: input values may also match overload ${earlier}. Runtime return validation selects the first declared match.`, + }); + } + } + } + for (const value of values) { + const directDataclass = + (value.logicalType.kind === 'custom' || value.logicalType.kind === 'generic') && + dataclassNames.has(leafName(value.logicalType) ?? ''); + const adaptedDataclass = + value.direction === 'output' && + directDataclass && + capabilities.get('dataclass-adapter')?.available === true && + value.resolution.status === 'supported' && + value.resolution.value.kind === 'record'; + if (adaptedDataclass) { + requiredCapabilities.add('dataclass-adapter'); + } else if (containsNamedType(value.logicalType, dataclassNames)) { + diagnostics.push({ + severity: 'error', + code: 'dataclass-unsupported', + path: value.path, + message: `${value.path}: a dataclass annotation has no value adapter. Use a TypedDict or an explicit record adapter until #339.`, + }); + value.resolution = { + status: 'unsupported', + reason: 'No dataclass value adapter is available.', + guidance: 'Use a TypedDict or an explicit record adapter.', + }; + } + if (value.direction === 'output' && func.isAsync) { + if (capabilities.get('coroutine-execution')?.available !== true) { + diagnostics.push({ + severity: 'error', + code: 'coroutine-unsupported', + path: value.path, + message: `${value.path}: this callable requires a runtime bridge that awaits Python coroutine results.`, + }); + value.resolution = { + status: 'unsupported', + reason: 'Coroutine execution is unavailable.', + guidance: 'Use a coroutine-capable runtime bridge or a synchronous adapter.', + }; + } else { + requiredCapabilities.add('coroutine-execution'); + } + } + if (value.resolution.status === 'supported') { + const needed = capabilityNamesFor(value.resolution.value); + needed.forEach(capability => requiredCapabilities.add(capability)); + const missing = needed.filter(capability => capabilities.get(capability)?.available !== true); + if (missing.length > 0) { + missing.forEach(capability => { + const description = capabilities.get(capability); + diagnostics.push({ + severity: 'error', + code: 'capability-unavailable', + path: value.path, + message: `${value.path}: ${capability} is required. ${description?.guidance ?? 'Add a declared capability before generating this callable.'}`, + }); + }); + value.resolution = { + status: 'unsupported', + reason: `Required capability is unavailable: ${missing.join(', ')}.`, + guidance: 'Provide the capability or use a supported value adapter.', + }; + } + } + } + + const resolvedParameters: Parameter[] = parameters.map((parameter, index) => ({ + ...func.parameters[index]!, + type: parameter.resolution.status === 'unsupported' ? UNKNOWN_TYPE : parameter.logicalType, + })); + const resolvedOverloads = (func.overloads ?? []).map((overload, index) => ({ + ...overload, + parameters: overload.parameters.map((parameter, parameterIndex) => ({ + ...parameter, + type: + overloadParameters[index]?.[parameterIndex]?.resolution.status === 'unsupported' + ? UNKNOWN_TYPE + : parameter.type, + })), + returnType: overloadResults[index] ? outputType(overloadResults[index]) : overload.returnType, + })); + const supportedValue = (value: ResolvedCallableValue): ValueContract | undefined => + value.resolution.status === 'supported' ? value.resolution.value : undefined; + const resolvedFunction: PythonFunction = { + ...func, + parameters: resolvedParameters, + signature: { + ...func.signature, + parameters: resolvedParameters, + returnType: outputType(result), + }, + returnType: outputType(result), + overloads: resolvedOverloads, + callableContract: { + parameterValues: parameters.map(supportedValue), + returnValue: supportedValue(result), + returnValidationType: result.resolution.status === 'unresolved' ? func.returnType : undefined, + overloads: overloadResults.map((overload, index) => ({ + parameterValues: overloadParameters[index]!.map(supportedValue), + returnValue: supportedValue(overload), + returnValidationType: + overload.resolution.status === 'unresolved' + ? func.overloads![index]!.returnType + : undefined, + })), + }, + }; + return { + function: resolvedFunction, + callable: { + name: func.name, + path, + parameters, + result, + overloadParameters, + overloadResults, + requiredCapabilities: [...requiredCapabilities].sort(), + }, + }; +} + +/** + * Compile already-validated IR without filesystem, subprocess, cache, or + * registry access. `generate()` selects exports and writes these files. + */ +export function compileContract( + ir: ValidatedIrContract, + options: CompileContractOptions +): CompiledContract { + if (ir.module !== options.module.name) { + throw new Error( + `Compiled module ${options.module.name} does not match IR module ${ir.module}.` + ); + } + const canonical = transformIrToTsModel(ir); + const select = ( + kind: string, + source: readonly T[], + requested: readonly { name: string }[] + ): T[] => { + const byName = new Map(source.map(item => [item.name, item] as const)); + const seen = new Set(); + return requested.map(item => { + const match = byName.get(item.name); + if (!match || seen.has(item.name)) { + throw new Error(`Selected ${kind} ${item.name} does not match validated IR.`); + } + seen.add(item.name); + return match; + }); + }; + const selectedFunctions = select('function', canonical.functions, options.module.functions); + const selectedClasses = select('class', canonical.classes, options.module.classes); + const selectedAliases = select( + 'type alias', + canonical.typeAliases ?? [], + options.module.typeAliases ?? canonical.typeAliases ?? [] + ); + const localTypedDicts = new Map( + selectedClasses + .filter(cls => cls.kind === 'typed_dict' && !cls.typeParameters?.length) + .map(cls => [cls.name, cls] as const) + ); + const localAliases = new Map( + selectedAliases + .filter(alias => !alias.typeParameters?.length) + .map(alias => [alias.name, alias] as const) + ); + const conversion: ValueConversionDescription = { + revision: options.conversion.revision, + resolve(request): ValueResolution { + const type = request.logicalType; + const direct = options.conversion.resolve({ ...request, resolveNested: conversion.resolve }); + if ( + direct.status !== 'unresolved' || + type.kind !== 'custom' || + (type.module !== undefined && type.module !== ir.module) + ) { + return direct; + } + const typedDict = localTypedDicts.get(type.name); + if (typedDict) { + const key = `typed-dict:${type.name}`; + if (request.activeNames?.includes(key)) { + return { status: 'unresolved', annotation: annotationName(type) }; + } + if ((request.depth ?? 0) >= 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } + const fields: ValueContractField[] = []; + for (const property of typedDict.properties) { + const field = conversion.resolve({ + ...request, + logicalType: property.type, + path: `${request.path}.${property.name}`, + depth: (request.depth ?? 0) + 1, + activeNames: [...(request.activeNames ?? []), key], + }); + if (field.status !== 'supported') { + return field; + } + fields.push({ name: property.name, value: field.value, required: !property.optional }); + } + return { + status: 'supported', + value: { kind: 'record', wire: 'json', decodedAs: 'object', fields }, + }; + } + const alias = localAliases.get(type.name); + if (alias) { + const key = `alias:${type.name}`; + if (request.activeNames?.includes(key)) { + return { status: 'unresolved', annotation: annotationName(type) }; + } + if ((request.depth ?? 0) >= 64) { + return { + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + guidance: 'Flatten the value or provide a bounded adapter.', + }; + } + return conversion.resolve({ + ...request, + logicalType: alias.type, + depth: (request.depth ?? 0) + 1, + activeNames: [...(request.activeNames ?? []), key], + }); + } + return direct; + }, + }; + const capabilities = new Map( + options.capabilities.map(capability => [capability.name, capability]) + ); + const dataclassNames = new Set(ir.classes.filter(cls => cls.is_dataclass).map(cls => cls.name)); + const diagnostics: CallableCompilationDiagnostic[] = []; + const callables: ResolvedCallable[] = []; + const compileFunction = (func: PythonFunction, path: string): PythonFunction => { + const result = resolveCallable( + func, + path, + conversion, + capabilities, + dataclassNames, + diagnostics + ); + callables.push(result.callable); + return result.function; + }; + const module: PythonModule = { + ...canonical, + functions: selectedFunctions.map(func => + compileFunction( + func, + `$.functions[${ir.functions.findIndex(entry => entry.name === func.name)}]` + ) + ), + classes: selectedClasses.map(cls => ({ + ...cls, + methods: + cls.kind === 'protocol' + ? cls.methods + : cls.methods.map(method => { + const sourceClassIndex = ir.classes.findIndex(entry => entry.name === cls.name); + const sourceMethodIndex = ir.classes[sourceClassIndex]!.methods.findIndex( + entry => entry.name === method.name + ); + return compileFunction( + method, + `$.classes[${sourceClassIndex}].methods[${sourceMethodIndex}]` + ); + }), + })), + typeAliases: selectedAliases, + }; + const generated = options.generator.generateModuleDefinition(module, options.annotatedJSDoc); + return { + ir: { ir_version: ir.ir_version, module: ir.module }, + module, + callables, + diagnostics, + files: [ + { kind: 'typescript', content: generated.typescript }, + { kind: 'declaration', content: generated.declaration }, + ], + generated, + }; +} diff --git a/src/core/generator.ts b/src/core/generator.ts index 899fa6a1..41cddf4e 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -9,9 +9,12 @@ import type { PythonModule, PythonType, PythonTypeAlias, + Parameter, GeneratedCode, TypescriptType, } from '../types/index.js'; +import type { ValueContract } from '../contracts/value-contract.js'; +import type { OverloadReturnSchema, ReturnSchema } from '../runtime/validators.js'; import { globalCache } from '../utils/cache.js'; import { @@ -36,7 +39,49 @@ interface GenericRenderContext { emittedParamSpecs: Set; } -type ReturnDefinitionNames = ReadonlySet; +type ReturnDefinitionNames = ReadonlySet & { readonly moduleName?: string }; + +function valueContractToReturnSchema(value: ValueContract): ReturnSchema { + switch (value.kind) { + case 'null': + return { kind: 'primitive', type: 'null' }; + case 'boolean': + return { kind: 'primitive', type: 'boolean' }; + case 'integer': + return { kind: 'primitive', type: 'number', constraint: value.constraint }; + case 'float': + return { kind: 'primitive', type: 'number', constraint: value.constraint }; + case 'string': + return { kind: 'primitive', type: 'string' }; + case 'bytes': + return { kind: 'primitive', type: 'Uint8Array' }; + case 'sequence': + return { kind: 'array', element: valueContractToReturnSchema(value.item) }; + case 'tuple': + return { kind: 'tuple', elements: value.items.map(valueContractToReturnSchema) }; + case 'union': + return { kind: 'union', options: value.options.map(valueContractToReturnSchema) }; + case 'record': + return { + kind: 'record', + fields: Object.fromEntries( + value.fields.map(field => [ + field.name, + { schema: valueContractToReturnSchema(field.value), optional: !field.required }, + ]) + ), + values: value.additionalValues + ? valueContractToReturnSchema(value.additionalValues) + : undefined, + }; + case 'ndarray-float16': + return { kind: 'marker', marker: 'ndarray', dtype: value.dtype, dims: value.rank }; + case 'torch-float16': + return { kind: 'marker', marker: 'torch.tensor', dtype: value.dtype }; + case 'unsupported': + return { kind: 'any' }; + } +} export interface CodeGeneratorOptions { /** Reports a generated annotation that cannot be represented by emitted declarations. */ @@ -101,6 +146,60 @@ export class CodeGenerator { this.onTypeDegrade = options.onTypeDegrade; } + private assertRuntimeGetterIdentifier(identifier: string): void { + if ( + identifier !== 'getRuntimeBridge' && + !/^__tywrapRuntimeProvider(?:[1-9][0-9]*)?$/.test(identifier) + ) { + throw new Error(`Invalid runtime getter identifier: ${identifier}`); + } + } + + private assertBindingGetterAvailable(module: PythonModule, identifier: string): void { + const occupied = new Set([ + 'getRuntimeBridge', + 'createReturnValidator', + 'selectOverloadReturnValidator', + 'ReturnSchema', + '__tywrapReturnDefinitions', + '__tywrapFloat16Value', + '__tywrapFloat16Tensor', + '__tywrapCreateApi', + '__args', + '__kwargs', + '__candidate', + '__varargs', + '__positionalOnly', + '__requiredKwOnly', + '__missing', + '__selectedReturnValidator', + 'kwargs', + 'key', + ]); + const includeCallable = (func: PythonFunction, owner = ''): void => { + occupied.add(this.escapeIdentifier(func.name)); + occupied.add( + `__validate${owner}${this.escapeIdentifier(func.name, { preserveCase: true })}Result` + ); + for (const parameter of [ + ...func.parameters, + ...(func.overloads ?? []).flatMap(overload => overload.parameters), + ]) { + occupied.add(this.escapeIdentifier(parameter.name)); + } + }; + module.functions.forEach(func => includeCallable(func)); + for (const cls of module.classes) { + occupied.add(this.escapeIdentifier(cls.name)); + const owner = this.escapeIdentifier(cls.name, { preserveCase: true }); + cls.methods.forEach(method => includeCallable(method, owner)); + } + (module.typeAliases ?? []).forEach(alias => occupied.add(this.escapeIdentifier(alias.name))); + if (occupied.has(identifier)) { + throw new Error(`Runtime getter identifier ${identifier} conflicts with a generated binding`); + } + } + /** * Convert Python snake_case to TypeScript camelCase and escape reserved words */ @@ -349,6 +448,20 @@ export class CodeGenerator { ); } + private resolvedReturnType( + contract: ValueContract | undefined, + logicalType: PythonType, + ctx: GenericRenderContext + ): string { + if (contract?.kind === 'ndarray-float16') { + return '__tywrapFloat16Value'; + } + if (contract?.kind === 'torch-float16') { + return '__tywrapFloat16Tensor'; + } + return this.typeToTsFromPython(logicalType, ctx, 'return'); + } + private isLocalTypeIdentity( type: { name: string; module?: string }, ctx: GenericRenderContext @@ -492,7 +605,10 @@ export class CodeGenerator { values: schema(current.typeArgs[1] ?? { kind: 'custom', name: 'Any' }), }; } - return definitions.has(leaf) ? { kind: 'ref', name: leaf } : { kind: 'any' }; + return definitions.has(leaf) && + (current.module === undefined || current.module === definitions.moduleName) + ? { kind: 'ref', name: leaf } + : { kind: 'any' }; } case 'custom': { const leaf = current.name.split('.').at(-1) ?? current.name; @@ -527,7 +643,10 @@ export class CodeGenerator { ) { return { kind: 'marker', marker: 'sklearn.estimator' }; } - return definitions.has(leaf) ? { kind: 'ref', name: leaf } : { kind: 'any' }; + return definitions.has(leaf) && + (current.module === undefined || current.module === definitions.moduleName) + ? { kind: 'ref', name: leaf } + : { kind: 'any' }; } case 'typevar': case 'paramspec': @@ -542,12 +661,71 @@ export class CodeGenerator { return schema(type); } + private resolvedReturnSchema( + func: PythonFunction, + definitions: ReturnDefinitionNames + ): ReturnSchema { + const value = func.callableContract?.returnValue; + return value + ? valueContractToReturnSchema(value) + : (this.returnSchema( + func.callableContract?.returnValidationType ?? func.returnType, + definitions + ) as ReturnSchema); + } + + private overloadReturnSchemas( + func: PythonFunction, + definitions: ReturnDefinitionNames + ): OverloadReturnSchema[] { + return (func.overloads ?? []).map((overload, overloadIndex) => { + const contract = func.callableContract?.overloads[overloadIndex]; + return { + parameters: overload.parameters + .map((parameter, parameterIndex) => ({ parameter, parameterIndex })) + .filter(({ parameter }) => parameter.name !== 'self' && parameter.name !== 'cls') + .map(({ parameter, parameterIndex }) => ({ + name: parameter.name, + kind: parameter.varArgs + ? ('var-positional' as const) + : parameter.kwArgs + ? ('var-keyword' as const) + : parameter.keywordOnly + ? ('keyword-only' as const) + : parameter.positionalOnly + ? ('positional-only' as const) + : ('positional-or-keyword' as const), + optional: parameter.optional, + value: contract?.parameterValues[parameterIndex] + ? valueContractToReturnSchema(contract.parameterValues[parameterIndex]) + : (this.returnSchema(parameter.type, definitions) as ReturnSchema), + })), + result: contract?.returnValue + ? valueContractToReturnSchema(contract.returnValue) + : (this.returnSchema( + contract?.returnValidationType ?? overload.returnType, + definitions + ) as ReturnSchema), + selectable: + contract !== undefined && + contract.returnValue !== undefined && + overload.parameters.every( + (parameter, index) => + parameter.name === 'self' || + parameter.name === 'cls' || + contract.parameterValues[index] !== undefined + ), + }; + }); + } + private returnDefinitions(module: PythonModule): ReturnDefinitionNames { - return new Set( + const names = new Set( module.classes .filter(cls => cls.kind === 'typed_dict' || cls.decorators.includes('__typed_dict__')) .map(cls => cls.name) ); + return Object.assign(names, { moduleName: module.name }); } private emitReturnDefinitions(module: PythonModule): string { @@ -574,13 +752,16 @@ export class CodeGenerator { moduleName?: string, annotatedJSDoc = false, localDeclaredNames: Set = new Set(), - returnDefinitions: ReturnDefinitionNames = new Set() + returnDefinitions: ReturnDefinitionNames = new Set(), + runtimeGetterIdentifier = 'getRuntimeBridge' ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const jsdoc = this.generateJsDoc( func.docstring, annotatedJSDoc ? func.parameters.map(p => String(p.type)) : undefined ); const filteredParams = func.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'); + const hasDeclaredOverloads = (func.overloads?.length ?? 0) > 0; const keywordOnlyParams = filteredParams.filter(p => p.keywordOnly); const positionalOnlyNames = filteredParams.filter(p => p.positionalOnly).map(p => p.name); const hasVarKwArgs = filteredParams.some(p => p.kwArgs); @@ -592,19 +773,29 @@ export class CodeGenerator { const positionalParams = filteredParams.filter(p => !p.keywordOnly && !p.varArgs && !p.kwArgs); const genericContext = this.buildGenericRenderContext( this.getTypeParameters(func.typeParameters), - [func.returnType, ...filteredParams.map(param => param.type)], + [ + func.returnType, + ...filteredParams.map(param => param.type), + ...(func.overloads ?? []).flatMap(overload => [ + overload.returnType, + ...overload.parameters.map(parameter => parameter.type), + ]), + ], moduleName, localDeclaredNames ); const typeParamDecl = genericContext.declaration; - const tsTypeForValue = (p: (typeof filteredParams)[number]): string => + const tsTypeForValue = (p: Parameter): string => this.typeToTsFromPython(p.type, genericContext, 'value'); const kwargsType = (() => { if (!needsKwargsParam) { return ''; } + if (hasDeclaredOverloads) { + return 'Record'; + } if (keywordOnlyParams.length === 0 && hasVarKwArgs) { return 'Record'; } @@ -621,7 +812,7 @@ export class CodeGenerator { ): string => { const pname = this.escapeIdentifier(p.name); const opt = !forceRequired && p.optional ? '?' : ''; - return `${pname}${opt}: ${tsTypeForValue(p)}`; + return `${pname}${opt}: ${hasDeclaredOverloads ? 'unknown' : tsTypeForValue(p)}`; }; const renderVarArgsParam = (forceRequired = false): string | null => { @@ -659,13 +850,116 @@ export class CodeGenerator { const paramDecl = implParams.join(', '); const hasKwArgs = needsKwargsParam; - const returnType = this.typeToTsFromPython(func.returnType, genericContext, 'return'); + const returnType = this.resolvedReturnType( + func.callableContract?.returnValue, + func.returnType, + genericContext + ); + const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const fname = this.escapeIdentifier(func.name); const moduleId = moduleName ?? '__main__'; const validatorName = `__validate${this.escapeIdentifier(func.name, { preserveCase: true })}Result`; - const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.returnSchema(func.returnType, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; + const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(func, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n\n`; + + const renderDeclaredOverload = ( + overload: NonNullable[number], + overloadIndex: number + ): string[] => { + const overloadParams = overload.parameters.filter( + parameter => parameter.name !== 'self' && parameter.name !== 'cls' + ); + const overloadPositional = overloadParams.filter( + parameter => !parameter.keywordOnly && !parameter.varArgs && !parameter.kwArgs + ); + const overloadKeywordOnly = overloadParams.filter(parameter => parameter.keywordOnly); + const overloadVarArgs = overloadParams.find(parameter => parameter.varArgs); + const overloadHasVarKwArgs = overloadParams.some(parameter => parameter.kwArgs); + const overloadNeedsKwargs = overloadKeywordOnly.length > 0 || overloadHasVarKwArgs; + const overloadNeedsVarArgsArray = Boolean(overloadVarArgs) && overloadNeedsKwargs; + const overloadKwargsType = (() => { + if (!overloadNeedsKwargs) { + return ''; + } + if (overloadKeywordOnly.length === 0 && overloadHasVarKwArgs) { + return 'Record'; + } + const properties = overloadKeywordOnly + .map( + parameter => + `${JSON.stringify(parameter.name)}${parameter.optional ? '?' : ''}: ${tsTypeForValue(parameter)};` + ) + .join(' '); + const object = `{ ${properties} }`; + return overloadHasVarKwArgs ? `(${object} & Record)` : object; + })(); + const renderPositional = (parameter: Parameter, forceRequired = false): string => + `${this.escapeIdentifier(parameter.name)}${!forceRequired && parameter.optional ? '?' : ''}: ${tsTypeForValue(parameter)}`; + const renderVarArgs = (forceRequired = false): string | null => { + if (!overloadVarArgs) { + return null; + } + if (!overloadNeedsVarArgsArray) { + return `...${this.escapeIdentifier(overloadVarArgs.name)}: unknown[]`; + } + return `${this.escapeIdentifier(overloadVarArgs.name)}${forceRequired ? '' : '?'}: unknown[]`; + }; + const renderKwargs = (forceRequired = false): string | null => + overloadNeedsKwargs ? `kwargs${forceRequired ? '' : '?'}: ${overloadKwargsType}` : null; + const overloadReturnType = this.resolvedReturnType( + func.callableContract?.overloads[overloadIndex]?.returnValue, + overload.returnType, + genericContext + ); + const signatures: string[] = []; + const addSignature = (parameters: string[]): void => { + signatures.push( + `export function ${fname}${typeParamDecl}(${parameters.join(', ')}): Promise<${overloadReturnType}>;` + ); + }; + const firstOptional = overloadPositional.findIndex(parameter => parameter.optional); + const requiredPositionalCount = + firstOptional >= 0 ? firstOptional : overloadPositional.length; + const requiredKwOnly = overloadKeywordOnly.some(parameter => !parameter.optional); + + if (requiredKwOnly) { + for (let count = requiredPositionalCount; count <= overloadPositional.length; count += 1) { + const head = overloadPositional + .slice(0, count) + .map(parameter => renderPositional(parameter, true)); + const rest: string[] = []; + if (overloadVarArgs) { + if (overloadNeedsVarArgsArray) { + rest.push(`${this.escapeIdentifier(overloadVarArgs.name)}: unknown[] | undefined`); + } else { + rest.push(`...${this.escapeIdentifier(overloadVarArgs.name)}: unknown[]`); + } + } + const kwargs = renderKwargs(true); + if (kwargs) { + rest.push(kwargs); + } + addSignature([...head, ...rest]); + if (overloadVarArgs && overloadNeedsVarArgsArray && kwargs) { + addSignature([...head, kwargs]); + } + } + return signatures; + } + + const parameters = overloadPositional.map(parameter => renderPositional(parameter)); + const varArgs = renderVarArgs(); + if (varArgs) { + parameters.push(varArgs); + } + const kwargs = renderKwargs(); + if (kwargs) { + parameters.push(kwargs); + } + addSignature(parameters); + return signatures; + }; - // Overloads: generate trailing optional parameter drop variants (exclude *args/**kwargs). + // Optional parameter overloads make Python's trailing defaults available to TypeScript callers. // Why: Python APIs frequently have many optional tail params. TypeScript callers expect // `fn(a)`, `fn(a, b)`, ... all to typecheck. We emit a family of overloads that progressively // "drop" optional tail args, but also include the full positional signature (<= length) so a @@ -673,8 +967,8 @@ export class CodeGenerator { const firstOptionalIndex = positionalParams.findIndex(p => p.optional); const requiredKwOnlyNames = keywordOnlyParams.filter(p => !p.optional).map(p => p.name); const keywordOnlyNames = keywordOnlyParams.map(p => p.name); - const overloads: string[] = []; - if (requiredKwOnlyNames.length > 0) { + const overloads: string[] = (func.overloads ?? []).flatMap(renderDeclaredOverload); + if ((func.overloads?.length ?? 0) === 0 && requiredKwOnlyNames.length > 0) { // Required keyword-only params must be represented with a required `kwargs` parameter. // Avoid "required after optional" by emitting overloads where all preceding parameters are required. const requiredPosCount = @@ -711,7 +1005,12 @@ export class CodeGenerator { ); } } - } else if (firstOptionalIndex >= 0 && !varArgsParam && !needsKwargsParam) { + } else if ( + (func.overloads?.length ?? 0) === 0 && + firstOptionalIndex >= 0 && + !varArgsParam && + !needsKwargsParam + ) { for (let i = firstOptionalIndex; i <= positionalParams.length; i++) { const head = positionalParams.slice(0, i).map(p => renderPositionalParam(p)); const rest: string[] = []; @@ -751,9 +1050,14 @@ export class CodeGenerator { const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; const callPreludeLines = emitCallPrelude(callDescriptor, this.callEmitHelpers()); const callPrelude = callPreludeLines.length > 0 ? `${callPreludeLines.join('\n')}\n` : ''; - - const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${typeParamDecl}(${paramDecl}): Promise<${returnType}> { -${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}); + const overloadValidator = + (func.overloads?.length ?? 0) > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(func, returnDefinitions))}, __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${func.name}`)}, __tywrapReturnDefinitions);\n` + : ''; + const selectedValidatorName = overloadValidator ? '__selectedReturnValidator' : validatorName; + + const ts = `${jsdoc}${returnValidator}${overloadDecl}export async function ${fname}${hasDeclaredOverloads ? '' : typeParamDecl}(${paramDecl}): Promise<${implementationReturnType}> { +${callPrelude}${guards}${overloadValidator} return ${runtimeGetterIdentifier}().call<${implementationReturnType}>('${moduleId}', '${func.name}', __args, ${hasKwArgs ? '__kwargs' : 'undefined'}, ${selectedValidatorName}); } `; @@ -770,8 +1074,10 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module moduleName?: string, _annotatedJSDoc = false, localDeclaredNames: Set = new Set(), - returnDefinitions: ReturnDefinitionNames = new Set() + returnDefinitions: ReturnDefinitionNames = new Set(), + runtimeGetterIdentifier = 'getRuntimeBridge' ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const moduleDeclaredNames = new Set(localDeclaredNames); moduleDeclaredNames.add(cls.name); const jsdoc = this.generateJsDoc(cls.docstring); @@ -886,6 +1192,7 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module // the ordinary module call path and never retain process-local state. const staticPrefix = 'static '; const fparams = method.parameters.filter(p => p.name !== 'self' && p.name !== 'cls'); + const hasDeclaredOverloads = (method.overloads?.length ?? 0) > 0; const methodOwnGenericContext = this.buildGenericRenderContext( this.getTypeParameters(method.typeParameters), [method.returnType, ...fparams.map(param => param.type)], @@ -917,13 +1224,16 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module ): string => { const pname = this.escapeIdentifier(p.name); const opt = !forceRequired && p.optional ? '?' : ''; - return `${pname}${opt}: ${methodTsValueType(p)}`; + return `${pname}${opt}: ${hasDeclaredOverloads ? 'unknown' : methodTsValueType(p)}`; }; const kwargsType = (() => { if (!needsKwargsParam) { return ''; } + if (hasDeclaredOverloads) { + return 'Record'; + } if (keywordOnlyParams.length === 0 && hasVarKwArgs) { return 'Record'; } @@ -950,17 +1260,40 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module const paramsDecl = paramsDeclParts.join(', '); const requiredKwOnlyNames = keywordOnlyParams.filter(p => !p.optional).map(p => p.name); - const returnType = this.typeToTsFromPython( + const returnType = this.resolvedReturnType( + method.callableContract?.returnValue, method.returnType, - methodGenericContext, - 'return' + methodGenericContext ); + const implementationReturnType = hasDeclaredOverloads ? 'unknown' : returnType; const mname = this.escapeIdentifier(method.name); const validatorName = `__validate${this.escapeIdentifier(cls.name, { preserveCase: true })}${this.escapeIdentifier(method.name, { preserveCase: true })}Result`; - const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.returnSchema(method.returnType, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; - - const overloads: string[] = []; - if (needsKwargsParam && requiredKwOnlyNames.length > 0) { + const returnValidator = `const ${validatorName} = createReturnValidator(${JSON.stringify(this.resolvedReturnSchema(method, returnDefinitions))}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n\n`; + + const declaredOverloads = + (method.overloads?.length ?? 0) > 0 + ? (this.generateFunctionWrapper( + { + ...method, + parameters: fparams, + overloads: method.overloads?.map(overload => ({ + ...overload, + parameters: overload.parameters.filter( + p => p.name !== 'self' && p.name !== 'cls' + ), + })), + }, + moduleName, + false, + moduleDeclaredNames, + returnDefinitions, + runtimeGetterIdentifier + ) + .declaration.match(/^export function .*;$/gm) + ?.map(line => ` static ${line.slice('export function '.length)}`) ?? []) + : []; + const overloads: string[] = [...declaredOverloads]; + if (declaredOverloads.length === 0 && needsKwargsParam && requiredKwOnlyNames.length > 0) { const firstOptionalIndex = positionalParams.findIndex(p => p.optional); const requiredPosCount = firstOptionalIndex >= 0 ? firstOptionalIndex : positionalParams.length; @@ -1004,10 +1337,17 @@ ${callPrelude}${guards} return getRuntimeBridge().call<${returnType}>('${module const guardLines = emitArgGuards(callDescriptor); const guards = guardLines.length > 0 ? `${guardLines.join('\n')}\n` : ''; - - const callExpr = `getRuntimeBridge().call<${returnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName})`; - methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${methodTypeParamDecl}(${paramsDecl}): Promise<${returnType}> { - ${returnValidator}${callPrelude}${guards} return ${callExpr}; + const overloadValidator = + declaredOverloads.length > 0 + ? ` const __selectedReturnValidator = selectOverloadReturnValidator(${JSON.stringify(this.overloadReturnSchemas(method, returnDefinitions))}, __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${validatorName}, ${JSON.stringify(`${moduleId}.${cls.name}.${method.name}`)}, __tywrapReturnDefinitions);\n` + : ''; + const selectedValidatorName = overloadValidator + ? '__selectedReturnValidator' + : validatorName; + + const callExpr = `${runtimeGetterIdentifier}().call<${implementationReturnType}>('${moduleId}', '${cls.name}.${method.name}', __args, ${needsKwargsParam ? '__kwargs' : 'undefined'}, ${selectedValidatorName})`; + methodBodies.push(`${overloadDecl} ${staticPrefix}async ${mname}${hasDeclaredOverloads ? '' : methodTypeParamDecl}(${paramsDecl}): Promise<${implementationReturnType}> { + ${returnValidator}${callPrelude}${guards}${overloadValidator} return ${callExpr}; }`); methodDeclarations.push( `${overloadDecl}${overloads.length > 0 ? '' : ` ${staticPrefix}${mname}${methodTypeParamDecl}(${paramsDecl}): Promise<${returnType}>;\n`}` @@ -1081,6 +1421,30 @@ ${migrationNote}${declarationMethodsSection} } generateModuleDefinition(module: PythonModule, annotatedJSDoc = false): GeneratedCode { + return this.generateModuleWithGetter(module, annotatedJSDoc, 'getRuntimeBridge', true); + } + + /** Build an intermediate template. The caller must bind its provider before emission. */ + generateModuleBindingTemplate( + module: PythonModule, + annotatedJSDoc = false, + runtimeGetterIdentifier = '__tywrapRuntimeProvider' + ): GeneratedCode { + if (runtimeGetterIdentifier === 'getRuntimeBridge') { + throw new Error('Binding templates require a distinct runtime getter'); + } + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); + this.assertBindingGetterAvailable(module, runtimeGetterIdentifier); + return this.generateModuleWithGetter(module, annotatedJSDoc, runtimeGetterIdentifier, false); + } + + private generateModuleWithGetter( + module: PythonModule, + annotatedJSDoc: boolean, + runtimeGetterIdentifier: string, + importRuntimeGetter: boolean + ): GeneratedCode { + this.assertRuntimeGetterIdentifier(runtimeGetterIdentifier); const localDeclaredNames = new Set([ ...module.classes.map(cls => cls.name), ...(module.typeAliases ?? []).map(alias => alias.name), @@ -1093,7 +1457,8 @@ ${migrationNote}${declarationMethodsSection} module.name, annotatedJSDoc, localDeclaredNames, - this.returnDefinitions(module) + this.returnDefinitions(module), + runtimeGetterIdentifier ) ); const classResults = [...module.classes] @@ -1104,7 +1469,8 @@ ${migrationNote}${declarationMethodsSection} module.name, annotatedJSDoc, localDeclaredNames, - this.returnDefinitions(module) + this.returnDefinitions(module), + runtimeGetterIdentifier ) ); const typeAliasResults = [...(module.typeAliases ?? [])] @@ -1122,12 +1488,39 @@ ${migrationNote}${declarationMethodsSection} return kind === 'class' && !c.decorators.includes('__typed_dict__'); }); const needsRuntime = module.functions.length > 0 || hasRuntimeClasses; + const hasOverloads = + module.functions.some(func => (func.overloads?.length ?? 0) > 0) || + module.classes.some(cls => cls.methods.some(method => (method.overloads?.length ?? 0) > 0)); + const emittedCallables = [ + ...module.functions, + ...module.classes.flatMap(cls => + cls.methods.filter( + method => + method.name !== '__init__' && + (method.methodKind === 'class' || method.methodKind === 'static') + ) + ), + ]; + const resolvedReturns = emittedCallables.flatMap(func => [ + func.callableContract?.returnValue, + ...(func.callableContract?.overloads.map(overload => overload.returnValue) ?? []), + ]); + const needsTorchFloat16 = resolvedReturns.some(value => value?.kind === 'torch-float16'); + const needsFloat16 = + needsTorchFloat16 || resolvedReturns.some(value => value?.kind === 'ndarray-float16'); + const scientificTypes = needsFloat16 + ? `type __tywrapFloat16Value = number | __tywrapFloat16Value[];\n${ + needsTorchFloat16 + ? `type __tywrapFloat16Tensor = { data: __tywrapFloat16Value; shape: number[]; dtype: 'torch.float16'; device?: string; sourceDtype?: string; sourceDevice?: string };\n` + : '' + }\n` + : ''; const bridgeDecl = needsRuntime - ? `import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` + ? `import { createReturnValidator, ${hasOverloads ? 'selectOverloadReturnValidator, ' : ''}${importRuntimeGetter ? 'getRuntimeBridge, ' : ''}type ReturnSchema } from 'tywrap/runtime';\n\n${this.emitReturnDefinitions(module)}` : ''; - const ts = `${`${header}${bridgeDecl}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; - const declaration = `${`${declarationHeader}${functionResults + const ts = `${`${header}${bridgeDecl}${scientificTypes}${functionCodes}\n${classCodes}\n${typeAliasCodes}`.trimEnd()}\n`; + const declaration = `${`${declarationHeader}${scientificTypes}${functionResults .map(result => result.declaration) .join('\n')}\n${classResults .map(result => result.declaration) @@ -1158,7 +1551,7 @@ ${migrationNote}${declarationMethodsSection} declaration, sourceMap: undefined, metadata: { - generatedAt: new Date(), + generatedAt: new Date(0), sourceFiles: [], runtime: 'auto', optimizations: [], diff --git a/src/core/ir-contract.ts b/src/core/ir-contract.ts new file mode 100644 index 00000000..9ff2ded2 --- /dev/null +++ b/src/core/ir-contract.ts @@ -0,0 +1,553 @@ +/** + * Validate the JSON contract emitted by tywrap_ir 0.4 before compilation. + * + * This module does not read files, start Python, or write generated output. + * It only accepts unknown JSON-compatible input and returns located diagnostics. + */ + +import type { IrContract } from '../types/index.js'; + +export const TYWRAP_IR_VERSION = '0.4.0' as const; + +const PARAMETER_KINDS = new Set([ + 'POSITIONAL_ONLY', + 'POSITIONAL_OR_KEYWORD', + 'VAR_POSITIONAL', + 'KEYWORD_ONLY', + 'VAR_KEYWORD', +]); +const FIELD_KINDS = new Set([...PARAMETER_KINDS, 'FIELD']); + +const METHOD_KINDS = new Set(['instance', 'class', 'static']); +const TYPE_PARAMETER_KINDS = new Set(['typevar', 'paramspec', 'typevartuple']); +const VARIANCES = new Set(['covariant', 'contravariant', 'invariant']); + +export interface IrParameter { + name: string; + kind: string; + annotation: string | null; + default: boolean; +} + +export interface IrTypeParameter { + name: string; + kind: 'typevar' | 'paramspec' | 'typevartuple'; + bound: string | null; + constraints: string[] | null; + variance: 'covariant' | 'contravariant' | 'invariant' | null; +} + +export interface IrOverload { + parameters: IrParameter[]; + returns: string | null; +} + +export interface IrFunction { + name: string; + qualname: string; + docstring: string | null; + parameters: IrParameter[]; + returns: string | null; + is_async: boolean; + is_generator: boolean; + type_params: IrTypeParameter[]; + method_kind: 'instance' | 'class' | 'static'; + overloads: IrOverload[]; +} + +export interface IrAccessor { + name: string; + returns: string | null; + docstring: string | null; + read_only: boolean | null; + is_cached: boolean; +} + +export interface IrClass { + name: string; + qualname: string; + docstring: string | null; + bases: string[]; + methods: IrFunction[]; + typed_dict: boolean; + total: boolean | null; + fields: IrParameter[]; + is_protocol: boolean; + is_namedtuple: boolean; + is_dataclass: boolean; + is_pydantic: boolean; + type_params: IrTypeParameter[]; + accessors: IrAccessor[]; +} + +export interface IrConstant { + name: string; + annotation: string | null; + value_repr: string | null; + is_final: boolean; +} + +export interface IrTypeAlias { + name: string; + definition: string; + is_generic: boolean; + type_params: IrTypeParameter[]; +} + +export interface ValidatedIrContract extends IrContract { + ir_version: typeof TYWRAP_IR_VERSION; + module: string; + functions: IrFunction[]; + classes: IrClass[]; + constants: IrConstant[]; + type_aliases: IrTypeAlias[]; + metadata: Record; + warnings: string[]; +} + +export interface IrDiagnostic { + code: 'ir-version-mismatch' | 'contract-invalid'; + path: string; + message: string; +} + +export interface ValidateIrContractOptions { + /** + * Generated offline contracts omit extractor metadata so their bytes stay + * stable. The reader restores an empty metadata object for that format. + */ + allowOmittedMetadata?: boolean; +} + +export type IrValidationResult = + | { ok: true; contract: ValidatedIrContract; diagnostics: readonly [] } + | { ok: false; contract: null; diagnostics: readonly IrDiagnostic[] }; + +class ValidationState { + readonly diagnostics: IrDiagnostic[] = []; + private overflowed = false; + private entriesVisited = 0; + private entryBudgetExceeded = false; + + private static readonly maxDiagnostics = 100; + private static readonly maxEntries = 100_000; + + constructor(private readonly source: string) {} + + invalid(path: string, message: string): void { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return; + } + this.diagnostics.push({ code: 'contract-invalid', path, message: `${this.source} ${message}` }); + } + + versionMismatch(path: string, found: string): void { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return; + } + this.diagnostics.push({ + code: 'ir-version-mismatch', + path, + message: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${this.source} declares ${found}. Regenerate the contract with a matching tywrap_ir.`, + }); + } + + visit(path: string): boolean { + if (this.diagnostics.length >= ValidationState.maxDiagnostics) { + this.overflowed = true; + return false; + } + if (this.entryBudgetExceeded) { + return false; + } + this.entriesVisited += 1; + if (this.entriesVisited > ValidationState.maxEntries) { + this.entryBudgetExceeded = true; + this.invalid(path, `IR exceeds the ${ValidationState.maxEntries} entry validation limit.`); + return false; + } + return true; + } + + finish(): readonly IrDiagnostic[] { + if (!this.overflowed) { + return this.diagnostics; + } + return [ + ...this.diagnostics, + { + code: 'contract-invalid', + path: '$', + message: `${this.source} has more than ${ValidationState.maxDiagnostics} contract errors.`, + }, + ]; + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function visitArray( + values: readonly unknown[], + path: string, + state: ValidationState, + validate: (value: unknown, path: string) => void +): void { + for (let index = 0; index < values.length; index += 1) { + const entryPath = `${path}[${index}]`; + if (!state.visit(entryPath)) { + break; + } + validate(values[index], entryPath); + } +} + +function recordAt( + value: unknown, + path: string, + state: ValidationState +): Record | null { + if (!isRecord(value)) { + state.invalid(path, `${path} must be an object.`); + return null; + } + return value; +} + +function requiredValue( + object: Record, + name: string, + path: string, + state: ValidationState +): unknown { + if (!(name in object)) { + state.invalid(`${path}.${name}`, `${path} is missing required field ${name}.`); + return undefined; + } + return object[name]; +} + +function requiredString( + object: Record, + name: string, + path: string, + state: ValidationState, + allowEmpty = false +): string | null { + const value = requiredValue(object, name, path, state); + if (typeof value !== 'string') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a string.`); + return null; + } + if (!allowEmpty && value.length === 0) { + state.invalid(`${path}.${name}`, `${path}.${name} must not be empty.`); + return null; + } + return value; +} + +function requiredNullableString( + object: Record, + name: string, + path: string, + state: ValidationState +): string | null { + const value = requiredValue(object, name, path, state); + if (value === null) { + return null; + } + if (typeof value !== 'string') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a string or null.`); + return null; + } + return value; +} + +function requiredBoolean( + object: Record, + name: string, + path: string, + state: ValidationState +): boolean | null { + const value = requiredValue(object, name, path, state); + if (typeof value !== 'boolean') { + state.invalid(`${path}.${name}`, `${path}.${name} must be a boolean.`); + return null; + } + return value; +} + +function requiredArray( + object: Record, + name: string, + path: string, + state: ValidationState +): unknown[] | null { + if (!(name in object)) { + const noun = path === '$' ? `required array field ${name}` : `${path}.${name}`; + state.invalid(`${path}.${name}`, `${path} is missing ${noun}.`); + return null; + } + const value = object[name]; + if (!Array.isArray(value)) { + state.invalid(`${path}.${name}`, `${path}.${name} must be an array.`); + return null; + } + return value; +} + +function validateStringArray(value: unknown, path: string, state: ValidationState): void { + if (!Array.isArray(value)) { + state.invalid(path, `${path} must be an array.`); + return; + } + visitArray(value, path, state, (entry, entryPath) => { + if (typeof entry !== 'string') { + state.invalid(entryPath, `${entryPath} must be a string.`); + } + }); +} + +function validateParameter( + value: unknown, + path: string, + state: ValidationState, + allowedKinds: ReadonlySet = PARAMETER_KINDS +): void { + const parameter = recordAt(value, path, state); + if (!parameter) { + return; + } + requiredString(parameter, 'name', path, state); + const kind = requiredString(parameter, 'kind', path, state); + if (kind !== null && !allowedKinds.has(kind)) { + state.invalid(`${path}.kind`, `${path}.kind must be a supported Python parameter kind.`); + } + requiredNullableString(parameter, 'annotation', path, state); + requiredBoolean(parameter, 'default', path, state); +} + +function validateTypeParameter(value: unknown, path: string, state: ValidationState): void { + const parameter = recordAt(value, path, state); + if (!parameter) { + return; + } + requiredString(parameter, 'name', path, state); + const kind = requiredString(parameter, 'kind', path, state); + if (kind !== null && !TYPE_PARAMETER_KINDS.has(kind)) { + state.invalid(`${path}.kind`, `${path}.kind must be typevar, paramspec, or typevartuple.`); + } + requiredNullableString(parameter, 'bound', path, state); + const constraints = requiredValue(parameter, 'constraints', path, state); + if (constraints !== null) { + validateStringArray(constraints, `${path}.constraints`, state); + } + const variance = requiredValue(parameter, 'variance', path, state); + if (variance !== null && (typeof variance !== 'string' || !VARIANCES.has(variance))) { + state.invalid(`${path}.variance`, `${path}.variance must be a supported variance or null.`); + } +} + +function validateFunction(value: unknown, path: string, state: ValidationState): void { + const callable = recordAt(value, path, state); + if (!callable) { + return; + } + requiredString(callable, 'name', path, state); + requiredString(callable, 'qualname', path, state); + requiredNullableString(callable, 'docstring', path, state); + const parameters = requiredArray(callable, 'parameters', path, state); + if (parameters) { + visitArray(parameters, `${path}.parameters`, state, (parameter, entryPath) => + validateParameter(parameter, entryPath, state) + ); + } + requiredNullableString(callable, 'returns', path, state); + requiredBoolean(callable, 'is_async', path, state); + requiredBoolean(callable, 'is_generator', path, state); + const typeParameters = requiredArray(callable, 'type_params', path, state); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state) + ); + } + const methodKind = requiredString(callable, 'method_kind', path, state); + if (methodKind !== null && !METHOD_KINDS.has(methodKind)) { + state.invalid(`${path}.method_kind`, `${path}.method_kind must be instance, class, or static.`); + } + const overloads = requiredArray(callable, 'overloads', path, state); + if (overloads) { + visitArray(overloads, `${path}.overloads`, state, (overload, overloadPath) => { + const signature = recordAt(overload, overloadPath, state); + if (!signature) { + return; + } + const overloadParameters = requiredArray(signature, 'parameters', overloadPath, state); + if (overloadParameters) { + visitArray( + overloadParameters, + `${overloadPath}.parameters`, + state, + (parameter, entryPath) => validateParameter(parameter, entryPath, state) + ); + } + requiredNullableString(signature, 'returns', overloadPath, state); + }); + } +} + +function validateAccessor(value: unknown, path: string, state: ValidationState): void { + const accessor = recordAt(value, path, state); + if (!accessor) { + return; + } + requiredString(accessor, 'name', path, state); + requiredNullableString(accessor, 'returns', path, state); + requiredNullableString(accessor, 'docstring', path, state); + const readOnly = requiredValue(accessor, 'read_only', path, state); + if (readOnly !== null && typeof readOnly !== 'boolean') { + state.invalid(`${path}.read_only`, `${path}.read_only must be a boolean or null.`); + } + requiredBoolean(accessor, 'is_cached', path, state); +} + +function validateClass(value: unknown, path: string, state: ValidationState): void { + const cls = recordAt(value, path, state); + if (!cls) { + return; + } + requiredString(cls, 'name', path, state); + requiredString(cls, 'qualname', path, state); + requiredNullableString(cls, 'docstring', path, state); + validateStringArray(requiredValue(cls, 'bases', path, state), `${path}.bases`, state); + const methods = requiredArray(cls, 'methods', path, state); + if (methods) { + visitArray(methods, `${path}.methods`, state, (method, entryPath) => + validateFunction(method, entryPath, state) + ); + } + requiredBoolean(cls, 'typed_dict', path, state); + const total = requiredValue(cls, 'total', path, state); + if (total !== null && typeof total !== 'boolean') { + state.invalid(`${path}.total`, `${path}.total must be a boolean or null.`); + } + const fields = requiredArray(cls, 'fields', path, state); + if (fields) { + visitArray(fields, `${path}.fields`, state, (field, entryPath) => + validateParameter(field, entryPath, state, FIELD_KINDS) + ); + } + requiredBoolean(cls, 'is_protocol', path, state); + requiredBoolean(cls, 'is_namedtuple', path, state); + requiredBoolean(cls, 'is_dataclass', path, state); + requiredBoolean(cls, 'is_pydantic', path, state); + const typeParameters = requiredArray(cls, 'type_params', path, state); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state) + ); + } + const accessors = requiredArray(cls, 'accessors', path, state); + if (accessors) { + visitArray(accessors, `${path}.accessors`, state, (accessor, entryPath) => + validateAccessor(accessor, entryPath, state) + ); + } +} + +function validateConstant(value: unknown, path: string, state: ValidationState): void { + const constant = recordAt(value, path, state); + if (!constant) { + return; + } + requiredString(constant, 'name', path, state); + requiredNullableString(constant, 'annotation', path, state); + requiredNullableString(constant, 'value_repr', path, state); + requiredBoolean(constant, 'is_final', path, state); +} + +function validateTypeAlias(value: unknown, path: string, state: ValidationState): void { + const alias = recordAt(value, path, state); + if (!alias) { + return; + } + requiredString(alias, 'name', path, state); + requiredString(alias, 'definition', path, state, true); + requiredBoolean(alias, 'is_generic', path, state); + const typeParameters = requiredArray(alias, 'type_params', path, state); + if (typeParameters) { + visitArray(typeParameters, `${path}.type_params`, state, (parameter, entryPath) => + validateTypeParameter(parameter, entryPath, state) + ); + } +} + +/** + * Validate every field emitted by `tywrap_ir` 0.4. + * + * The return value is safe to compile without further structural checks. The + * caller owns all filesystem, cache, interpreter, and output operations. + */ +export function validateIrContract( + input: unknown, + source = 'IR', + options: ValidateIrContractOptions = {} +): IrValidationResult { + const state = new ValidationState(source); + const contract = recordAt(input, '$', state); + if (!contract) { + return { ok: false, contract: null, diagnostics: state.finish() }; + } + + const version = requiredString(contract, 'ir_version', '$', state); + if (version !== null && version !== TYWRAP_IR_VERSION) { + state.versionMismatch('$.ir_version', version); + } + requiredString(contract, 'module', '$', state); + const functions = requiredArray(contract, 'functions', '$', state); + if (functions) { + visitArray(functions, '$.functions', state, (functionValue, entryPath) => + validateFunction(functionValue, entryPath, state) + ); + } + const classes = requiredArray(contract, 'classes', '$', state); + if (classes) { + visitArray(classes, '$.classes', state, (classValue, entryPath) => + validateClass(classValue, entryPath, state) + ); + } + const constants = requiredArray(contract, 'constants', '$', state); + if (constants) { + visitArray(constants, '$.constants', state, (constant, entryPath) => + validateConstant(constant, entryPath, state) + ); + } + const aliases = requiredArray(contract, 'type_aliases', '$', state); + if (aliases) { + visitArray(aliases, '$.type_aliases', state, (alias, entryPath) => + validateTypeAlias(alias, entryPath, state) + ); + } + const metadata = contract.metadata; + if (metadata === undefined && !options.allowOmittedMetadata) { + state.invalid('$.metadata', '$ is missing required field metadata.'); + } else if (metadata !== undefined && !isRecord(metadata)) { + state.invalid('$.metadata', '$.metadata must be an object.'); + } + validateStringArray(requiredValue(contract, 'warnings', '$', state), '$.warnings', state); + + if (state.diagnostics.length > 0) { + return { ok: false, contract: null, diagnostics: state.finish() }; + } + return { + ok: true, + contract: { + ...contract, + metadata: isRecord(metadata) ? metadata : {}, + } as ValidatedIrContract, + diagnostics: [], + }; +} diff --git a/src/core/ir-model.ts b/src/core/ir-model.ts new file mode 100644 index 00000000..b4526b12 --- /dev/null +++ b/src/core/ir-model.ts @@ -0,0 +1,165 @@ +/** Map validated Python IR to the generator model without runtime effects. */ + +import { parseAnnotationToPythonType } from './annotation-parser.js'; +import type { + IrClass, + IrFunction, + IrParameter, + IrTypeAlias, + IrTypeParameter, + ValidatedIrContract, +} from './ir-contract.js'; +import type { + Parameter, + PythonClass, + PythonFunction, + PythonGenericParameter, + PythonModule, + PythonType, + PythonTypeAlias, +} from '../types/index.js'; + +function collectModuleTypeVarNames(ir: ValidatedIrContract): Set { + const names = new Set(); + for (const constant of ir.constants) { + const name = constant.name.trim(); + if (!name) { + continue; + } + const valueRepr = constant.value_repr?.trim() ?? ''; + if (/^~?[A-Za-z_][A-Za-z0-9_]*$/.test(valueRepr) && valueRepr.replace(/^~/, '') === name) { + names.add(name); + continue; + } + const annotation = constant.annotation?.trim() ?? ''; + if (/(^|\.)(TypeVar|ParamSpec|TypeVarTuple)(\[|\(|$)/.test(annotation)) { + names.add(name); + } + } + return names; +} + +export function transformIrToTsModel( + ir: ValidatedIrContract, + onUnknownTypeName?: (name: string) => void +): PythonModule { + const moduleTypeVarNames = collectModuleTypeVarNames(ir); + const parseType = ( + annotation: unknown, + typeParameters: readonly PythonGenericParameter[] = [] + ): PythonType => + parseAnnotationToPythonType(annotation, { + onUnknownTypeName, + knownTypeVarNames: moduleTypeVarNames, + typeParameters, + }); + const mapTypeParameters = ( + typeParameters: readonly IrTypeParameter[] + ): PythonGenericParameter[] => + typeParameters.map(param => ({ + name: param.name, + kind: param.kind, + bound: param.bound ? parseType(param.bound) : undefined, + constraints: param.constraints ? param.constraints.map(item => parseType(item)) : undefined, + variance: param.variance ?? undefined, + })); + const mapParam = ( + param: IrParameter, + typeParameters: readonly PythonGenericParameter[] = [] + ): Parameter => ({ + name: param.name, + type: parseType(param.annotation, typeParameters), + optional: param.default, + varArgs: param.kind === 'VAR_POSITIONAL', + kwArgs: param.kind === 'VAR_KEYWORD', + positionalOnly: param.kind === 'POSITIONAL_ONLY', + keywordOnly: param.kind === 'KEYWORD_ONLY', + }); + const mapMethodKind = (value: unknown): PythonFunction['methodKind'] => + value === 'class' || value === 'static' ? value : 'instance'; + const mapFunc = ( + func: IrFunction, + inheritedTypeParameters: readonly PythonGenericParameter[] = [] + ): PythonFunction => { + const localTypeParameters = mapTypeParameters(func.type_params); + const annotationTypeParameters = [...inheritedTypeParameters, ...localTypeParameters]; + return { + name: func.name, + signature: { + parameters: func.parameters.map(param => mapParam(param, annotationTypeParameters)), + returnType: parseType(func.returns, annotationTypeParameters), + isAsync: func.is_async, + isGenerator: func.is_generator, + }, + docstring: func.docstring ?? undefined, + decorators: [], + isAsync: func.is_async, + isGenerator: func.is_generator, + typeParameters: [...localTypeParameters], + returnType: parseType(func.returns, annotationTypeParameters), + parameters: func.parameters.map(param => mapParam(param, annotationTypeParameters)), + overloads: func.overloads.map(overload => ({ + parameters: overload.parameters.map(param => mapParam(param, annotationTypeParameters)), + returnType: parseType(overload.returns, annotationTypeParameters), + })), + methodKind: mapMethodKind(func.method_kind), + }; + }; + const mapClass = (cls: IrClass): PythonClass => { + const classTypeParameters = mapTypeParameters(cls.type_params); + return { + name: cls.name, + bases: cls.bases, + methods: cls.methods.map(method => mapFunc(method, classTypeParameters)), + properties: cls.fields.map(field => ({ + name: field.name, + type: parseType(field.annotation, classTypeParameters), + readonly: false, + setter: false, + getter: true, + // A constructor default does not make the returned dataclass field absent. + optional: cls.is_dataclass ? false : field.default, + })), + accessors: cls.accessors.map(accessor => ({ + name: accessor.name, + type: parseType(accessor.returns, classTypeParameters), + docstring: accessor.docstring ?? undefined, + readOnly: accessor.read_only ?? undefined, + isCached: accessor.is_cached, + })), + docstring: cls.docstring ?? undefined, + decorators: cls.typed_dict ? ['__typed_dict__'] : [], + kind: cls.typed_dict + ? 'typed_dict' + : cls.is_protocol + ? 'protocol' + : cls.is_namedtuple + ? 'namedtuple' + : cls.is_dataclass + ? 'dataclass' + : cls.is_pydantic + ? 'pydantic' + : 'class', + typeParameters: classTypeParameters, + }; + }; + const mapTypeAlias = (alias: IrTypeAlias): PythonTypeAlias => { + const typeParameters = mapTypeParameters(alias.type_params); + return { + name: alias.name, + type: parseType(alias.definition, typeParameters), + typeParameters, + }; + }; + return { + name: ir.module, + path: undefined, + version: + typeof ir.metadata.package_version === 'string' ? ir.metadata.package_version : undefined, + functions: ir.functions.map(func => mapFunc(func)), + classes: ir.classes.map(mapClass), + typeAliases: ir.type_aliases.map(mapTypeAlias), + imports: [], + exports: [], + }; +} diff --git a/src/runtime/base-bridge.ts b/src/runtime/base-bridge.ts index 0749a9d2..e3fd049d 100644 --- a/src/runtime/base-bridge.ts +++ b/src/runtime/base-bridge.ts @@ -18,6 +18,7 @@ import type { PythonRuntime, BridgeInfo } from '../types/index.js'; import { DisposableBase } from './bounded-context.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; import type { RpcClient, GetBridgeInfoOptions } from './rpc-client.js'; import type { TransportCapabilities } from './transport.js'; @@ -44,7 +45,7 @@ export abstract class BasePythonBridge extends DisposableBase implements PythonR functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { await this.ensureReady(); return this.getRpcClient().call(module, functionName, args, kwargs, validate); diff --git a/src/runtime/bridge-codec.ts b/src/runtime/bridge-codec.ts index 2edc44bf..f0fc56f8 100644 --- a/src/runtime/bridge-codec.ts +++ b/src/runtime/bridge-codec.ts @@ -16,6 +16,7 @@ import { ScientificDecodeError, } from '../utils/codec.js'; import { PROTOCOL_ID } from './transport.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; // ═══════════════════════════════════════════════════════════════════════════ // TYPES @@ -662,13 +663,13 @@ export class BridgeCodec { * @throws BridgeProtocolError if the RPC response is invalid * @throws BridgeExecutionError if response contains a Python error */ - async decodeResponseAsync(payload: string): Promise { + async decodeResponseAsync(payload: string, provenance?: DecodedProvenance): Promise { const result = this.parseResponseResult(payload); // Decode scientific envelopes in the result. let decoded: unknown; try { - decoded = await decodeScientificValue(result); + decoded = await decodeScientificValue(result, provenance); } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); const decodeError = err instanceof ScientificDecodeError ? err : undefined; diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts new file mode 100644 index 00000000..945936af --- /dev/null +++ b/src/runtime/decoded-provenance.ts @@ -0,0 +1,72 @@ +import type { ScientificMarker } from '../utils/codec.js'; + +/** Metadata from one successfully decoded scientific envelope. */ +export interface DecodedShapeMetadata { + readonly marker: ScientificMarker; + readonly dims?: number; + readonly dtype?: string; +} + +/** Scalar provenance stays with one decoded call, including nested values. */ +export class DecodedProvenance { + private rootScalar?: DecodedShapeMetadata; + private readonly children = new WeakMap>(); + private entries = 0; + private readonly maxEntries: number; + + constructor(maxEntries = 1_000_000) { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + throw new Error('Decoded provenance entry limit must be a positive safe integer'); + } + this.maxEntries = maxEntries; + } + + recordRoot(metadata: DecodedShapeMetadata): void { + if (this.rootScalar !== undefined) { + throw new Error('Duplicate root scalar provenance'); + } + this.claimEntry(); + this.rootScalar = Object.freeze({ ...metadata }); + } + + recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + key = this.normalizeKey(parent, key); + let entries = this.children.get(parent); + if (!entries) { + entries = new Map(); + this.children.set(parent, entries); + } + if (entries.has(key)) { + throw new Error('Duplicate child scalar provenance'); + } + this.claimEntry(); + entries.set(key, Object.freeze({ ...metadata })); + } + + atRoot(): DecodedShapeMetadata | undefined { + return this.rootScalar; + } + + atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { + return this.children.get(parent)?.get(this.normalizeKey(parent, key)); + } + + private normalizeKey(parent: object, key: string | number): string | number { + if ( + Array.isArray(parent) && + typeof key === 'string' && + /^(0|[1-9]\d*)$/.test(key) && + Number.isSafeInteger(Number(key)) + ) { + return Number(key); + } + return key; + } + + private claimEntry(): void { + if (this.entries >= this.maxEntries) { + throw new Error(`Decoded scalar provenance exceeds ${this.maxEntries} entries`); + } + this.entries += 1; + } +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index c7f335ad..e19a87a7 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -13,7 +13,10 @@ import type { RuntimeExecution } from '../types/index.js'; export { BridgeCodec, type CodecOptions } from './bridge-codec.js'; export { createReturnValidator, + selectOverloadReturnValidator, describeReceivedShape, + type OverloadParameterSchema, + type OverloadReturnSchema, type ReturnSchema, type ReturnValidator, } from './validators.js'; diff --git a/src/runtime/pyodide-bootstrap-core.generated.ts b/src/runtime/pyodide-bootstrap-core.generated.ts index 68bb32c9..59dc856f 100644 --- a/src/runtime/pyodide-bootstrap-core.generated.ts +++ b/src/runtime/pyodide-bootstrap-core.generated.ts @@ -9,4 +9,4 @@ * Regenerate with: node scripts/generate-pyodide-bootstrap.mjs */ -export const PYODIDE_BRIDGE_CORE_SOURCE: string = "\"\"\"\nShared tywrap bridge core: protocol dispatch + value (de)serialization.\n\nThis module is the SINGLE source of truth for the \"tywrap/1\" server-side\nprotocol. It is imported by:\n\n - runtime/python_bridge.py (the Node/Bun/Deno subprocess server and the HTTP\n server), which owns I/O concerns: the stdin/stdout JSONL loop, env-var size\n guards, the real OS pid, bridge='python-subprocess', and the final BridgeCodec\n encode wrapper.\n\n - the in-WASM Pyodide server (src/runtime/pyodide-transport.ts). Pyodide cannot read\n this file from disk, so it is shipped as a build-time-generated TypeScript\n string constant (src/runtime/pyodide-bootstrap-core.generated.ts) produced by\n scripts/generate-pyodide-bootstrap.mjs and exec'd into a module registered in\n sys.modules. A conformance drift guard (test/runtime_conformance.test.ts)\n asserts the generated constant stays byte-identical to this file.\n\nCROSS-LANGUAGE CONTRACT (Python <-> the TypeScript decoder in src/utils/codec.ts\nand the request encoder in src/runtime/bridge-codec.ts):\n\n * Every value-type \"marker\" envelope carries {'__tywrap__': ,\n 'codecVersion': 1, 'encoding': ...}. The 6 markers are: ndarray, dataframe,\n series, scipy.sparse, torch.tensor, sklearn.estimator.\n * bytes round-trip both ways via base64 envelopes (see _deserialize_bytes_*\n and the bytes branch of default_encoder).\n * NaN/Infinity are rejected (the JS client cannot parse the non-standard tokens\n that allow_nan=True would emit).\n\nPURITY: This module depends only on the standard library plus LAZY optional\nimports (numpy/pandas/scipy/torch/sklearn/pyarrow are each imported inside the\nfunction that needs them). It performs no stdin/stdout I/O and reads no env vars,\nso it runs unchanged under CPython-in-WASM (Pyodide).\n\nforce_json_markers: a *parameter* threaded through every serializer (including\nthe nested torch.tensor -> ndarray call). When True, ndarray/dataframe/series are\nforced down their JSON path regardless of pyarrow availability. Pyodide passes\nTrue (Arrow is unavailable in WASM); the subprocess server passes the boolean\nderived from TYWRAP_CODEC_FALLBACK=json so that \"Node in json-fallback mode\" and\n\"Pyodide\" produce byte-identical marker envelopes.\n\"\"\"\n\nimport base64\nimport datetime as dt\nimport decimal\nimport importlib\nimport importlib.util\nimport json\nimport math\nimport re\nimport sys\nimport traceback\nimport uuid\nfrom pathlib import Path, PurePath\n\n# Protocol constants. These MUST match src/runtime/protocol.ts (PROTOCOL_ID,\n# TYWRAP_PROTOCOL_VERSION) and the codec version baked into marker envelopes.\nPROTOCOL = 'tywrap/1'\nPROTOCOL_VERSION = 1\nCODEC_VERSION = 1\nMAX_SERIALIZE_DEPTH = 900\nMAX_SERIALIZE_NODES = 1_000_000\nJS_SAFE_INTEGER_MAX = 2**53 - 1\n_SERIALIZE_PATH_IDENTIFIER = re.compile(r'^[A-Za-z_$][\\w$]*$', re.ASCII)\n\n\nclass ProtocolError(Exception):\n \"\"\"Raised for malformed requests (bad protocol/id/method/params).\"\"\"\n\n\nclass ImportNotAllowedError(PermissionError):\n \"\"\"Raised when a requested module import is not on the active allowlist.\"\"\"\n\n def __init__(self, module_name):\n super().__init__(\n f'Import of module {module_name!r} is not permitted by the tywrap bridge '\n 'allowlist; add it to TYWRAP_ALLOWED_MODULES (subprocess) or the '\n 'allowed_modules parameter to enable it'\n )\n\n\nclass AttributeNotAllowedError(PermissionError):\n \"\"\"Raised when access to a private/dunder attribute is denied by policy.\"\"\"\n\n def __init__(self, attr_name):\n super().__init__(\n f'Access to attribute {attr_name!r} is not permitted by the tywrap bridge: '\n 'underscore-prefixed (private/dunder) attributes are blocked to prevent '\n 'sandbox-escape via attributes like __globals__/__subclasses__/__builtins__; '\n 'set TYWRAP_ALLOW_PRIVATE_ATTRS=1 (subprocess) or pass allow_private_attrs=True '\n 'to override'\n )\n\n\n# =============================================================================\n# IMPORT / ATTRIBUTE ALLOWLIST (trust boundary enforcement)\n# =============================================================================\n#\n# The bridge dispatches call requests by importing the requested module and\n# getattr-ing the requested function/class method. That is an\n# arbitrary import+getattr+call surface, so two complementary guards live here.\n# Both are PURE (no env reads) so the rules behave identically under the\n# subprocess server and the in-WASM Pyodide server; the subprocess server derives\n# the parameters from env vars (TYWRAP_ALLOWED_MODULES / TYWRAP_ALLOW_PRIVATE_ATTRS)\n# and threads them in, exactly like force_json_markers / torch_allow_copy.\n#\n# 1. MODULE ALLOWLIST (opt-in, default = allow all):\n# allowed_modules=None means \"no restriction\" so existing configurations keep\n# working unchanged. When a caller supplies a set, only those modules (plus the\n# stdlib the bridge itself needs to serialize results, see _BRIDGE_REQUIRED_MODULES)\n# may be imported; submodules of an allowed module are permitted (e.g. allowing\n# 'scipy' also allows 'scipy.sparse'). A non-allowlisted import fails LOUDLY with\n# ImportNotAllowedError rather than silently importing.\n#\n# 2. PRIVATE-ATTRIBUTE BLOCK (default ON):\n# getattr of any name starting with '_' (single-underscore private OR dunder) is\n# rejected. This blocks the classic escape chain (obj.__class__.__subclasses__()\n# /__globals__/__builtins__/__import__) without depending on the module allowlist.\n# tywrap-generated wrappers never reference underscore-prefixed names (the IR\n# analyzer skips them), so this does not regress generated code. Set\n# allow_private_attrs=True to restore unrestricted getattr for trusted callers.\n\n# Stdlib modules the bridge's own serialization/handlers may need to import even\n# when a caller-supplied allowlist is active. Optional codec deps (numpy, pandas,\n# scipy, torch, sklearn, pyarrow) are intentionally NOT here: if a caller restricts\n# modules, they must opt those in explicitly. These names cover only what the\n# bridge core itself imports.\n_BRIDGE_REQUIRED_MODULES = frozenset(\n {\n 'base64',\n 'datetime',\n 'decimal',\n 'importlib',\n 'json',\n 'math',\n 'sys',\n 'traceback',\n 'uuid',\n 'pathlib',\n }\n)\n\n\ndef _top_level_package(module_name):\n \"\"\"Return the top-level package of a dotted module name ('a.b.c' -> 'a').\"\"\"\n return module_name.split('.', 1)[0]\n\n\ndef _is_module_allowed(module_name, allowed_modules):\n \"\"\"\n Return True when module_name may be imported under the active policy.\n\n allowed_modules=None disables enforcement (allow all). Otherwise a module is\n allowed when it (or its top-level package) is explicitly listed, or it is one\n of the stdlib modules the bridge itself requires.\n \"\"\"\n if allowed_modules is None:\n return True\n if module_name in allowed_modules or module_name in _BRIDGE_REQUIRED_MODULES:\n return True\n top = _top_level_package(module_name)\n return top in allowed_modules or top in _BRIDGE_REQUIRED_MODULES\n\n\ndef import_allowed_module(module_name, allowed_modules):\n \"\"\"\n Import module_name only if permitted by the allowlist, else raise loudly.\n\n This is the single chokepoint every handler routes module imports through.\n \"\"\"\n if not _is_module_allowed(module_name, allowed_modules):\n raise ImportNotAllowedError(module_name)\n return importlib.import_module(module_name)\n\n\ndef get_allowed_attr(obj, attr_name, *, allow_private_attrs):\n \"\"\"\n getattr(obj, attr_name) with the private/dunder block applied.\n\n Rejects any underscore-prefixed name unless allow_private_attrs is True. This\n is the single chokepoint every handler routes attribute access through.\n \"\"\"\n if not allow_private_attrs and attr_name.startswith('_'):\n raise AttributeNotAllowedError(attr_name)\n return getattr(obj, attr_name)\n\n\ndef resolve_allowed_attr_path(root, dotted_name, *, allow_private_attrs):\n \"\"\"\n Resolve a possibly-dotted attribute path from root, applying the\n private/dunder getattr guard to EVERY segment.\n\n A single segment (the common case, e.g. a module-level function) behaves\n exactly like get_allowed_attr. Dotted names exist because @classmethod and\n @staticmethod are invoked through their owning class: the generated wrapper\n emits call(module, 'Class.method', ...), so the bridge must walk\n module -> Class -> method. Guarding each segment means 'Class._secret' or\n '_Hidden.method' are rejected exactly as a direct private getattr would be —\n the dotted path opens no access the single-getattr path did not already.\n \"\"\"\n obj = root\n for segment in dotted_name.split('.'):\n obj = get_allowed_attr(obj, segment, allow_private_attrs=allow_private_attrs)\n return obj\n\n\nclass CodecError(Exception):\n \"\"\"Raised when value encoding fails (e.g. NaN/Infinity not allowed).\"\"\"\n\n\n# =============================================================================\n# REQUEST-SIDE DESERIALIZATION (bytes envelopes -> Python bytes)\n# =============================================================================\n\n_NO_DESERIALIZE = object()\n_ERR_BYTES_MISSING_B64 = 'Invalid bytes envelope: missing b64'\n_ERR_BYTES_MISSING_DATA = 'Invalid bytes envelope: missing data'\n_ERR_BYTES_INVALID_BASE64 = 'Invalid bytes envelope: invalid base64'\n\n\ndef _deserialize_bytes_envelope(value):\n \"\"\"\n Decode base64-encoded bytes envelopes from JS into Python bytes.\n\n Supported shapes:\n - { \"__tywrap_bytes__\": true, \"b64\": \"...\" } (JS BridgeCodec.encodeRequest)\n - { \"__type__\": \"bytes\", \"encoding\": \"base64\", \"data\": \"...\" } (legacy/compat)\n\n Why: TS BridgeCodec encodes Uint8Array/ArrayBuffer as base64 objects, but\n Python handlers expect real bytes/bytearray to preserve behavior (e.g., len()).\n \"\"\"\n if not isinstance(value, dict):\n return _NO_DESERIALIZE\n\n if value.get('__tywrap_bytes__') is True:\n b64 = value.get('b64')\n if not isinstance(b64, str):\n raise ProtocolError(_ERR_BYTES_MISSING_B64)\n try:\n return base64.b64decode(b64, validate=True)\n except Exception as exc:\n raise ProtocolError(_ERR_BYTES_INVALID_BASE64) from exc\n\n if value.get('__type__') == 'bytes' and value.get('encoding') == 'base64':\n data = value.get('data')\n if not isinstance(data, str):\n raise ProtocolError(_ERR_BYTES_MISSING_DATA)\n try:\n return base64.b64decode(data, validate=True)\n except Exception as exc:\n raise ProtocolError(_ERR_BYTES_INVALID_BASE64) from exc\n\n return _NO_DESERIALIZE\n\n\ndef deserialize(value, *, has_envelope_markers=True):\n \"\"\"\n Recursively deserialize request values into Python-native types.\n\n Why: requests are JSON-only; we need a small set of explicit decoders\n (currently bytes) to restore Python semantics at the boundary.\n \"\"\"\n if not has_envelope_markers:\n return value\n decoded = _deserialize_bytes_envelope(value)\n if decoded is not _NO_DESERIALIZE:\n return decoded\n\n if isinstance(value, list):\n return [deserialize(item, has_envelope_markers=True) for item in value]\n if isinstance(value, dict):\n # Preserve dict shape while decoding nested values.\n return {k: deserialize(v, has_envelope_markers=True) for k, v in value.items()}\n return value\n\n\n# =============================================================================\n# CAPABILITY DETECTION (lazy, best-effort)\n# =============================================================================\n\ndef arrow_available():\n \"\"\"Return True when pyarrow can be imported.\"\"\"\n try:\n import pyarrow # noqa: F401\n except (ImportError, OSError):\n return False\n return True\n\n\ndef module_available(module_name):\n \"\"\"\n Lightweight feature detection for optional codec dependencies via find_spec.\n\n Why: exposes availability in bridge metadata without importing heavy modules.\n \"\"\"\n try:\n return importlib.util.find_spec(module_name) is not None\n except (ImportError, AttributeError, TypeError, ValueError):\n return False\n\n\ndef is_numpy_array(obj):\n \"\"\"Return whether obj is an ndarray without importing NumPy.\"\"\"\n np = sys.modules.get('numpy')\n if np is None:\n return False\n return isinstance(obj, np.ndarray)\n\n\ndef is_pandas_dataframe(obj):\n \"\"\"Return whether obj is a DataFrame without importing pandas.\"\"\"\n pd = sys.modules.get('pandas')\n if pd is None:\n return False\n return isinstance(obj, pd.DataFrame)\n\n\ndef is_pandas_series(obj):\n \"\"\"Return whether obj is a Series without importing pandas.\"\"\"\n pd = sys.modules.get('pandas')\n if pd is None:\n return False\n return isinstance(obj, pd.Series)\n\n\ndef is_scipy_sparse(obj):\n \"\"\"Return whether obj is sparse without importing SciPy.\"\"\"\n sp = sys.modules.get('scipy.sparse')\n if sp is None:\n return False\n try:\n return sp.issparse(obj)\n except Exception:\n return False\n\n\ndef is_torch_tensor(obj):\n \"\"\"Return whether obj is a Tensor without importing PyTorch.\"\"\"\n torch = sys.modules.get('torch')\n if torch is None:\n return False\n try:\n return torch.is_tensor(obj)\n except Exception:\n return False\n\n\ndef is_sklearn_estimator(obj):\n \"\"\"Return whether obj is an estimator without importing scikit-learn.\"\"\"\n sklearn_base = sys.modules.get('sklearn.base')\n if sklearn_base is None:\n return False\n return isinstance(obj, sklearn_base.BaseEstimator)\n\n\n# =============================================================================\n# MARKER SERIALIZERS (6 __tywrap__ value types)\n# =============================================================================\n#\n# Each serializer accepts force_json_markers. When True, the Arrow path is never\n# taken (used by Pyodide and by the subprocess server in TYWRAP_CODEC_FALLBACK=json\n# mode). The JSON fallback envelopes are byte-identical across both callers, which\n# is what the conformance suite asserts.\n\ndef serialize_ndarray(obj, *, force_json_markers):\n \"\"\"\n Encode a NumPy ndarray. Arrow IPC (compact, lossless) by default; JSON when\n force_json_markers is set or pyarrow is unavailable in fallback mode.\n\n Note: pa.array() only handles 1D arrays; multi-dimensional arrays are\n flattened with shape metadata for JS-side reconstruction. See\n https://github.com/apache/arrow-js/issues/115\n \"\"\"\n if force_json_markers:\n return serialize_ndarray_json(obj)\n try:\n import pyarrow as pa # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for ndarray; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n original_shape = list(obj.shape) if hasattr(obj, 'shape') else None\n flat = obj.flatten() if hasattr(obj, 'ndim') and obj.ndim != 1 else obj\n arr = pa.array(flat)\n table = pa.Table.from_arrays([arr], names=['value'])\n sink = pa.BufferOutputStream()\n with pa.ipc.new_stream(sink, table.schema) as writer:\n writer.write_table(table)\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'ndarray',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n 'shape': original_shape,\n 'dtype': str(obj.dtype) if hasattr(obj, 'dtype') else None,\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for ndarray') from exc\n\n\ndef serialize_ndarray_json(obj):\n \"\"\"JSON fallback for ndarray values that JavaScript can represent safely.\"\"\"\n dtype = obj.dtype\n dtype_label = str(dtype)\n\n if not dtype.isnative:\n raise RuntimeError(\n f'JSON ndarray encoding does not support big-endian dtype={dtype_label}; '\n \"convert to native byte order with a.byteswap().view(a.dtype.newbyteorder('='))\"\n )\n\n if dtype.kind in ('i', 'u'):\n # JSON numbers become JavaScript Number values. Scan only on this explicit\n # fallback path and reject before tolist() can silently round an integer.\n if obj.size and (\n (obj < -JS_SAFE_INTEGER_MAX).any() or (obj > JS_SAFE_INTEGER_MAX).any()\n ):\n raise RuntimeError(\n f'JSON ndarray encoding cannot safely represent dtype={dtype_label} values '\n 'outside the JavaScript safe integer range; use Arrow encoding or '\n \"cast/encode explicitly (e.g. .astype('float64') or str)\"\n )\n elif dtype.kind == 'M':\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '\n \"or convert explicitly (e.g. .astype('datetime64[ms]').astype(str) or int with \"\n 'declared unit)'\n )\n elif dtype.kind == 'm':\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '\n \"or convert explicitly (e.g. .astype('timedelta64[ms]').astype(str) or int with \"\n 'declared unit)'\n )\n elif dtype.kind == 'V':\n if dtype.fields is not None:\n raise RuntimeError(\n f'JSON ndarray encoding does not support structured dtype={dtype_label}; '\n 'encode each named field explicitly (e.g. as a plain JSON object)'\n )\n raise RuntimeError(\n f'JSON ndarray encoding does not support void dtype={dtype_label}; convert the '\n \"raw bytes explicitly (e.g. .view('uint8'))\"\n )\n elif dtype.kind == 'O':\n raise RuntimeError(\n f'JSON ndarray encoding does not support object dtype={dtype_label}; cast to a '\n 'concrete numeric dtype or encode elements explicitly as plain JSON'\n )\n elif dtype.kind == 'S':\n raise RuntimeError(\n f'JSON ndarray encoding does not support byte-string dtype={dtype_label}; '\n 'decode elements and return a plain JSON list explicitly'\n )\n elif dtype.kind == 'U':\n raise RuntimeError(\n f'JSON ndarray encoding does not support unicode dtype={dtype_label}; convert '\n 'explicitly to plain JSON strings with .tolist()'\n )\n elif dtype.kind == 'c':\n raise RuntimeError(\n f'JSON ndarray encoding does not support complex dtype={dtype_label}; encode '\n '.real and .imag arrays explicitly'\n )\n elif dtype.kind == 'f' and dtype.itemsize > 8:\n raise RuntimeError(\n f'JSON ndarray encoding does not support float dtype={dtype_label} wider than '\n \"64 bits; cast explicitly (e.g. .astype('float64')) or use Arrow encoding\"\n )\n elif dtype.kind not in ('b', 'f'):\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; cast to bool, '\n 'integer, or float dtype, or encode values explicitly as plain JSON'\n )\n\n try:\n data = obj.tolist()\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for ndarray') from exc\n return {\n '__tywrap__': 'ndarray',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n 'shape': getattr(obj, 'shape', None),\n 'dtype': dtype.name,\n }\n\n\ndef serialize_dataframe(obj, *, force_json_markers):\n \"\"\"\n Encode a pandas DataFrame. Feather/Arrow-IPC (uncompressed, so apache-arrow\n in JS can read it) by default; JSON when force_json_markers is set.\n \"\"\"\n if force_json_markers:\n return serialize_dataframe_json(obj)\n try:\n import pyarrow as pa # type: ignore\n import pyarrow.feather as feather # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for pandas.DataFrame; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n table = pa.Table.from_pandas(obj) # type: ignore\n sink = pa.BufferOutputStream()\n feather.write_feather(table, sink, compression='uncompressed')\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'dataframe',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for pandas.DataFrame') from exc\n\n\ndef serialize_dataframe_json(obj):\n \"\"\"JSON fallback for DataFrame values that JavaScript can represent safely.\"\"\"\n import pandas as pd # type: ignore\n\n _validate_pandas_json_index(obj.index, pd, 'DataFrame')\n json_columns = [_json_object_key(column) for column in obj.columns]\n supported_json_columns = [column for column in json_columns if column is not None]\n if not obj.columns.is_unique or len(set(supported_json_columns)) != len(\n supported_json_columns\n ):\n raise RuntimeError(\n 'JSON pandas.DataFrame encoding requires column labels to remain unique after '\n 'JSON object-key coercion; rename columns or make them distinct before applying '\n '.columns.astype(str)'\n )\n for column, dtype in obj.dtypes.items():\n if isinstance(dtype, pd.CategoricalDtype):\n raise RuntimeError(\n f'JSON pandas.DataFrame encoding does not support categorical dtype in '\n f'column {column!r}; use Arrow encoding or convert explicitly '\n \"(e.g. .astype(str))\"\n )\n try:\n data = (\n [{} for _ in range(len(obj.index))]\n if len(obj.columns) == 0\n else obj.to_dict(orient='records')\n )\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for pandas.DataFrame') from exc\n for row_number, row in enumerate(data):\n for column, value in row.items():\n row[column] = _normalize_pandas_json_scalar(\n value,\n f'DataFrame cell at row {row_number}, column {column!r}',\n pd,\n )\n return {\n '__tywrap__': 'dataframe',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n }\n\n\ndef serialize_series(obj, *, force_json_markers):\n \"\"\"\n Encode a pandas Series as a single-column Arrow Table stream (the JS decoder\n contract is \"table-like\"); JSON when force_json_markers is set.\n \"\"\"\n if force_json_markers:\n return serialize_series_json(obj)\n try:\n import pyarrow as pa # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for pandas.Series; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n arr = pa.Array.from_pandas(obj) # type: ignore\n table = pa.Table.from_arrays([arr], names=['value'])\n sink = pa.BufferOutputStream()\n with pa.ipc.new_stream(sink, table.schema) as writer:\n writer.write_table(table)\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'series',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n 'name': getattr(obj, 'name', None),\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for pandas.Series') from exc\n\n\ndef serialize_series_json(obj):\n \"\"\"JSON fallback for Series values that JavaScript can represent safely.\"\"\"\n import pandas as pd # type: ignore\n\n _validate_pandas_json_index(obj.index, pd, 'Series')\n if isinstance(obj.dtype, pd.CategoricalDtype):\n raise RuntimeError(\n 'JSON pandas.Series encoding does not support categorical dtype; use Arrow '\n \"encoding or convert explicitly (e.g. .astype(str))\"\n )\n try:\n data = obj.to_list() # type: ignore\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for pandas.Series') from exc\n data = [\n _normalize_pandas_json_scalar(value, f'Series value at position {position}', pd)\n for position, value in enumerate(data)\n ]\n return {\n '__tywrap__': 'series',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n 'name': getattr(obj, 'name', None),\n }\n\n\ndef _validate_pandas_json_index(index, pd, container_name):\n \"\"\"Reject index metadata that records/list-oriented JSON would discard.\"\"\"\n if isinstance(index, pd.MultiIndex):\n raise RuntimeError(\n f'JSON pandas.{container_name} encoding does not support MultiIndex; use Arrow '\n 'encoding or flatten the index explicitly with .reset_index()'\n )\n if not (\n isinstance(index, pd.RangeIndex)\n and index.start == 0\n and index.step == 1\n and index.stop == len(index)\n and index.name is None\n ):\n raise RuntimeError(\n f'JSON pandas.{container_name} encoding requires an unnamed RangeIndex starting '\n 'at 0 with step 1; use Arrow encoding or normalize explicitly with '\n '.reset_index(drop=True)'\n )\n\n\ndef _json_object_key(value):\n \"\"\"Return json.dumps' object-key spelling, or None for unsupported keys.\"\"\"\n try:\n encoded = json.dumps({value: None})\n except (TypeError, ValueError):\n return None\n return next(iter(json.loads(encoded)))\n\n\ndef _normalize_pandas_json_scalar(value, location, pd):\n \"\"\"Normalize pandas nulls and reject values outside the plain JSON domain.\"\"\"\n np = sys.modules.get('numpy')\n if np is not None and isinstance(value, np.generic):\n value = value.item()\n if value is None or value is pd.NA or value is pd.NaT:\n return None\n if type(value) is bool:\n return value\n if type(value) is int:\n if value < -JS_SAFE_INTEGER_MAX or value > JS_SAFE_INTEGER_MAX:\n raise RuntimeError(\n f'JSON pandas encoding cannot safely represent {location} integer values '\n 'outside the JavaScript safe integer range; use Arrow encoding or '\n \"cast/encode explicitly (e.g. .astype('float64') or str)\"\n )\n return value\n if type(value) is float:\n if not math.isfinite(value):\n raise RuntimeError(\n f'JSON pandas encoding cannot represent non-finite {location} float values '\n '(NaN or Infinity); use .fillna(...) for intentional missing values or '\n 'Arrow encoding'\n )\n return value\n if type(value) is str:\n return value\n raise RuntimeError(\n f'JSON pandas encoding does not support {location} value of type '\n f'{type(value).__name__}; use Arrow encoding or convert explicitly '\n '(e.g. .astype(str))'\n )\n\n\ndef serialize_sparse_matrix(obj):\n \"\"\"\n Serialize scipy sparse matrices into structured JSON envelopes (json-only;\n there is no Arrow path). Preserves sparsity; rejects unsupported formats and\n complex dtypes explicitly.\n \"\"\"\n try:\n fmt = obj.getformat()\n except Exception as exc:\n raise RuntimeError('Failed to inspect scipy sparse matrix format') from exc\n\n if fmt not in ('csr', 'csc', 'coo'):\n raise RuntimeError(\n f'Unsupported scipy sparse format: {fmt}; only csr/csc/coo are supported. '\n 'Convert explicitly (e.g. matrix.tocsr()) before returning'\n )\n\n dtype = None\n try:\n dtype = str(obj.dtype)\n except Exception:\n dtype = None\n if getattr(obj.dtype, 'kind', None) == 'c':\n raise RuntimeError(\n 'Complex scipy sparse matrices are not supported by the JSON codec; '\n 'split into real/imag components explicitly before returning'\n )\n\n if fmt in ('csr', 'csc'):\n data = obj.data.tolist()\n indices = obj.indices.tolist()\n indptr = obj.indptr.tolist()\n return {\n '__tywrap__': 'scipy.sparse',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'format': fmt,\n 'shape': list(obj.shape),\n 'data': data,\n 'indices': indices,\n 'indptr': indptr,\n 'dtype': dtype,\n }\n\n # coo\n data = obj.data.tolist()\n row = obj.row.tolist()\n col = obj.col.tolist()\n return {\n '__tywrap__': 'scipy.sparse',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'format': fmt,\n 'shape': list(obj.shape),\n 'data': data,\n 'row': row,\n 'col': col,\n 'dtype': dtype,\n }\n\n\ndef serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):\n \"\"\"\n Serialize torch.Tensor values via the nested ndarray envelope. CPU-only by\n default; device/copy behavior is explicit. force_json_markers is threaded\n into the nested ndarray serialization so Pyodide gets a JSON ndarray value.\n\n Rejection order is significant: the categorical rejections (sparse / quantized\n / meta / complex) are checked BEFORE the device/contiguous opt-in branch so\n they fail with a clear, specific message and are NOT bypassable by\n TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the device transfer and\n contiguous copy, never an unrepresentable layout/dtype.\n \"\"\"\n import torch # already importable: is_torch_tensor() gated the dispatch\n\n tensor = obj.detach()\n source_device = None\n source_dtype = None\n\n # Sparse tensors (COO/CSR/CSC/BSR/BSC -> any non-strided layout) have no dense\n # numpy representation without a densify step, which is not the round-trip this\n # envelope promises. Reject explicitly rather than emitting a misleading\n # \"not contiguous\" error or silently densifying.\n layout = getattr(tensor, 'layout', None)\n if getattr(tensor, 'is_sparse', False) or (\n layout is not None and layout != torch.strided\n ):\n raise RuntimeError(\n f'Torch sparse tensors are not supported (layout={layout}); '\n 'convert to a dense CPU tensor explicitly (e.g. tensor.to_dense()) before returning'\n )\n\n # Quantized tensors carry a qscheme/scale/zero_point that numpy() cannot\n # represent; .numpy() raises an opaque \"unsupported ScalarType\" deep in torch.\n # Reject up front with an actionable message.\n if getattr(tensor, 'is_quantized', False):\n raise RuntimeError(\n 'Torch quantized tensors are not supported; dequantize explicitly '\n '(e.g. tensor.dequantize()) before returning'\n )\n\n # Meta tensors have shape/dtype but NO storage; copying to CPU yields garbage,\n # so this is never a lossy-but-honest transfer the opt-in could authorize.\n if getattr(tensor, 'is_meta', False) or (\n getattr(tensor, 'device', None) is not None and tensor.device.type == 'meta'\n ):\n raise RuntimeError(\n 'Torch meta tensors carry no data and cannot be serialized; '\n 'materialize the tensor on a real device before returning'\n )\n\n # Complex tensors round-trip to numpy complex arrays, which are not\n # JSON-serializable and have no codec envelope. Reject explicitly instead of\n # emitting Python complex tuples that the JS decoder cannot parse.\n if torch.is_complex(tensor):\n raise RuntimeError(\n f'Torch complex tensors are not supported (dtype={tensor.dtype}); '\n 'split into real/imag components explicitly before returning'\n )\n\n if getattr(tensor, 'device', None) is not None and tensor.device.type != 'cpu':\n if not torch_allow_copy:\n raise RuntimeError(\n 'Torch tensor is on a non-CPU device; set TYWRAP_TORCH_ALLOW_COPY=1 to allow CPU transfer'\n )\n source_device = str(tensor.device)\n tensor = tensor.to('cpu')\n if hasattr(tensor, 'is_contiguous') and not tensor.is_contiguous():\n if not torch_allow_copy:\n raise RuntimeError(\n 'Torch tensor is not contiguous; set TYWRAP_TORCH_ALLOW_COPY=1 to allow contiguous copy'\n )\n tensor = tensor.contiguous()\n if tensor.dtype == torch.bfloat16:\n source_dtype = str(tensor.dtype)\n tensor = tensor.float()\n try:\n arr = tensor.numpy()\n except Exception as exc:\n raise RuntimeError('Failed to convert torch.Tensor to numpy') from exc\n\n envelope = {\n '__tywrap__': 'torch.tensor',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'ndarray',\n 'value': serialize_ndarray(arr, force_json_markers=force_json_markers),\n 'shape': list(tensor.shape),\n 'dtype': str(tensor.dtype),\n 'device': str(tensor.device),\n }\n if source_dtype is not None:\n envelope['sourceDtype'] = source_dtype\n if source_device is not None:\n envelope['sourceDevice'] = source_device\n return envelope\n\n\ndef serialize_sklearn_estimator(obj):\n \"\"\"Serialize sklearn estimators as metadata only (json-only); no pickling.\"\"\"\n try:\n import sklearn # noqa: F401\n except Exception as exc:\n raise RuntimeError('scikit-learn is not available') from exc\n\n params = obj.get_params(deep=False)\n\n # Metadata-only: NEVER pickle/joblib. Every param value must be plain JSON\n # (no callables, nested estimators, numpy arrays, or other objects). Probe\n # each value individually so the error names the offending param instead of\n # failing opaquely on the whole dict. allow_nan=False also rejects NaN/Inf\n # params here for parity with the response codec.\n for key, value in params.items():\n try:\n json.dumps(value, allow_nan=False)\n except (TypeError, ValueError) as exc:\n raise RuntimeError(\n f'scikit-learn estimator param {key!r} is not JSON-serializable '\n f'(got {type(value).__name__}); estimators are serialized as metadata only '\n '(no pickle/joblib), so every param must be a plain JSON value. '\n 'Sanitize or drop the param before returning'\n ) from exc\n\n return {\n '__tywrap__': 'sklearn.estimator',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'className': obj.__class__.__name__,\n 'module': obj.__class__.__module__,\n 'version': getattr(sklearn, '__version__', None),\n 'params': params,\n }\n\n\n_NO_PYDANTIC = object()\n\n\ndef serialize_pydantic(obj):\n \"\"\"\n Serialize Pydantic v2 models via model_dump(by_alias=True, mode='json')\n without importing Pydantic. Returns _NO_PYDANTIC when obj is not a model.\n \"\"\"\n model_dump = getattr(obj, 'model_dump', None)\n if not callable(model_dump):\n return _NO_PYDANTIC\n try:\n try:\n return model_dump(by_alias=True, mode='json')\n except TypeError:\n # Older Pydantic versions may not support `mode=...`.\n return model_dump(by_alias=True)\n except Exception as exc:\n raise RuntimeError(f'model_dump failed: {exc}') from exc\n\n\ndef serialize_stdlib(obj):\n \"\"\"Coerce common stdlib scalar types to JSON-safe forms; None otherwise.\"\"\"\n if isinstance(obj, dt.datetime):\n return obj.isoformat()\n if isinstance(obj, dt.date):\n return obj.isoformat()\n if isinstance(obj, dt.time):\n return obj.isoformat()\n if isinstance(obj, dt.timedelta):\n return obj.total_seconds()\n if isinstance(obj, decimal.Decimal):\n return str(obj)\n if isinstance(obj, uuid.UUID):\n return str(obj)\n if isinstance(obj, (Path, PurePath)):\n return str(obj)\n return None\n\n\n_NO_SCIENTIFIC = object()\n\n\ndef _check_serialize_depth(depth, path):\n if depth > MAX_SERIALIZE_DEPTH:\n raise RuntimeError(\n f'Scientific envelope serialization maximum depth '\n f'{MAX_SERIALIZE_DEPTH} exceeded at {path}'\n )\n\n\ndef _check_serialize_nodes(nodes, path):\n if nodes > MAX_SERIALIZE_NODES:\n raise RuntimeError(\n f'Scientific envelope serialization maximum visited nodes '\n f'{MAX_SERIALIZE_NODES} exceeded at {path}'\n )\n\n\ndef _serialize_scientific(obj, *, force_json_markers, torch_allow_copy, depth, path):\n \"\"\"Serialize a supported scientific value, or return _NO_SCIENTIFIC.\"\"\"\n package = type(obj).__module__.split('.', 1)[0]\n\n if package == 'numpy' and 'numpy' in sys.modules:\n if is_numpy_array(obj):\n _check_serialize_depth(depth, path)\n return serialize_ndarray(obj, force_json_markers=force_json_markers)\n elif package == 'pandas' and 'pandas' in sys.modules:\n if is_pandas_dataframe(obj):\n _check_serialize_depth(depth, path)\n return serialize_dataframe(obj, force_json_markers=force_json_markers)\n if is_pandas_series(obj):\n _check_serialize_depth(depth, path)\n return serialize_series(obj, force_json_markers=force_json_markers)\n elif package == 'scipy' and 'scipy.sparse' in sys.modules:\n if is_scipy_sparse(obj):\n _check_serialize_depth(depth, path)\n return serialize_sparse_matrix(obj)\n elif package == 'torch' and 'torch' in sys.modules:\n if is_torch_tensor(obj):\n _check_serialize_depth(depth, path)\n _check_serialize_depth(depth + 1, _serialize_path(path, 'value'))\n return serialize_torch_tensor(\n obj, force_json_markers=force_json_markers, torch_allow_copy=torch_allow_copy\n )\n elif 'sklearn.base' in sys.modules and is_sklearn_estimator(obj):\n # No package gate here, unlike the branches above: subclassing\n # BaseEstimator is sklearn's documented extension point, so user-defined\n # estimators live outside the 'sklearn' package and must still get the\n # estimator serializer (and its param-naming errors).\n _check_serialize_depth(depth, path)\n return serialize_sklearn_estimator(obj)\n\n return _NO_SCIENTIFIC\n\n\ndef _serialize_path(base, key):\n \"\"\"Build a decoder-compatible JSONPath-like result path.\"\"\"\n if isinstance(key, int):\n return f'{base}[{key}]'\n if _SERIALIZE_PATH_IDENTIFIER.fullmatch(key):\n return f'{base}.{key}'\n return f'{base}[{json.dumps(key, ensure_ascii=False)}]'\n\n\ndef _invalid_key_path(base, key):\n \"\"\"Name a dict key that cannot be represented by JSON.\"\"\"\n return f'{base}[{key!r}]'\n\n\ndef _needs_serialize_visit(value):\n \"\"\"Return whether value needs container or scientific traversal work.\"\"\"\n if type(value) in (type(None), bool, int, float, str):\n return False\n if type(value) in (dict, list, tuple):\n return True\n package = type(value).__module__.split('.', 1)[0]\n if package in ('numpy', 'pandas', 'scipy', 'torch'):\n return True\n return 'sklearn.base' in sys.modules and is_sklearn_estimator(value)\n\n\ndef _serialize_leaf(value):\n \"\"\"Apply non-container conversions without allocating a traversal frame.\"\"\"\n if type(value) in (type(None), bool, int, float, str):\n return value\n pydantic_value = serialize_pydantic(value)\n if pydantic_value is not _NO_PYDANTIC:\n return pydantic_value\n stdlib_value = serialize_stdlib(value)\n if stdlib_value is not None:\n return stdlib_value\n return value\n\n\ndef serialize(obj, *, force_json_markers, torch_allow_copy=False):\n \"\"\"\n Top-level result serializer.\n\n Scientific codecs are type-first and only inspect packages that the value can\n belong to. A value from an optional package implies that package is already in\n sys.modules, so these checks never cold-import the scientific stack. The\n package dispatch deliberately precedes the JSON-native fast path: e.g. a\n package-defined subclass of dict still receives its relevant codec check.\n Every other value is left untouched so the shared JSON encoder applies the\n exact same default conversion at the root and at any nested depth.\n \"\"\"\n root = [None]\n active_ids = set()\n stack = [('visit', obj, 0, 'result', root, 0)]\n visited_nodes = 0\n\n # Repeated aliases have value semantics and are intentionally serialized twice.\n while stack:\n frame = stack.pop()\n action = frame[0]\n if action == 'dict':\n _, current, depth, path, parent, key, output, iterator = frame\n try:\n item_key, item = next(iterator)\n except StopIteration:\n active_ids.remove(id(current))\n parent[key] = output\n continue\n stack.append(frame)\n if not (isinstance(item_key, (str, int, float, bool)) or item_key is None):\n invalid_path = _invalid_key_path(path, item_key)\n raise TypeError(\n f'keys must be str, int, float, bool or None, not '\n f'{type(item_key).__name__} at {invalid_path}'\n )\n child_key = _json_object_key(item_key)\n child_path = _serialize_path(path, child_key)\n if _needs_serialize_visit(item):\n stack.append(('visit', item, depth + 1, child_path, output, item_key))\n else:\n output[item_key] = _serialize_leaf(item)\n continue\n if action == 'sequence':\n _, current, depth, path, parent, key, output, index = frame\n if index == len(output):\n active_ids.remove(id(current))\n parent[key] = output if type(current) is list else tuple(output)\n continue\n stack.append(('sequence', current, depth, path, parent, key, output, index + 1))\n item = current[index]\n if _needs_serialize_visit(item):\n stack.append(\n ('visit', item, depth + 1, _serialize_path(path, index), output, index)\n )\n else:\n output[index] = _serialize_leaf(item)\n continue\n\n _, current, depth, path, parent, key = frame\n try:\n scientific = _serialize_scientific(\n current,\n force_json_markers=force_json_markers,\n torch_allow_copy=torch_allow_copy,\n depth=depth,\n path=path,\n )\n except Exception as exc:\n if path == 'result':\n raise\n raise RuntimeError(f'Scientific value serialization failed at {path}: {exc}') from exc\n if scientific is not _NO_SCIENTIFIC:\n # Recognized envelopes are terminal containers to the JS decoder.\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, path)\n if scientific.get('__tywrap__') == 'torch.tensor':\n nested_path = _serialize_path(path, 'value')\n try:\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, nested_path)\n except Exception as exc:\n if path == 'result':\n raise\n raise RuntimeError(\n f'Scientific value serialization failed at {path}: {exc}'\n ) from exc\n parent[key] = scientific\n continue\n\n container_type = type(current)\n if container_type in (dict, list, tuple):\n _check_serialize_depth(depth, path)\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, path)\n current_id = id(current)\n if current_id in active_ids:\n raise RuntimeError(f'Circular reference detected at {path}')\n active_ids.add(current_id)\n\n if container_type is dict:\n output = {}\n parent[key] = output\n stack.append(\n ('dict', current, depth, path, parent, key, output, iter(current.items()))\n )\n continue\n\n output = [None] * len(current)\n if container_type is list:\n parent[key] = output\n stack.append(('sequence', current, depth, path, parent, key, output, 0))\n continue\n\n parent[key] = _serialize_leaf(current)\n\n return root[0]\n\n\n# =============================================================================\n# JSON CODEC: value handling and size-limited encode/decode\n# =============================================================================\n#\n# This is the single Python implementation used by both the subprocess bridge and\n# the embedded Pyodide core. BridgeCodec adds payload-size enforcement around the\n# shared encoder; encode_value is the unbounded form used by the in-memory bridge.\n\ndef _is_nan_or_inf(value):\n if not isinstance(value, (int, float)):\n return False\n try:\n return math.isnan(value) or math.isinf(value)\n except (TypeError, ValueError):\n return False\n\n\ndef _is_numpy_scalar(obj):\n try:\n import numpy as np\n except ImportError:\n return False\n return isinstance(obj, (np.generic, np.ndarray)) and obj.ndim == 0\n\n\ndef _is_pandas_scalar(obj):\n try:\n import pandas as pd\n except ImportError:\n return False\n return isinstance(obj, (pd.Timestamp, pd.Timedelta, type(pd.NaT)))\n\n\ndef make_default_encoder(*, allow_nan):\n \"\"\"\n Build a json.dumps default= encoder matching BridgeCodec's value handling.\n\n Raises CodecError for NaN/Infinity extracted from numpy scalars (json.dumps\n itself rejects top-level/nested NaN/Infinity floats when allow_nan=False).\n \"\"\"\n\n def default_encoder(obj):\n # numpy/pandas scalars first (need .item() extraction).\n if _is_numpy_scalar(obj):\n extracted = obj.item()\n if not allow_nan and _is_nan_or_inf(extracted):\n raise CodecError('Cannot serialize NaN - NaN/Infinity not allowed in JSON')\n return extracted\n\n if _is_pandas_scalar(obj):\n try:\n import pandas as pd\n except ImportError:\n pass\n else:\n if obj is pd.NaT or (hasattr(pd, 'isna') and pd.isna(obj)):\n return None\n if isinstance(obj, pd.Timestamp):\n return obj.isoformat()\n if isinstance(obj, pd.Timedelta):\n return obj.total_seconds()\n\n stdlib_value = serialize_stdlib(obj)\n if stdlib_value is not None:\n return stdlib_value\n\n if isinstance(obj, (bytes, bytearray)):\n return {\n '__type__': 'bytes',\n 'encoding': 'base64',\n 'data': base64.b64encode(obj).decode('ascii'),\n }\n\n model_dump = getattr(obj, 'model_dump', None)\n if callable(model_dump):\n try:\n return model_dump(by_alias=True, mode='json')\n except TypeError:\n return model_dump(by_alias=True)\n\n if isinstance(obj, (set, frozenset)):\n return list(obj)\n\n if isinstance(obj, complex):\n raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')\n\n raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')\n\n return default_encoder\n\n\ndef encode_value(value, *, allow_nan):\n \"\"\"\n JSON-encode a fully-serialized response value, applying the BridgeCodec-equivalent\n default encoder and rejecting NaN/Infinity when allow_nan is False.\n\n Raises CodecError (wrapping the json.dumps ValueError) on NaN/Infinity, matching\n BridgeCodec's \"Cannot serialize NaN...\" wording so error parity holds.\n \"\"\"\n try:\n return json.dumps(value, default=make_default_encoder(allow_nan=allow_nan), allow_nan=allow_nan)\n except ValueError as exc:\n error_msg = str(exc).lower()\n # json.dumps(allow_nan=False) rejects NaN/Infinity with a ValueError whose\n # wording is Python-version dependent: 3.12+ appends the offending value\n # (\"...not JSON compliant: nan\"), but 3.10/3.11 emit only the canonical\n # \"Out of range float values are not JSON compliant\". Match that phrase too\n # so the typed error message is stable across versions.\n nonfinite_token = re.search(r'(^|[^a-z])(nan|inf|infinity)([^a-z]|$)', error_msg)\n if (\n not allow_nan and 'out of range float values are not json compliant' in error_msg\n ) or nonfinite_token:\n raise CodecError('Cannot serialize NaN - NaN/Infinity not allowed in JSON') from exc\n raise CodecError(f'JSON encoding failed: {exc}') from exc\n except TypeError as exc:\n raise CodecError(f'JSON encoding failed: {exc}') from exc\n\n\nclass BridgeCodec:\n \"\"\"Safe JSON codec with explicit value handling and payload-size limits.\"\"\"\n\n def __init__(self, allow_nan=False, max_payload_bytes=10 * 1024 * 1024):\n self.allow_nan = allow_nan\n self.max_payload_bytes = max_payload_bytes\n self._encoder = make_default_encoder(allow_nan=allow_nan)\n\n def encode(self, value):\n result = encode_value(value, allow_nan=self.allow_nan)\n if len(result.encode('utf-8')) > self.max_payload_bytes:\n raise CodecError(f'Payload exceeds {self.max_payload_bytes} bytes')\n return result\n\n def decode(self, payload):\n if len(payload.encode('utf-8')) > self.max_payload_bytes:\n raise CodecError(f'Payload exceeds {self.max_payload_bytes} bytes')\n try:\n return json.loads(payload)\n except json.JSONDecodeError as exc:\n raise CodecError(f'JSON decoding failed: {exc}') from exc\n\n def _default_encoder(self, obj):\n \"\"\"Compatibility hook for callers that used the codec's JSON encoder.\"\"\"\n return self._encoder(obj)\n\n\n_default_codec = None\n\n\ndef get_default_codec():\n \"\"\"Return the lazily-created default BridgeCodec instance.\"\"\"\n global _default_codec\n if _default_codec is None:\n _default_codec = BridgeCodec()\n return _default_codec\n\n\ndef encode(value, *, allow_nan=False):\n \"\"\"Encode a value with the default codec settings.\"\"\"\n if allow_nan:\n return BridgeCodec(allow_nan=True).encode(value)\n return get_default_codec().encode(value)\n\n\ndef decode(payload):\n \"\"\"Decode a JSON payload with the default codec settings.\"\"\"\n return get_default_codec().decode(payload)\n\n\n# =============================================================================\n# REQUEST VALIDATION + HANDLERS + DISPATCH\n# =============================================================================\n\ndef require_protocol(msg):\n if not isinstance(msg, dict):\n raise ProtocolError('Invalid request payload')\n proto = msg.get('protocol')\n if proto != PROTOCOL:\n raise ProtocolError(f'Invalid protocol: {proto}')\n mid = msg.get('id')\n if not isinstance(mid, int):\n raise ProtocolError(f'Invalid request id: {mid}')\n return mid\n\n\ndef require_str(params, key):\n value = params.get(key)\n if not isinstance(value, str) or not value:\n raise ProtocolError(f'Missing {key}')\n return value\n\n\ndef coerce_list(value, key):\n if value is None:\n return []\n if not isinstance(value, list):\n raise ProtocolError(f'Invalid {key}')\n return value\n\n\ndef coerce_dict(value, key):\n if value is None:\n return {}\n if not isinstance(value, dict):\n raise ProtocolError(f'Invalid {key}')\n return value\n\n\ndef handle_call(\n params,\n *,\n force_json_markers,\n torch_allow_copy,\n allowed_modules,\n allow_private_attrs,\n has_envelope_markers,\n):\n module_name = require_str(params, 'module')\n function_name = require_str(params, 'functionName')\n args = deserialize(coerce_list(params.get('args'), 'args'), has_envelope_markers=has_envelope_markers)\n kwargs = deserialize(coerce_dict(params.get('kwargs'), 'kwargs'), has_envelope_markers=has_envelope_markers)\n mod = import_allowed_module(module_name, allowed_modules)\n # function_name may be dotted ('Class.method') for @classmethod/@staticmethod\n # calls, which the generated wrapper routes through call() rather than an\n # instance handle. resolve_allowed_attr_path guards each segment.\n func = resolve_allowed_attr_path(mod, function_name, allow_private_attrs=allow_private_attrs)\n res = func(*args, **kwargs)\n return serialize(res, force_json_markers=force_json_markers, torch_allow_copy=torch_allow_copy)\n\n\ndef build_meta(\n *,\n bridge,\n pid,\n python_version,\n codec_fallback,\n arrow_available_override=None,\n):\n \"\"\"\n Build the bridge metadata payload.\n\n Field order here is part of the wire contract (the JS validator and the\n documented BridgeInfo shape). Callers supply the backend-specific identity:\n the subprocess server passes bridge='python-subprocess' and a real pid; the\n Pyodide server passes bridge='pyodide' and pid=None.\n\n arrow_available_override: when not None, report this value for arrowAvailable\n instead of probing pyarrow. The Pyodide server forces markers to JSON\n unconditionally, so it advertises arrowAvailable=False regardless of whether\n pyarrow happens to be importable in the WASM environment.\n \"\"\"\n arrow = arrow_available() if arrow_available_override is None else arrow_available_override\n meta = {\n 'protocol': PROTOCOL,\n 'protocolVersion': PROTOCOL_VERSION,\n 'bridge': bridge,\n 'pythonVersion': python_version,\n 'pid': pid,\n 'codecFallback': codec_fallback,\n 'arrowAvailable': arrow,\n 'scipyAvailable': module_available('scipy'),\n 'torchAvailable': module_available('torch'),\n 'sklearnAvailable': module_available('sklearn'),\n 'instances': 0,\n }\n return meta\n\n\ndef dispatch_request(\n msg,\n *,\n bridge,\n pid,\n force_json_markers,\n allow_nan=False,\n python_version=None,\n torch_allow_copy=False,\n arrow_available_override=None,\n allowed_modules=None,\n allow_private_attrs=False,\n has_envelope_markers=True,\n):\n \"\"\"\n Validate and route a request, returning the fully-serialized response dict\n ({'id', 'protocol', 'result'}). Raises ProtocolError for malformed requests\n and propagates handler exceptions to the caller, which is responsible for\n building the error envelope (so it controls traceback inclusion).\n\n allow_nan is accepted for signature symmetry; NaN rejection happens during\n the final encode_value() call, which the caller performs.\n\n allowed_modules: None (default) disables the import allowlist so existing\n behavior is preserved. Supplying a set restricts call imports to\n those modules (plus the stdlib the bridge itself needs) and raises\n ImportNotAllowedError otherwise. allow_private_attrs=False (default) blocks\n getattr of underscore-prefixed names; True restores unrestricted access. See\n the IMPORT / ATTRIBUTE ALLOWLIST section above for the full trust model.\n \"\"\"\n mid = require_protocol(msg)\n method = msg.get('method')\n if not isinstance(method, str):\n raise ProtocolError('Missing method')\n params = coerce_dict(msg.get('params'), 'params')\n if method == 'call':\n result = handle_call(\n params,\n force_json_markers=force_json_markers,\n torch_allow_copy=torch_allow_copy,\n allowed_modules=allowed_modules,\n allow_private_attrs=allow_private_attrs,\n has_envelope_markers=has_envelope_markers,\n )\n elif method == 'meta':\n if python_version is None:\n import sys\n python_version = sys.version.split()[0]\n codec_fallback = 'json' if force_json_markers else 'none'\n result = build_meta(\n bridge=bridge,\n pid=pid,\n python_version=python_version,\n codec_fallback=codec_fallback,\n arrow_available_override=arrow_available_override,\n )\n else:\n raise ProtocolError(f'Unknown method: {method}')\n return {'id': mid, 'protocol': PROTOCOL, 'result': result}\n\n\ndef build_error_payload(mid, exc, *, include_traceback):\n \"\"\"\n Build a protocol error response. Protocol/validation errors omit traceback;\n handler errors include it. Field order matches the reference server.\n \"\"\"\n error = {'type': type(exc).__name__, 'message': str(exc)}\n if include_traceback:\n error['traceback'] = traceback.format_exc()\n return {\n 'id': mid if mid is not None else -1,\n 'protocol': PROTOCOL,\n 'error': error,\n }\n"; +export const PYODIDE_BRIDGE_CORE_SOURCE: string = "\"\"\"\nShared tywrap bridge core: protocol dispatch + value (de)serialization.\n\nThis module is the SINGLE source of truth for the \"tywrap/1\" server-side\nprotocol. It is imported by:\n\n - runtime/python_bridge.py (the Node/Bun/Deno subprocess server and the HTTP\n server), which owns I/O concerns: the stdin/stdout JSONL loop, env-var size\n guards, the real OS pid, bridge='python-subprocess', and the final BridgeCodec\n encode wrapper.\n\n - the in-WASM Pyodide server (src/runtime/pyodide-transport.ts). Pyodide cannot read\n this file from disk, so it is shipped as a build-time-generated TypeScript\n string constant (src/runtime/pyodide-bootstrap-core.generated.ts) produced by\n scripts/generate-pyodide-bootstrap.mjs and exec'd into a module registered in\n sys.modules. A conformance drift guard (test/runtime_conformance.test.ts)\n asserts the generated constant stays byte-identical to this file.\n\nCROSS-LANGUAGE CONTRACT (Python <-> the TypeScript decoder in src/utils/codec.ts\nand the request encoder in src/runtime/bridge-codec.ts):\n\n * Every value-type \"marker\" envelope carries {'__tywrap__': ,\n 'codecVersion': 1, 'encoding': ...}. The 6 markers are: ndarray, dataframe,\n series, scipy.sparse, torch.tensor, sklearn.estimator.\n * bytes round-trip both ways via base64 envelopes (see _deserialize_bytes_*\n and the bytes branch of default_encoder).\n * NaN/Infinity are rejected (the JS client cannot parse the non-standard tokens\n that allow_nan=True would emit).\n\nPURITY: This module depends only on the standard library plus LAZY optional\nimports (numpy/pandas/scipy/torch/sklearn/pyarrow are each imported inside the\nfunction that needs them). It performs no stdin/stdout I/O and reads no env vars,\nso it runs unchanged under CPython-in-WASM (Pyodide).\n\nforce_json_markers: a *parameter* threaded through every serializer (including\nthe nested torch.tensor -> ndarray call). When True, ndarray/dataframe/series are\nforced down their JSON path regardless of pyarrow availability. Pyodide passes\nTrue (Arrow is unavailable in WASM); the subprocess server passes the boolean\nderived from TYWRAP_CODEC_FALLBACK=json so that \"Node in json-fallback mode\" and\n\"Pyodide\" produce byte-identical marker envelopes.\n\"\"\"\n\nimport base64\nimport datetime as dt\nimport decimal\nimport importlib\nimport importlib.util\nimport json\nimport math\nimport re\nimport sys\nimport traceback\nimport uuid\nfrom pathlib import Path, PurePath\n\n# Protocol constants. These MUST match src/runtime/protocol.ts (PROTOCOL_ID,\n# TYWRAP_PROTOCOL_VERSION) and the codec version baked into marker envelopes.\nPROTOCOL = 'tywrap/1'\nPROTOCOL_VERSION = 1\nCODEC_VERSION = 1\nMAX_SERIALIZE_DEPTH = 900\nMAX_SERIALIZE_NODES = 1_000_000\nJS_SAFE_INTEGER_MAX = 2**53 - 1\n_SERIALIZE_PATH_IDENTIFIER = re.compile(r'^[A-Za-z_$][\\w$]*$', re.ASCII)\n\n\nclass ProtocolError(Exception):\n \"\"\"Raised for malformed requests (bad protocol/id/method/params).\"\"\"\n\n\nclass ImportNotAllowedError(PermissionError):\n \"\"\"Raised when a requested module import is not on the active allowlist.\"\"\"\n\n def __init__(self, module_name):\n super().__init__(\n f'Import of module {module_name!r} is not permitted by the tywrap bridge '\n 'allowlist; add it to TYWRAP_ALLOWED_MODULES (subprocess) or the '\n 'allowed_modules parameter to enable it'\n )\n\n\nclass AttributeNotAllowedError(PermissionError):\n \"\"\"Raised when access to a private/dunder attribute is denied by policy.\"\"\"\n\n def __init__(self, attr_name):\n super().__init__(\n f'Access to attribute {attr_name!r} is not permitted by the tywrap bridge: '\n 'underscore-prefixed (private/dunder) attributes are blocked to prevent '\n 'sandbox-escape via attributes like __globals__/__subclasses__/__builtins__; '\n 'set TYWRAP_ALLOW_PRIVATE_ATTRS=1 (subprocess) or pass allow_private_attrs=True '\n 'to override'\n )\n\n\n# =============================================================================\n# IMPORT / ATTRIBUTE ALLOWLIST (trust boundary enforcement)\n# =============================================================================\n#\n# The bridge dispatches call requests by importing the requested module and\n# getattr-ing the requested function/class method. That is an\n# arbitrary import+getattr+call surface, so two complementary guards live here.\n# Both are PURE (no env reads) so the rules behave identically under the\n# subprocess server and the in-WASM Pyodide server; the subprocess server derives\n# the parameters from env vars (TYWRAP_ALLOWED_MODULES / TYWRAP_ALLOW_PRIVATE_ATTRS)\n# and threads them in, exactly like force_json_markers / torch_allow_copy.\n#\n# 1. MODULE ALLOWLIST (opt-in, default = allow all):\n# allowed_modules=None means \"no restriction\" so existing configurations keep\n# working unchanged. When a caller supplies a set, only those modules (plus the\n# stdlib the bridge itself needs to serialize results, see _BRIDGE_REQUIRED_MODULES)\n# may be imported; submodules of an allowed module are permitted (e.g. allowing\n# 'scipy' also allows 'scipy.sparse'). A non-allowlisted import fails LOUDLY with\n# ImportNotAllowedError rather than silently importing.\n#\n# 2. PRIVATE-ATTRIBUTE BLOCK (default ON):\n# getattr of any name starting with '_' (single-underscore private OR dunder) is\n# rejected. This blocks the classic escape chain (obj.__class__.__subclasses__()\n# /__globals__/__builtins__/__import__) without depending on the module allowlist.\n# tywrap-generated wrappers never reference underscore-prefixed names (the IR\n# analyzer skips them), so this does not regress generated code. Set\n# allow_private_attrs=True to restore unrestricted getattr for trusted callers.\n\n# Stdlib modules the bridge's own serialization/handlers may need to import even\n# when a caller-supplied allowlist is active. Optional codec deps (numpy, pandas,\n# scipy, torch, sklearn, pyarrow) are intentionally NOT here: if a caller restricts\n# modules, they must opt those in explicitly. These names cover only what the\n# bridge core itself imports.\n_BRIDGE_REQUIRED_MODULES = frozenset(\n {\n 'base64',\n 'datetime',\n 'decimal',\n 'importlib',\n 'json',\n 'math',\n 'sys',\n 'traceback',\n 'uuid',\n 'pathlib',\n }\n)\n\n\ndef _top_level_package(module_name):\n \"\"\"Return the top-level package of a dotted module name ('a.b.c' -> 'a').\"\"\"\n return module_name.split('.', 1)[0]\n\n\ndef _is_module_allowed(module_name, allowed_modules):\n \"\"\"\n Return True when module_name may be imported under the active policy.\n\n allowed_modules=None disables enforcement (allow all). Otherwise a module is\n allowed when it (or its top-level package) is explicitly listed, or it is one\n of the stdlib modules the bridge itself requires.\n \"\"\"\n if allowed_modules is None:\n return True\n if module_name in allowed_modules or module_name in _BRIDGE_REQUIRED_MODULES:\n return True\n top = _top_level_package(module_name)\n return top in allowed_modules or top in _BRIDGE_REQUIRED_MODULES\n\n\ndef import_allowed_module(module_name, allowed_modules):\n \"\"\"\n Import module_name only if permitted by the allowlist, else raise loudly.\n\n This is the single chokepoint every handler routes module imports through.\n \"\"\"\n if not _is_module_allowed(module_name, allowed_modules):\n raise ImportNotAllowedError(module_name)\n return importlib.import_module(module_name)\n\n\ndef get_allowed_attr(obj, attr_name, *, allow_private_attrs):\n \"\"\"\n getattr(obj, attr_name) with the private/dunder block applied.\n\n Rejects any underscore-prefixed name unless allow_private_attrs is True. This\n is the single chokepoint every handler routes attribute access through.\n \"\"\"\n if not allow_private_attrs and attr_name.startswith('_'):\n raise AttributeNotAllowedError(attr_name)\n return getattr(obj, attr_name)\n\n\ndef resolve_allowed_attr_path(root, dotted_name, *, allow_private_attrs):\n \"\"\"\n Resolve a possibly-dotted attribute path from root, applying the\n private/dunder getattr guard to EVERY segment.\n\n A single segment (the common case, e.g. a module-level function) behaves\n exactly like get_allowed_attr. Dotted names exist because @classmethod and\n @staticmethod are invoked through their owning class: the generated wrapper\n emits call(module, 'Class.method', ...), so the bridge must walk\n module -> Class -> method. Guarding each segment means 'Class._secret' or\n '_Hidden.method' are rejected exactly as a direct private getattr would be —\n the dotted path opens no access the single-getattr path did not already.\n \"\"\"\n obj = root\n for segment in dotted_name.split('.'):\n obj = get_allowed_attr(obj, segment, allow_private_attrs=allow_private_attrs)\n return obj\n\n\nclass CodecError(Exception):\n \"\"\"Raised when value encoding fails (e.g. NaN/Infinity not allowed).\"\"\"\n\n\n# =============================================================================\n# REQUEST-SIDE DESERIALIZATION (bytes envelopes -> Python bytes)\n# =============================================================================\n\n_NO_DESERIALIZE = object()\n_ERR_BYTES_MISSING_B64 = 'Invalid bytes envelope: missing b64'\n_ERR_BYTES_MISSING_DATA = 'Invalid bytes envelope: missing data'\n_ERR_BYTES_INVALID_BASE64 = 'Invalid bytes envelope: invalid base64'\n\n\ndef _deserialize_bytes_envelope(value):\n \"\"\"\n Decode base64-encoded bytes envelopes from JS into Python bytes.\n\n Supported shapes:\n - { \"__tywrap_bytes__\": true, \"b64\": \"...\" } (JS BridgeCodec.encodeRequest)\n - { \"__type__\": \"bytes\", \"encoding\": \"base64\", \"data\": \"...\" } (legacy/compat)\n\n Why: TS BridgeCodec encodes Uint8Array/ArrayBuffer as base64 objects, but\n Python handlers expect real bytes/bytearray to preserve behavior (e.g., len()).\n \"\"\"\n if not isinstance(value, dict):\n return _NO_DESERIALIZE\n\n if value.get('__tywrap_bytes__') is True:\n b64 = value.get('b64')\n if not isinstance(b64, str):\n raise ProtocolError(_ERR_BYTES_MISSING_B64)\n try:\n return base64.b64decode(b64, validate=True)\n except Exception as exc:\n raise ProtocolError(_ERR_BYTES_INVALID_BASE64) from exc\n\n if value.get('__type__') == 'bytes' and value.get('encoding') == 'base64':\n data = value.get('data')\n if not isinstance(data, str):\n raise ProtocolError(_ERR_BYTES_MISSING_DATA)\n try:\n return base64.b64decode(data, validate=True)\n except Exception as exc:\n raise ProtocolError(_ERR_BYTES_INVALID_BASE64) from exc\n\n return _NO_DESERIALIZE\n\n\ndef deserialize(value, *, has_envelope_markers=True):\n \"\"\"\n Recursively deserialize request values into Python-native types.\n\n Why: requests are JSON-only; we need a small set of explicit decoders\n (currently bytes) to restore Python semantics at the boundary.\n \"\"\"\n if not has_envelope_markers:\n return value\n decoded = _deserialize_bytes_envelope(value)\n if decoded is not _NO_DESERIALIZE:\n return decoded\n\n if isinstance(value, list):\n return [deserialize(item, has_envelope_markers=True) for item in value]\n if isinstance(value, dict):\n # Preserve dict shape while decoding nested values.\n return {k: deserialize(v, has_envelope_markers=True) for k, v in value.items()}\n return value\n\n\n# =============================================================================\n# CAPABILITY DETECTION (lazy, best-effort)\n# =============================================================================\n\ndef arrow_available():\n \"\"\"Return True when pyarrow can be imported.\"\"\"\n try:\n import pyarrow # noqa: F401\n except (ImportError, OSError):\n return False\n return True\n\n\ndef module_available(module_name):\n \"\"\"\n Lightweight feature detection for optional codec dependencies via find_spec.\n\n Why: exposes availability in bridge metadata without importing heavy modules.\n \"\"\"\n try:\n return importlib.util.find_spec(module_name) is not None\n except (ImportError, AttributeError, TypeError, ValueError):\n return False\n\n\ndef is_numpy_array(obj):\n \"\"\"Return whether obj is an ndarray without importing NumPy.\"\"\"\n np = sys.modules.get('numpy')\n if np is None:\n return False\n return isinstance(obj, np.ndarray)\n\n\ndef is_pandas_dataframe(obj):\n \"\"\"Return whether obj is a DataFrame without importing pandas.\"\"\"\n pd = sys.modules.get('pandas')\n if pd is None:\n return False\n return isinstance(obj, pd.DataFrame)\n\n\ndef is_pandas_series(obj):\n \"\"\"Return whether obj is a Series without importing pandas.\"\"\"\n pd = sys.modules.get('pandas')\n if pd is None:\n return False\n return isinstance(obj, pd.Series)\n\n\ndef is_scipy_sparse(obj):\n \"\"\"Return whether obj is sparse without importing SciPy.\"\"\"\n sp = sys.modules.get('scipy.sparse')\n if sp is None:\n return False\n try:\n return sp.issparse(obj)\n except Exception:\n return False\n\n\ndef is_torch_tensor(obj):\n \"\"\"Return whether obj is a Tensor without importing PyTorch.\"\"\"\n torch = sys.modules.get('torch')\n if torch is None:\n return False\n try:\n return torch.is_tensor(obj)\n except Exception:\n return False\n\n\ndef is_sklearn_estimator(obj):\n \"\"\"Return whether obj is an estimator without importing scikit-learn.\"\"\"\n sklearn_base = sys.modules.get('sklearn.base')\n if sklearn_base is None:\n return False\n return isinstance(obj, sklearn_base.BaseEstimator)\n\n\n# =============================================================================\n# MARKER SERIALIZERS (6 __tywrap__ value types)\n# =============================================================================\n#\n# Each serializer accepts force_json_markers. When True, the Arrow path is never\n# taken (used by Pyodide and by the subprocess server in TYWRAP_CODEC_FALLBACK=json\n# mode). The JSON fallback envelopes are byte-identical across both callers, which\n# is what the conformance suite asserts.\n\ndef serialize_ndarray(obj, *, force_json_markers):\n \"\"\"\n Encode a NumPy ndarray. Arrow IPC (compact, lossless) by default; JSON when\n force_json_markers is set or pyarrow is unavailable in fallback mode.\n\n Note: pa.array() only handles 1D arrays; multi-dimensional arrays are\n flattened with shape metadata for JS-side reconstruction. See\n https://github.com/apache/arrow-js/issues/115\n \"\"\"\n if force_json_markers:\n return serialize_ndarray_json(obj)\n try:\n import pyarrow as pa # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for ndarray; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n original_shape = list(obj.shape) if hasattr(obj, 'shape') else None\n flat = obj.flatten() if hasattr(obj, 'ndim') and obj.ndim != 1 else obj\n arr = pa.array(flat)\n table = pa.Table.from_arrays([arr], names=['value'])\n sink = pa.BufferOutputStream()\n with pa.ipc.new_stream(sink, table.schema) as writer:\n writer.write_table(table)\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'ndarray',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n 'shape': original_shape,\n 'dtype': str(obj.dtype) if hasattr(obj, 'dtype') else None,\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for ndarray') from exc\n\n\ndef serialize_ndarray_json(obj):\n \"\"\"JSON fallback for ndarray values that JavaScript can represent safely.\"\"\"\n dtype = obj.dtype\n dtype_label = str(dtype)\n\n if not dtype.isnative:\n raise RuntimeError(\n f'JSON ndarray encoding does not support big-endian dtype={dtype_label}; '\n \"convert to native byte order with a.byteswap().view(a.dtype.newbyteorder('='))\"\n )\n\n if dtype.kind in ('i', 'u'):\n # JSON numbers become JavaScript Number values. Scan only on this explicit\n # fallback path and reject before tolist() can silently round an integer.\n if obj.size and (\n (obj < -JS_SAFE_INTEGER_MAX).any() or (obj > JS_SAFE_INTEGER_MAX).any()\n ):\n raise RuntimeError(\n f'JSON ndarray encoding cannot safely represent dtype={dtype_label} values '\n 'outside the JavaScript safe integer range; use Arrow encoding or '\n \"cast/encode explicitly (e.g. .astype('float64') or str)\"\n )\n elif dtype.kind == 'M':\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '\n \"or convert explicitly (e.g. .astype('datetime64[ms]').astype(str) or int with \"\n 'declared unit)'\n )\n elif dtype.kind == 'm':\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '\n \"or convert explicitly (e.g. .astype('timedelta64[ms]').astype(str) or int with \"\n 'declared unit)'\n )\n elif dtype.kind == 'V':\n if dtype.fields is not None:\n raise RuntimeError(\n f'JSON ndarray encoding does not support structured dtype={dtype_label}; '\n 'encode each named field explicitly (e.g. as a plain JSON object)'\n )\n raise RuntimeError(\n f'JSON ndarray encoding does not support void dtype={dtype_label}; convert the '\n \"raw bytes explicitly (e.g. .view('uint8'))\"\n )\n elif dtype.kind == 'O':\n raise RuntimeError(\n f'JSON ndarray encoding does not support object dtype={dtype_label}; cast to a '\n 'concrete numeric dtype or encode elements explicitly as plain JSON'\n )\n elif dtype.kind == 'S':\n raise RuntimeError(\n f'JSON ndarray encoding does not support byte-string dtype={dtype_label}; '\n 'decode elements and return a plain JSON list explicitly'\n )\n elif dtype.kind == 'U':\n raise RuntimeError(\n f'JSON ndarray encoding does not support unicode dtype={dtype_label}; convert '\n 'explicitly to plain JSON strings with .tolist()'\n )\n elif dtype.kind == 'c':\n raise RuntimeError(\n f'JSON ndarray encoding does not support complex dtype={dtype_label}; encode '\n '.real and .imag arrays explicitly'\n )\n elif dtype.kind == 'f' and dtype.itemsize > 8:\n raise RuntimeError(\n f'JSON ndarray encoding does not support float dtype={dtype_label} wider than '\n \"64 bits; cast explicitly (e.g. .astype('float64')) or use Arrow encoding\"\n )\n elif dtype.kind not in ('b', 'f'):\n raise RuntimeError(\n f'JSON ndarray encoding does not support dtype={dtype_label}; cast to bool, '\n 'integer, or float dtype, or encode values explicitly as plain JSON'\n )\n\n try:\n data = obj.tolist()\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for ndarray') from exc\n return {\n '__tywrap__': 'ndarray',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n 'shape': getattr(obj, 'shape', None),\n 'dtype': dtype.name,\n }\n\n\ndef serialize_dataframe(obj, *, force_json_markers):\n \"\"\"\n Encode a pandas DataFrame. Feather/Arrow-IPC (uncompressed, so apache-arrow\n in JS can read it) by default; JSON when force_json_markers is set.\n \"\"\"\n if force_json_markers:\n return serialize_dataframe_json(obj)\n try:\n import pyarrow as pa # type: ignore\n import pyarrow.feather as feather # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for pandas.DataFrame; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n table = pa.Table.from_pandas(obj) # type: ignore\n sink = pa.BufferOutputStream()\n feather.write_feather(table, sink, compression='uncompressed')\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'dataframe',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for pandas.DataFrame') from exc\n\n\ndef serialize_dataframe_json(obj):\n \"\"\"JSON fallback for DataFrame values that JavaScript can represent safely.\"\"\"\n import pandas as pd # type: ignore\n\n _validate_pandas_json_index(obj.index, pd, 'DataFrame')\n json_columns = [_json_object_key(column) for column in obj.columns]\n supported_json_columns = [column for column in json_columns if column is not None]\n if not obj.columns.is_unique or len(set(supported_json_columns)) != len(\n supported_json_columns\n ):\n raise RuntimeError(\n 'JSON pandas.DataFrame encoding requires column labels to remain unique after '\n 'JSON object-key coercion; rename columns or make them distinct before applying '\n '.columns.astype(str)'\n )\n for column, dtype in obj.dtypes.items():\n if isinstance(dtype, pd.CategoricalDtype):\n raise RuntimeError(\n f'JSON pandas.DataFrame encoding does not support categorical dtype in '\n f'column {column!r}; use Arrow encoding or convert explicitly '\n \"(e.g. .astype(str))\"\n )\n try:\n data = (\n [{} for _ in range(len(obj.index))]\n if len(obj.columns) == 0\n else obj.to_dict(orient='records')\n )\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for pandas.DataFrame') from exc\n for row_number, row in enumerate(data):\n for column, value in row.items():\n row[column] = _normalize_pandas_json_scalar(\n value,\n f'DataFrame cell at row {row_number}, column {column!r}',\n pd,\n )\n return {\n '__tywrap__': 'dataframe',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n }\n\n\ndef serialize_series(obj, *, force_json_markers, path='result'):\n \"\"\"\n Encode a pandas Series as a single-column Arrow Table stream (the JS decoder\n contract is \"table-like\"); JSON when force_json_markers is set.\n \"\"\"\n if force_json_markers:\n return serialize_series_json(obj, path=path)\n name = getattr(obj, 'name', None)\n _check_terminal_json_integers(\n encode_value(name, allow_nan=False), _serialize_path(path, 'name')\n )\n try:\n import pyarrow as pa # type: ignore\n except Exception as exc:\n raise RuntimeError(\n 'Arrow encoding unavailable for pandas.Series; install pyarrow or set TYWRAP_CODEC_FALLBACK=json to enable JSON fallback'\n ) from exc\n try:\n arr = pa.Array.from_pandas(obj) # type: ignore\n table = pa.Table.from_arrays([arr], names=['value'])\n sink = pa.BufferOutputStream()\n with pa.ipc.new_stream(sink, table.schema) as writer:\n writer.write_table(table)\n buf = sink.getvalue()\n b64 = base64.b64encode(buf.to_pybytes()).decode('ascii')\n return {\n '__tywrap__': 'series',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'arrow',\n 'b64': b64,\n 'name': name,\n }\n except Exception as exc:\n raise RuntimeError('Arrow encoding failed for pandas.Series') from exc\n\n\ndef serialize_series_json(obj, *, path='result'):\n \"\"\"JSON fallback for Series values that JavaScript can represent safely.\"\"\"\n import pandas as pd # type: ignore\n\n _validate_pandas_json_index(obj.index, pd, 'Series')\n if isinstance(obj.dtype, pd.CategoricalDtype):\n raise RuntimeError(\n 'JSON pandas.Series encoding does not support categorical dtype; use Arrow '\n \"encoding or convert explicitly (e.g. .astype(str))\"\n )\n try:\n data = obj.to_list() # type: ignore\n except Exception as exc:\n raise RuntimeError('JSON fallback failed for pandas.Series') from exc\n data = [\n _normalize_pandas_json_scalar(value, f'Series value at position {position}', pd)\n for position, value in enumerate(data)\n ]\n name = getattr(obj, 'name', None)\n _check_terminal_json_integers(\n encode_value(name, allow_nan=False), _serialize_path(path, 'name')\n )\n return {\n '__tywrap__': 'series',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'data': data,\n 'name': name,\n }\n\n\ndef _validate_pandas_json_index(index, pd, container_name):\n \"\"\"Reject index metadata that records/list-oriented JSON would discard.\"\"\"\n if isinstance(index, pd.MultiIndex):\n raise RuntimeError(\n f'JSON pandas.{container_name} encoding does not support MultiIndex; use Arrow '\n 'encoding or flatten the index explicitly with .reset_index()'\n )\n if not (\n isinstance(index, pd.RangeIndex)\n and index.start == 0\n and index.step == 1\n and index.stop == len(index)\n and index.name is None\n ):\n raise RuntimeError(\n f'JSON pandas.{container_name} encoding requires an unnamed RangeIndex starting '\n 'at 0 with step 1; use Arrow encoding or normalize explicitly with '\n '.reset_index(drop=True)'\n )\n\n\ndef _json_object_key(value):\n \"\"\"Return json.dumps' object-key spelling, or None for unsupported keys.\"\"\"\n try:\n encoded = json.dumps({value: None})\n except (TypeError, ValueError):\n return None\n return next(iter(json.loads(encoded)))\n\n\ndef _normalize_pandas_json_scalar(value, location, pd):\n \"\"\"Normalize pandas nulls and reject values outside the plain JSON domain.\"\"\"\n np = sys.modules.get('numpy')\n if np is not None and isinstance(value, np.generic):\n value = value.item()\n if value is None or value is pd.NA or value is pd.NaT:\n return None\n if type(value) is bool:\n return value\n if type(value) is int:\n if value < -JS_SAFE_INTEGER_MAX or value > JS_SAFE_INTEGER_MAX:\n raise RuntimeError(\n f'JSON pandas encoding cannot safely represent {location} integer values '\n 'outside the JavaScript safe integer range; use Arrow encoding or '\n \"cast/encode explicitly (e.g. .astype('float64') or str)\"\n )\n return value\n if type(value) is float:\n if not math.isfinite(value):\n raise RuntimeError(\n f'JSON pandas encoding cannot represent non-finite {location} float values '\n '(NaN or Infinity); use .fillna(...) for intentional missing values or '\n 'Arrow encoding'\n )\n return value\n if type(value) is str:\n return value\n raise RuntimeError(\n f'JSON pandas encoding does not support {location} value of type '\n f'{type(value).__name__}; use Arrow encoding or convert explicitly '\n '(e.g. .astype(str))'\n )\n\n\ndef serialize_sparse_matrix(obj, *, path='result'):\n \"\"\"\n Serialize scipy sparse matrices into structured JSON envelopes (json-only;\n there is no Arrow path). Preserves sparsity; rejects unsupported formats and\n complex dtypes explicitly.\n \"\"\"\n try:\n fmt = obj.getformat()\n except Exception as exc:\n raise RuntimeError('Failed to inspect scipy sparse matrix format') from exc\n\n if fmt not in ('csr', 'csc', 'coo'):\n raise RuntimeError(\n f'Unsupported scipy sparse format: {fmt}; only csr/csc/coo are supported. '\n 'Convert explicitly (e.g. matrix.tocsr()) before returning'\n )\n\n dtype = None\n try:\n dtype = str(obj.dtype)\n except Exception:\n dtype = None\n if getattr(obj.dtype, 'kind', None) == 'c':\n raise RuntimeError(\n 'Complex scipy sparse matrices are not supported by the JSON codec; '\n 'split into real/imag components explicitly before returning'\n )\n if getattr(obj.dtype, 'kind', None) in ('i', 'u') and obj.data.size and (\n (obj.data < -JS_SAFE_INTEGER_MAX).any()\n or (obj.data > JS_SAFE_INTEGER_MAX).any()\n ):\n raise RuntimeError(\n 'JSON scipy sparse encoding cannot safely represent data outside the '\n 'JavaScript safe integer range; cast or encode the values explicitly'\n )\n\n for index, dimension in enumerate(obj.shape):\n if int.__int__(dimension) > JS_SAFE_INTEGER_MAX:\n raise RuntimeError(\n f'Unsafe Python integer at {_serialize_path(_serialize_path(path, \"shape\"), index)}: '\n f'{dimension} is outside the JavaScript safe integer range'\n )\n index_fields = ('indices', 'indptr') if fmt in ('csr', 'csc') else ('row', 'col')\n for field in index_fields:\n values = getattr(obj, field)\n if values.size and (\n (values < -JS_SAFE_INTEGER_MAX).any()\n or (values > JS_SAFE_INTEGER_MAX).any()\n ):\n raise RuntimeError(\n f'JSON scipy sparse encoding cannot safely represent {field} at '\n f'{_serialize_path(path, field)} outside the JavaScript safe integer range'\n )\n\n if fmt in ('csr', 'csc'):\n data = obj.data.tolist()\n indices = obj.indices.tolist()\n indptr = obj.indptr.tolist()\n return {\n '__tywrap__': 'scipy.sparse',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'format': fmt,\n 'shape': list(obj.shape),\n 'data': data,\n 'indices': indices,\n 'indptr': indptr,\n 'dtype': dtype,\n }\n\n # coo\n data = obj.data.tolist()\n row = obj.row.tolist()\n col = obj.col.tolist()\n return {\n '__tywrap__': 'scipy.sparse',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'format': fmt,\n 'shape': list(obj.shape),\n 'data': data,\n 'row': row,\n 'col': col,\n 'dtype': dtype,\n }\n\n\ndef serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):\n \"\"\"\n Serialize torch.Tensor values via the nested ndarray envelope. CPU-only by\n default; device/copy behavior is explicit. force_json_markers is threaded\n into the nested ndarray serialization so Pyodide gets a JSON ndarray value.\n\n Rejection order is significant: the categorical rejections (sparse / quantized\n / meta / complex) are checked BEFORE the device/contiguous opt-in branch so\n they fail with a clear, specific message and are NOT bypassable by\n TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the device transfer and\n contiguous copy, never an unrepresentable layout/dtype.\n \"\"\"\n import torch # already importable: is_torch_tensor() gated the dispatch\n\n tensor = obj.detach()\n source_device = None\n source_dtype = None\n\n # Sparse tensors (COO/CSR/CSC/BSR/BSC -> any non-strided layout) have no dense\n # numpy representation without a densify step, which is not the round-trip this\n # envelope promises. Reject explicitly rather than emitting a misleading\n # \"not contiguous\" error or silently densifying.\n layout = getattr(tensor, 'layout', None)\n if getattr(tensor, 'is_sparse', False) or (\n layout is not None and layout != torch.strided\n ):\n raise RuntimeError(\n f'Torch sparse tensors are not supported (layout={layout}); '\n 'convert to a dense CPU tensor explicitly (e.g. tensor.to_dense()) before returning'\n )\n\n # Quantized tensors carry a qscheme/scale/zero_point that numpy() cannot\n # represent; .numpy() raises an opaque \"unsupported ScalarType\" deep in torch.\n # Reject up front with an actionable message.\n if getattr(tensor, 'is_quantized', False):\n raise RuntimeError(\n 'Torch quantized tensors are not supported; dequantize explicitly '\n '(e.g. tensor.dequantize()) before returning'\n )\n\n # Meta tensors have shape/dtype but NO storage; copying to CPU yields garbage,\n # so this is never a lossy-but-honest transfer the opt-in could authorize.\n if getattr(tensor, 'is_meta', False) or (\n getattr(tensor, 'device', None) is not None and tensor.device.type == 'meta'\n ):\n raise RuntimeError(\n 'Torch meta tensors carry no data and cannot be serialized; '\n 'materialize the tensor on a real device before returning'\n )\n\n # Complex tensors round-trip to numpy complex arrays, which are not\n # JSON-serializable and have no codec envelope. Reject explicitly instead of\n # emitting Python complex tuples that the JS decoder cannot parse.\n if torch.is_complex(tensor):\n raise RuntimeError(\n f'Torch complex tensors are not supported (dtype={tensor.dtype}); '\n 'split into real/imag components explicitly before returning'\n )\n\n if getattr(tensor, 'device', None) is not None and tensor.device.type != 'cpu':\n if not torch_allow_copy:\n raise RuntimeError(\n 'Torch tensor is on a non-CPU device; set TYWRAP_TORCH_ALLOW_COPY=1 to allow CPU transfer'\n )\n source_device = str(tensor.device)\n tensor = tensor.to('cpu')\n if hasattr(tensor, 'is_contiguous') and not tensor.is_contiguous():\n if not torch_allow_copy:\n raise RuntimeError(\n 'Torch tensor is not contiguous; set TYWRAP_TORCH_ALLOW_COPY=1 to allow contiguous copy'\n )\n tensor = tensor.contiguous()\n if tensor.dtype == torch.bfloat16:\n source_dtype = str(tensor.dtype)\n tensor = tensor.float()\n try:\n arr = tensor.numpy()\n except Exception as exc:\n raise RuntimeError('Failed to convert torch.Tensor to numpy') from exc\n\n envelope = {\n '__tywrap__': 'torch.tensor',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'ndarray',\n 'value': serialize_ndarray(arr, force_json_markers=force_json_markers),\n 'shape': list(tensor.shape),\n 'dtype': str(tensor.dtype),\n 'device': str(tensor.device),\n }\n if source_dtype is not None:\n envelope['sourceDtype'] = source_dtype\n if source_device is not None:\n envelope['sourceDevice'] = source_device\n return envelope\n\n\ndef serialize_sklearn_estimator(obj, *, path):\n \"\"\"Serialize sklearn estimators as metadata only (json-only); no pickling.\"\"\"\n try:\n import sklearn # noqa: F401\n except Exception as exc:\n raise RuntimeError('scikit-learn is not available') from exc\n\n params = obj.get_params(deep=False)\n\n # Metadata-only: NEVER pickle/joblib. Every param value must be plain JSON\n # (no callables, nested estimators, numpy arrays, or other objects). Probe\n # each value individually so the error names the offending param instead of\n # failing opaquely on the whole dict. allow_nan=False also rejects NaN/Inf\n # params here for parity with the response codec.\n for key, value in params.items():\n try:\n encoded = json.dumps(value, allow_nan=False)\n param_path = _serialize_path(_serialize_path(path, 'params'), str(key))\n _check_terminal_json_integers(encoded, param_path)\n except RuntimeError:\n raise\n except (TypeError, ValueError) as exc:\n raise RuntimeError(\n f'scikit-learn estimator param {key!r} is not JSON-serializable '\n f'(got {type(value).__name__}); estimators are serialized as metadata only '\n '(no pickle/joblib), so every param must be a plain JSON value. '\n 'Sanitize or drop the param before returning'\n ) from exc\n\n return {\n '__tywrap__': 'sklearn.estimator',\n 'codecVersion': CODEC_VERSION,\n 'encoding': 'json',\n 'className': obj.__class__.__name__,\n 'module': obj.__class__.__module__,\n 'version': getattr(sklearn, '__version__', None),\n 'params': params,\n }\n\n\ndef _check_terminal_json_integers(encoded, path):\n \"\"\"Check integer tokens in a small JSON metadata value before emission.\"\"\"\n stack = [(json.loads(encoded), path)]\n while stack:\n current, current_path = stack.pop()\n if type(current) is int:\n _serialize_leaf(current, current_path)\n elif isinstance(current, dict):\n stack.extend(\n (item, _serialize_path(current_path, key))\n for key, item in current.items()\n )\n elif isinstance(current, list):\n stack.extend(\n (item, _serialize_path(current_path, index))\n for index, item in enumerate(current)\n )\n\n\n_NO_PYDANTIC = object()\n\n\ndef serialize_pydantic(obj):\n \"\"\"\n Serialize Pydantic v2 models via model_dump(by_alias=True, mode='json')\n without importing Pydantic. Returns _NO_PYDANTIC when obj is not a model.\n \"\"\"\n model_dump = getattr(obj, 'model_dump', None)\n if not callable(model_dump):\n return _NO_PYDANTIC\n try:\n try:\n return model_dump(by_alias=True, mode='json')\n except TypeError:\n # Older Pydantic versions may not support `mode=...`.\n return model_dump(by_alias=True)\n except Exception as exc:\n raise RuntimeError(f'model_dump failed: {exc}') from exc\n\n\ndef serialize_stdlib(obj):\n \"\"\"Coerce common stdlib scalar types to JSON-safe forms; None otherwise.\"\"\"\n if isinstance(obj, dt.datetime):\n return obj.isoformat()\n if isinstance(obj, dt.date):\n return obj.isoformat()\n if isinstance(obj, dt.time):\n return obj.isoformat()\n if isinstance(obj, dt.timedelta):\n return obj.total_seconds()\n if isinstance(obj, decimal.Decimal):\n return str(obj)\n if isinstance(obj, uuid.UUID):\n return str(obj)\n if isinstance(obj, (Path, PurePath)):\n return str(obj)\n return None\n\n\n_NO_SCIENTIFIC = object()\n\n\ndef _check_serialize_depth(depth, path):\n if depth > MAX_SERIALIZE_DEPTH:\n raise RuntimeError(\n f'Scientific envelope serialization maximum depth '\n f'{MAX_SERIALIZE_DEPTH} exceeded at {path}'\n )\n\n\ndef _check_serialize_nodes(nodes, path):\n if nodes > MAX_SERIALIZE_NODES:\n raise RuntimeError(\n f'Scientific envelope serialization maximum visited nodes '\n f'{MAX_SERIALIZE_NODES} exceeded at {path}'\n )\n\n\ndef _serialize_scientific(obj, *, force_json_markers, torch_allow_copy, depth, path):\n \"\"\"Serialize a supported scientific value, or return _NO_SCIENTIFIC.\"\"\"\n package = type(obj).__module__.split('.', 1)[0]\n\n if package == 'numpy' and 'numpy' in sys.modules:\n if is_numpy_array(obj):\n _check_serialize_depth(depth, path)\n return serialize_ndarray(obj, force_json_markers=force_json_markers)\n elif package == 'pandas' and 'pandas' in sys.modules:\n if is_pandas_dataframe(obj):\n _check_serialize_depth(depth, path)\n return serialize_dataframe(obj, force_json_markers=force_json_markers)\n if is_pandas_series(obj):\n _check_serialize_depth(depth, path)\n return serialize_series(obj, force_json_markers=force_json_markers, path=path)\n elif package == 'scipy' and 'scipy.sparse' in sys.modules:\n if is_scipy_sparse(obj):\n _check_serialize_depth(depth, path)\n return serialize_sparse_matrix(obj, path=path)\n elif package == 'torch' and 'torch' in sys.modules:\n if is_torch_tensor(obj):\n _check_serialize_depth(depth, path)\n _check_serialize_depth(depth + 1, _serialize_path(path, 'value'))\n return serialize_torch_tensor(\n obj, force_json_markers=force_json_markers, torch_allow_copy=torch_allow_copy\n )\n elif 'sklearn.base' in sys.modules and is_sklearn_estimator(obj):\n # No package gate here, unlike the branches above: subclassing\n # BaseEstimator is sklearn's documented extension point, so user-defined\n # estimators live outside the 'sklearn' package and must still get the\n # estimator serializer (and its param-naming errors).\n _check_serialize_depth(depth, path)\n return serialize_sklearn_estimator(obj, path=path)\n\n return _NO_SCIENTIFIC\n\n\ndef _serialize_path(base, key):\n \"\"\"Build a decoder-compatible JSONPath-like result path.\"\"\"\n if isinstance(key, int):\n return f'{base}[{key}]'\n if _SERIALIZE_PATH_IDENTIFIER.fullmatch(key):\n return f'{base}.{key}'\n return f'{base}[{json.dumps(key, ensure_ascii=False)}]'\n\n\ndef _invalid_key_path(base, key):\n \"\"\"Name a dict key that cannot be represented by JSON.\"\"\"\n return f'{base}[{key!r}]'\n\n\ndef _needs_serialize_visit(value):\n \"\"\"Return whether value needs container or scientific traversal work.\"\"\"\n if type(value) in (type(None), bool, int, float, str):\n return False\n if isinstance(value, (dict, list, tuple, set, frozenset)):\n return True\n package = type(value).__module__.split('.', 1)[0]\n if package in ('numpy', 'pandas', 'scipy', 'torch'):\n return True\n return callable(getattr(value, 'model_dump', None)) or (\n 'sklearn.base' in sys.modules and is_sklearn_estimator(value)\n )\n\n\ndef _serialize_leaf(value, path):\n \"\"\"Apply non-container conversions without allocating a traversal frame.\"\"\"\n integer_value = int.__int__(value) if isinstance(value, int) and not isinstance(value, bool) else None\n if integer_value is not None and (\n integer_value < -JS_SAFE_INTEGER_MAX or integer_value > JS_SAFE_INTEGER_MAX\n ):\n raise RuntimeError(\n f'Unsafe Python integer at {path}: {integer_value} is outside the JavaScript safe '\n 'integer range; return an explicit string or use an Arrow integer column'\n )\n if type(value) in (type(None), bool, int, float, str):\n return value\n np = sys.modules.get('numpy')\n if np is not None and isinstance(value, np.generic):\n extracted = value.item()\n if isinstance(extracted, np.generic):\n raise TypeError(\n f'NumPy scalar at {path} did not convert to a JSON value; '\n 'cast it explicitly before returning'\n )\n return _serialize_leaf(extracted, path)\n pydantic_value = serialize_pydantic(value)\n if pydantic_value is not _NO_PYDANTIC:\n return pydantic_value\n if isinstance(value, (set, frozenset)):\n return list(value)\n stdlib_value = serialize_stdlib(value)\n if stdlib_value is not None:\n return stdlib_value\n return value\n\n\ndef serialize(obj, *, force_json_markers, torch_allow_copy=False):\n \"\"\"\n Top-level result serializer.\n\n Scientific codecs are type-first and only inspect packages that the value can\n belong to. A value from an optional package implies that package is already in\n sys.modules, so these checks never cold-import the scientific stack. The\n package dispatch deliberately precedes the JSON-native fast path: e.g. a\n package-defined subclass of dict still receives its relevant codec check.\n Every other value is left untouched so the shared JSON encoder applies the\n exact same default conversion at the root and at any nested depth.\n \"\"\"\n root = [None]\n active_ids = set()\n stack = [('visit', obj, 0, 'result', root, 0)]\n visited_nodes = 0\n\n # Repeated aliases have value semantics and are intentionally serialized twice.\n while stack:\n frame = stack.pop()\n action = frame[0]\n if action == 'release-model':\n active_ids.remove(frame[1])\n continue\n if action == 'dict':\n _, current, depth, path, parent, key, output, iterator = frame\n try:\n item_key, item = next(iterator)\n except StopIteration:\n active_ids.remove(id(current))\n parent[key] = output\n continue\n stack.append(frame)\n if not (isinstance(item_key, (str, int, float, bool)) or item_key is None):\n invalid_path = _invalid_key_path(path, item_key)\n raise TypeError(\n f'keys must be str, int, float, bool or None, not '\n f'{type(item_key).__name__} at {invalid_path}'\n )\n child_key = _json_object_key(item_key)\n child_path = _serialize_path(path, child_key)\n if _needs_serialize_visit(item):\n stack.append(('visit', item, depth + 1, child_path, output, item_key))\n else:\n output[item_key] = _serialize_leaf(item, child_path)\n continue\n if action == 'sequence':\n _, current, depth, path, parent, key, output, index = frame\n if index == len(output):\n active_ids.remove(id(current))\n parent[key] = output if isinstance(current, list) else tuple(output)\n continue\n stack.append(('sequence', current, depth, path, parent, key, output, index + 1))\n item = current[index]\n child_path = _serialize_path(path, index)\n if _needs_serialize_visit(item):\n stack.append(\n ('visit', item, depth + 1, child_path, output, index)\n )\n else:\n output[index] = _serialize_leaf(item, child_path)\n continue\n\n _, current, depth, path, parent, key = frame\n try:\n scientific = _serialize_scientific(\n current,\n force_json_markers=force_json_markers,\n torch_allow_copy=torch_allow_copy,\n depth=depth,\n path=path,\n )\n except Exception as exc:\n if path == 'result' and 'safe integer range' not in str(exc):\n raise\n raise RuntimeError(f'Scientific value serialization failed at {path}: {exc}') from exc\n if scientific is not _NO_SCIENTIFIC:\n # Recognized envelopes are terminal containers to the JS decoder.\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, path)\n if scientific.get('__tywrap__') == 'torch.tensor':\n nested_path = _serialize_path(path, 'value')\n try:\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, nested_path)\n except Exception as exc:\n if path == 'result':\n raise\n raise RuntimeError(\n f'Scientific value serialization failed at {path}: {exc}'\n ) from exc\n parent[key] = scientific\n continue\n\n if callable(getattr(current, 'model_dump', None)):\n _check_serialize_depth(depth, path)\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, path)\n current_id = id(current)\n if current_id in active_ids:\n raise RuntimeError(f'Circular model conversion detected at {path}')\n active_ids.add(current_id)\n normalized = _serialize_leaf(current, path)\n if normalized is current:\n raise RuntimeError(f'Circular model conversion detected at {path}')\n stack.append(('release-model', current_id))\n if _needs_serialize_visit(normalized):\n stack.append(('visit', normalized, depth + 1, path, parent, key))\n else:\n parent[key] = _serialize_leaf(normalized, path)\n continue\n\n if isinstance(current, (dict, list, tuple)):\n _check_serialize_depth(depth, path)\n visited_nodes += 1\n _check_serialize_nodes(visited_nodes, path)\n current_id = id(current)\n if current_id in active_ids:\n raise RuntimeError(f'Circular reference detected at {path}')\n active_ids.add(current_id)\n\n if isinstance(current, dict):\n output = {}\n parent[key] = output\n stack.append(\n ('dict', current, depth, path, parent, key, output, iter(current.items()))\n )\n continue\n\n output = [None] * len(current)\n if isinstance(current, list):\n parent[key] = output\n stack.append(('sequence', current, depth, path, parent, key, output, 0))\n continue\n\n normalized = _serialize_leaf(current, path)\n if normalized is not current and _needs_serialize_visit(normalized):\n stack.append(('visit', normalized, depth, path, parent, key))\n elif normalized is not current:\n parent[key] = _serialize_leaf(normalized, path)\n else:\n parent[key] = normalized\n\n return root[0]\n\n\n# =============================================================================\n# JSON CODEC: value handling and size-limited encode/decode\n# =============================================================================\n#\n# This is the single Python implementation used by both the subprocess bridge and\n# the embedded Pyodide core. BridgeCodec adds payload-size enforcement around the\n# shared encoder; encode_value is the unbounded form used by the in-memory bridge.\n\ndef _is_nan_or_inf(value):\n if not isinstance(value, (int, float)):\n return False\n try:\n return math.isnan(value) or math.isinf(value)\n except (TypeError, ValueError):\n return False\n\n\ndef _is_numpy_scalar(obj):\n try:\n import numpy as np\n except ImportError:\n return False\n return isinstance(obj, (np.generic, np.ndarray)) and obj.ndim == 0\n\n\ndef _is_pandas_scalar(obj):\n try:\n import pandas as pd\n except ImportError:\n return False\n return isinstance(obj, (pd.Timestamp, pd.Timedelta, type(pd.NaT)))\n\n\ndef make_default_encoder(*, allow_nan):\n \"\"\"\n Build a json.dumps default= encoder matching BridgeCodec's value handling.\n\n Raises CodecError for NaN/Infinity extracted from numpy scalars (json.dumps\n itself rejects top-level/nested NaN/Infinity floats when allow_nan=False).\n \"\"\"\n\n def default_encoder(obj):\n # numpy/pandas scalars first (need .item() extraction).\n if _is_numpy_scalar(obj):\n extracted = obj.item()\n if not allow_nan and _is_nan_or_inf(extracted):\n raise CodecError('Cannot serialize NaN - NaN/Infinity not allowed in JSON')\n return extracted\n\n if _is_pandas_scalar(obj):\n try:\n import pandas as pd\n except ImportError:\n pass\n else:\n if obj is pd.NaT or (hasattr(pd, 'isna') and pd.isna(obj)):\n return None\n if isinstance(obj, pd.Timestamp):\n return obj.isoformat()\n if isinstance(obj, pd.Timedelta):\n return obj.total_seconds()\n\n stdlib_value = serialize_stdlib(obj)\n if stdlib_value is not None:\n return stdlib_value\n\n if isinstance(obj, (bytes, bytearray)):\n return {\n '__type__': 'bytes',\n 'encoding': 'base64',\n 'data': base64.b64encode(obj).decode('ascii'),\n }\n\n model_dump = getattr(obj, 'model_dump', None)\n if callable(model_dump):\n try:\n return model_dump(by_alias=True, mode='json')\n except TypeError:\n return model_dump(by_alias=True)\n\n if isinstance(obj, (set, frozenset)):\n return list(obj)\n\n if isinstance(obj, complex):\n raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')\n\n raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')\n\n return default_encoder\n\n\ndef encode_value(value, *, allow_nan):\n \"\"\"\n JSON-encode a fully-serialized response value, applying the BridgeCodec-equivalent\n default encoder and rejecting NaN/Infinity when allow_nan is False.\n\n Raises CodecError (wrapping the json.dumps ValueError) on NaN/Infinity, matching\n BridgeCodec's \"Cannot serialize NaN...\" wording so error parity holds.\n \"\"\"\n try:\n return json.dumps(value, default=make_default_encoder(allow_nan=allow_nan), allow_nan=allow_nan)\n except ValueError as exc:\n error_msg = str(exc).lower()\n # json.dumps(allow_nan=False) rejects NaN/Infinity with a ValueError whose\n # wording is Python-version dependent: 3.12+ appends the offending value\n # (\"...not JSON compliant: nan\"), but 3.10/3.11 emit only the canonical\n # \"Out of range float values are not JSON compliant\". Match that phrase too\n # so the typed error message is stable across versions.\n nonfinite_token = re.search(r'(^|[^a-z])(nan|inf|infinity)([^a-z]|$)', error_msg)\n if (\n not allow_nan and 'out of range float values are not json compliant' in error_msg\n ) or nonfinite_token:\n raise CodecError('Cannot serialize NaN - NaN/Infinity not allowed in JSON') from exc\n raise CodecError(f'JSON encoding failed: {exc}') from exc\n except TypeError as exc:\n raise CodecError(f'JSON encoding failed: {exc}') from exc\n\n\nclass BridgeCodec:\n \"\"\"Safe JSON codec with explicit value handling and payload-size limits.\"\"\"\n\n def __init__(self, allow_nan=False, max_payload_bytes=10 * 1024 * 1024):\n self.allow_nan = allow_nan\n self.max_payload_bytes = max_payload_bytes\n self._encoder = make_default_encoder(allow_nan=allow_nan)\n\n def encode(self, value):\n result = encode_value(value, allow_nan=self.allow_nan)\n if len(result.encode('utf-8')) > self.max_payload_bytes:\n raise CodecError(f'Payload exceeds {self.max_payload_bytes} bytes')\n return result\n\n def decode(self, payload):\n if len(payload.encode('utf-8')) > self.max_payload_bytes:\n raise CodecError(f'Payload exceeds {self.max_payload_bytes} bytes')\n try:\n return json.loads(payload)\n except json.JSONDecodeError as exc:\n raise CodecError(f'JSON decoding failed: {exc}') from exc\n\n def _default_encoder(self, obj):\n \"\"\"Compatibility hook for callers that used the codec's JSON encoder.\"\"\"\n return self._encoder(obj)\n\n\n_default_codec = None\n\n\ndef get_default_codec():\n \"\"\"Return the lazily-created default BridgeCodec instance.\"\"\"\n global _default_codec\n if _default_codec is None:\n _default_codec = BridgeCodec()\n return _default_codec\n\n\ndef encode(value, *, allow_nan=False):\n \"\"\"Encode a value with the default codec settings.\"\"\"\n if allow_nan:\n return BridgeCodec(allow_nan=True).encode(value)\n return get_default_codec().encode(value)\n\n\ndef decode(payload):\n \"\"\"Decode a JSON payload with the default codec settings.\"\"\"\n return get_default_codec().decode(payload)\n\n\n# =============================================================================\n# REQUEST VALIDATION + HANDLERS + DISPATCH\n# =============================================================================\n\ndef require_protocol(msg):\n if not isinstance(msg, dict):\n raise ProtocolError('Invalid request payload')\n proto = msg.get('protocol')\n if proto != PROTOCOL:\n raise ProtocolError(f'Invalid protocol: {proto}')\n mid = msg.get('id')\n if not isinstance(mid, int):\n raise ProtocolError(f'Invalid request id: {mid}')\n return mid\n\n\ndef require_str(params, key):\n value = params.get(key)\n if not isinstance(value, str) or not value:\n raise ProtocolError(f'Missing {key}')\n return value\n\n\ndef coerce_list(value, key):\n if value is None:\n return []\n if not isinstance(value, list):\n raise ProtocolError(f'Invalid {key}')\n return value\n\n\ndef coerce_dict(value, key):\n if value is None:\n return {}\n if not isinstance(value, dict):\n raise ProtocolError(f'Invalid {key}')\n return value\n\n\ndef handle_call(\n params,\n *,\n force_json_markers,\n torch_allow_copy,\n allowed_modules,\n allow_private_attrs,\n has_envelope_markers,\n):\n module_name = require_str(params, 'module')\n function_name = require_str(params, 'functionName')\n args = deserialize(coerce_list(params.get('args'), 'args'), has_envelope_markers=has_envelope_markers)\n kwargs = deserialize(coerce_dict(params.get('kwargs'), 'kwargs'), has_envelope_markers=has_envelope_markers)\n mod = import_allowed_module(module_name, allowed_modules)\n # function_name may be dotted ('Class.method') for @classmethod/@staticmethod\n # calls, which the generated wrapper routes through call() rather than an\n # instance handle. resolve_allowed_attr_path guards each segment.\n func = resolve_allowed_attr_path(mod, function_name, allow_private_attrs=allow_private_attrs)\n res = func(*args, **kwargs)\n return serialize(res, force_json_markers=force_json_markers, torch_allow_copy=torch_allow_copy)\n\n\ndef build_meta(\n *,\n bridge,\n pid,\n python_version,\n codec_fallback,\n arrow_available_override=None,\n):\n \"\"\"\n Build the bridge metadata payload.\n\n Field order here is part of the wire contract (the JS validator and the\n documented BridgeInfo shape). Callers supply the backend-specific identity:\n the subprocess server passes bridge='python-subprocess' and a real pid; the\n Pyodide server passes bridge='pyodide' and pid=None.\n\n arrow_available_override: when not None, report this value for arrowAvailable\n instead of probing pyarrow. The Pyodide server forces markers to JSON\n unconditionally, so it advertises arrowAvailable=False regardless of whether\n pyarrow happens to be importable in the WASM environment.\n \"\"\"\n arrow = arrow_available() if arrow_available_override is None else arrow_available_override\n meta = {\n 'protocol': PROTOCOL,\n 'protocolVersion': PROTOCOL_VERSION,\n 'bridge': bridge,\n 'pythonVersion': python_version,\n 'pid': pid,\n 'codecFallback': codec_fallback,\n 'arrowAvailable': arrow,\n 'scipyAvailable': module_available('scipy'),\n 'torchAvailable': module_available('torch'),\n 'sklearnAvailable': module_available('sklearn'),\n 'instances': 0,\n }\n return meta\n\n\ndef dispatch_request(\n msg,\n *,\n bridge,\n pid,\n force_json_markers,\n allow_nan=False,\n python_version=None,\n torch_allow_copy=False,\n arrow_available_override=None,\n allowed_modules=None,\n allow_private_attrs=False,\n has_envelope_markers=True,\n):\n \"\"\"\n Validate and route a request, returning the fully-serialized response dict\n ({'id', 'protocol', 'result'}). Raises ProtocolError for malformed requests\n and propagates handler exceptions to the caller, which is responsible for\n building the error envelope (so it controls traceback inclusion).\n\n allow_nan is accepted for signature symmetry; NaN rejection happens during\n the final encode_value() call, which the caller performs.\n\n allowed_modules: None (default) disables the import allowlist so existing\n behavior is preserved. Supplying a set restricts call imports to\n those modules (plus the stdlib the bridge itself needs) and raises\n ImportNotAllowedError otherwise. allow_private_attrs=False (default) blocks\n getattr of underscore-prefixed names; True restores unrestricted access. See\n the IMPORT / ATTRIBUTE ALLOWLIST section above for the full trust model.\n \"\"\"\n mid = require_protocol(msg)\n method = msg.get('method')\n if not isinstance(method, str):\n raise ProtocolError('Missing method')\n params = coerce_dict(msg.get('params'), 'params')\n if method == 'call':\n result = handle_call(\n params,\n force_json_markers=force_json_markers,\n torch_allow_copy=torch_allow_copy,\n allowed_modules=allowed_modules,\n allow_private_attrs=allow_private_attrs,\n has_envelope_markers=has_envelope_markers,\n )\n elif method == 'meta':\n if python_version is None:\n import sys\n python_version = sys.version.split()[0]\n codec_fallback = 'json' if force_json_markers else 'none'\n result = build_meta(\n bridge=bridge,\n pid=pid,\n python_version=python_version,\n codec_fallback=codec_fallback,\n arrow_available_override=arrow_available_override,\n )\n else:\n raise ProtocolError(f'Unknown method: {method}')\n return {'id': mid, 'protocol': PROTOCOL, 'result': result}\n\n\ndef build_error_payload(mid, exc, *, include_traceback):\n \"\"\"\n Build a protocol error response. Protocol/validation errors omit traceback;\n handler errors include it. Field order matches the reference server.\n \"\"\"\n error = {'type': type(exc).__name__, 'message': str(exc)}\n if include_traceback:\n error['traceback'] = traceback.format_exc()\n return {\n 'id': mid if mid is not None else -1,\n 'protocol': PROTOCOL,\n 'error': error,\n }\n"; diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index 3eee208f..e475d10a 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -23,6 +23,7 @@ import type { BridgeBackend, BridgeInfo } from '../types/index.js'; import { DisposableBase, type ExecuteOptions } from './bounded-context.js'; import { BridgeProtocolError } from './errors.js'; import { BridgeCodec, type CodecOptions } from './bridge-codec.js'; +import { DecodedProvenance } from './decoded-provenance.js'; import { PROTOCOL_ID, TYWRAP_PROTOCOL_VERSION, @@ -431,9 +432,10 @@ export class RpcClient extends DisposableBase { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { - return this.sendMessageAsync( + const provenance = validate ? new DecodedProvenance() : undefined; + return this.sendVia( { method: 'call', params: { @@ -446,11 +448,12 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - validate(result); + validate(result, provenance); return result; }, } - : undefined + : undefined, + responseStr => this.codec.decodeResponseAsync(responseStr, provenance) ); } diff --git a/src/runtime/validators.ts b/src/runtime/validators.ts index c3ade3e0..cabbafda 100644 --- a/src/runtime/validators.ts +++ b/src/runtime/validators.ts @@ -1,6 +1,9 @@ import { BridgeValidationError } from './errors.js'; +import type { DecodedProvenance, DecodedShapeMetadata } from './decoded-provenance.js'; import type { ScientificMarker } from '../utils/codec.js'; +export type { DecodedShapeMetadata } from './decoded-provenance.js'; + /** * Pure validation functions for runtime value checking. * @@ -24,6 +27,7 @@ export type ReturnSchema = | { kind: 'primitive'; type: 'number' | 'string' | 'boolean' | 'null' | 'undefined' | 'Uint8Array' | 'object'; + constraint?: 'safe-integer' | 'finite'; } | { kind: 'literal'; value: string | number | boolean | null } | { kind: 'array'; element: ReturnSchema } @@ -42,12 +46,26 @@ export type ReturnSchema = dtype?: string; }; -export type ReturnValidator = (result: T) => T; +export type ReturnValidator = (result: T, provenance?: DecodedProvenance) => T; + +/** One Python parameter used to select a declared overload after call binding. */ +export interface OverloadParameterSchema { + name: string; + kind: + | 'positional-only' + | 'positional-or-keyword' + | 'var-positional' + | 'keyword-only' + | 'var-keyword'; + optional: boolean; + value: ReturnSchema; +} -export interface DecodedShapeMetadata { - marker: ScientificMarker; - dims?: number; - dtype?: string; +/** An overload is selectable only when all of its value rules are supported. */ +export interface OverloadReturnSchema { + parameters: readonly OverloadParameterSchema[]; + result: ReturnSchema; + selectable: boolean; } const decodedShapeMetadata = new WeakMap(); @@ -99,7 +117,7 @@ function renderSchema(schema: ReturnSchema): string { case 'any': return 'unknown'; case 'primitive': - return schema.type; + return schema.constraint ?? schema.type; case 'literal': return JSON.stringify(schema.value); case 'array': @@ -120,9 +138,20 @@ function renderSchema(schema: ReturnSchema): string { interface CheckState { readonly definitions: Readonly>; readonly pairs: WeakMap>; + readonly provenance?: DecodedProvenance; } -function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean { +interface ValueLocation { + parent: object; + key: string | number; +} + +function check( + schema: ReturnSchema, + value: unknown, + state: CheckState, + location?: ValueLocation +): boolean { if (schema.kind === 'any') { return true; } @@ -140,7 +169,7 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean seen.add(schema.name); state.pairs.set(value, seen); } - return check(definition, value, state); + return check(definition, value, state, location); } switch (schema.kind) { @@ -157,16 +186,29 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean if (schema.type === 'object') { return isPlainObject(value); } + if (schema.type === 'number' && schema.constraint === 'safe-integer') { + return typeof value === 'number' && Number.isSafeInteger(value); + } + if (schema.type === 'number' && schema.constraint === 'finite') { + return typeof value === 'number' && Number.isFinite(value); + } return typeof value === schema.type; case 'literal': return Object.is(value, schema.value); case 'array': - return Array.isArray(value) && value.every(item => check(schema.element, item, state)); + return ( + Array.isArray(value) && + value.every((item, index) => + check(schema.element, item, state, { parent: value, key: index }) + ) + ); case 'tuple': return ( Array.isArray(value) && value.length === schema.elements.length && - schema.elements.every((entry, index) => check(entry, value[index], state)) + schema.elements.every((entry, index) => + check(entry, value[index], state, { parent: value, key: index }) + ) ); case 'record': { if (!isPlainObject(value)) { @@ -180,20 +222,26 @@ function check(schema: ReturnSchema, value: unknown, state: CheckState): boolean } return false; } - if (!check(field.schema, value[key], state)) { + if (!check(field.schema, value[key], state, { parent: value, key })) { return false; } } } return ( !schema.values || - Object.values(value).every(item => check(schema.values as ReturnSchema, item, state)) + Object.entries(value).every(([key, item]) => + check(schema.values as ReturnSchema, item, state, { parent: value, key }) + ) ); } case 'union': - return schema.options.some(option => check(option, value, state)); + return schema.options.some(option => check(option, value, state, location)); case 'marker': { - const metadata = isObjectLike(value) ? decodedShapeMetadata.get(value) : undefined; + const metadata = isObjectLike(value) + ? decodedShapeMetadata.get(value) + : location + ? state.provenance?.atChild(location.parent, location.key) + : state.provenance?.atRoot(); if (metadata?.marker !== schema.marker) { return false; } @@ -215,8 +263,10 @@ export function createReturnValidator( definitions: Readonly> = {} ): ReturnValidator { const declaredType = renderSchema(schema); - return (result: T): T => { - if (!check(schema, result, { definitions, pairs: new WeakMap>() })) { + return (result: T, provenance?: DecodedProvenance): T => { + if ( + !check(schema, result, { definitions, pairs: new WeakMap>(), provenance }) + ) { throw new BridgeValidationError({ declaredType, receivedShape: describeReceivedShape(result), @@ -227,6 +277,81 @@ export function createReturnValidator( }; } +function matchesOverload( + signature: OverloadReturnSchema, + args: readonly unknown[], + kwargs: Readonly> | undefined +): boolean { + if (!signature.selectable) { + return false; + } + const keywords = kwargs ?? {}; + const usedKeywords = new Set(); + let position = 0; + let acceptsOtherKeywords = false; + const matches = (schema: ReturnSchema, value: unknown): boolean => + check(schema, value, { definitions: {}, pairs: new WeakMap>() }); + + for (const parameter of signature.parameters) { + if (parameter.kind === 'var-positional') { + while (position < args.length) { + if (!matches(parameter.value, args[position])) { + return false; + } + position += 1; + } + continue; + } + if (parameter.kind === 'var-keyword') { + acceptsOtherKeywords = true; + for (const [name, value] of Object.entries(keywords)) { + if (!usedKeywords.has(name) && !matches(parameter.value, value)) { + return false; + } + } + continue; + } + const hasPosition = parameter.kind !== 'keyword-only' && position < args.length; + const hasKeyword = Object.prototype.hasOwnProperty.call(keywords, parameter.name); + if (hasKeyword && parameter.kind === 'positional-only') { + return false; + } + if (hasPosition && hasKeyword) { + return false; + } + if (!hasPosition && !hasKeyword) { + if (!parameter.optional) { + return false; + } + continue; + } + const value = hasPosition ? args[position++] : keywords[parameter.name]; + if (hasKeyword) { + usedKeywords.add(parameter.name); + } + if (!matches(parameter.value, value)) { + return false; + } + } + if (position < args.length) { + return false; + } + return acceptsOtherKeywords || Object.keys(keywords).every(name => usedKeywords.has(name)); +} + +/** Select the first supported match, as TypeScript does for declared overloads. */ +export function selectOverloadReturnValidator( + overloads: readonly OverloadReturnSchema[], + args: readonly unknown[], + kwargs: Readonly> | undefined, + fallback: ReturnValidator, + callSite: string, + definitions: Readonly> = {} +): ReturnValidator { + const selected = overloads.find(overload => matchesOverload(overload, args, kwargs)); + return selected ? createReturnValidator(selected.result, callSite, definitions) : fallback; +} + // ═══════════════════════════════════════════════════════════════════════════ // TYPE GUARDS // ═══════════════════════════════════════════════════════════════════════════ diff --git a/src/types/index.ts b/src/types/index.ts index c5b0d408..90546f36 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -2,6 +2,9 @@ * Core type definitions for tywrap */ +import type { ValueContract } from '../contracts/value-contract.js'; +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; + export interface PythonModule { name: string; path?: string; @@ -40,6 +43,21 @@ export interface PythonFunction { typeParameters?: PythonGenericParameter[]; returnType: PythonType; parameters: Parameter[]; + /** + * Distinct callable signatures extracted from `@typing.overload`. + * + * The implementation signature stays in `parameters` and `returnType` for + * runtime binding. Each overload retains its own input-to-result relation for + * generated declarations. + */ + overloads?: PythonFunctionOverload[]; + /** + * Internal value contracts resolved before code emission. + * + * The compiler sets this field. Parser-created models leave it absent, and + * the generator then uses its legacy schema derivation. + */ + callableContract?: ResolvedCallableContract; /** * Binding of this callable on its owning class. Defaults to `'instance'`. * @see PythonMethodKind @@ -47,6 +65,24 @@ export interface PythonFunction { methodKind?: PythonMethodKind; } +export interface PythonFunctionOverload { + parameters: Parameter[]; + returnType: PythonType; +} + +/** A resolved value contract for a callable signature. */ +export interface ResolvedCallableSignatureContract { + parameterValues: readonly (ValueContract | undefined)[]; + returnValue?: ValueContract; + /** Original annotation used for best-effort validation when no value contract resolves. */ + returnValidationType?: PythonType; +} + +/** Internal compiler data consumed by generated validators. */ +export interface ResolvedCallableContract extends ResolvedCallableSignatureContract { + overloads: readonly ResolvedCallableSignatureContract[]; +} + /** * A `@property` or `functools.cached_property` exposed on a class. * @@ -503,7 +539,7 @@ export interface PythonRuntime { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise; } diff --git a/src/tywrap.ts b/src/tywrap.ts index 4e079c4f..f31deeee 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -3,20 +3,23 @@ */ import { CodeGenerator } from './core/generator.js'; +import { + compileContract, + DEFAULT_CALLABLE_CAPABILITIES, + DEFAULT_VALUE_CONVERSION, +} from './core/callable-compiler.js'; +import { + TYWRAP_IR_VERSION, + validateIrContract, + type ValidatedIrContract, +} from './core/ir-contract.js'; +import { transformIrToTsModel } from './core/ir-model.js'; import { TypeMapper } from './core/mapper.js'; -import { parseAnnotationToPythonType } from './core/annotation-parser.js'; import { createConfig } from './config/index.js'; import type { TywrapOptions, - PythonGenericParameter, - PythonFunction, PythonModule as TSPythonModule, - PythonClass, - PythonTypeAlias, - Parameter, - PythonType, PythonModuleConfig, - IrContract, } from './types/index.js'; import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js'; import { globalCache } from './utils/cache.js'; @@ -24,8 +27,6 @@ import { resolvePythonExecutable } from './utils/python.js'; import { computeIrCacheFilename } from './utils/ir-cache.js'; import { logger } from './utils/logger.js'; -const TYWRAP_IR_VERSION = '0.4.0'; - // Collect unknown typing constructs encountered during annotation parsing (per-generate run) let unknownTypeNamesCollector: Map = new Map(); function recordUnknown(name: string): void { @@ -101,52 +102,25 @@ function stableJson(value: unknown): unknown { return value; } -function serializeContract(ir: IrContract): string { - const contract = { ...ir }; - delete contract.metadata; +function serializeContract(ir: ValidatedIrContract): string { + const contract = Object.fromEntries(Object.entries(ir).filter(([key]) => key !== 'metadata')); return `${JSON.stringify(stableJson(contract), null, 2)}\n`; } function validateIrVersion( ir: unknown, source: string -): { ir: IrContract | null; error?: string; code?: GenerateFailure['code'] } { - if (ir === null || typeof ir !== 'object' || Array.isArray(ir)) { - return { ir: null, code: 'contract-invalid', error: `${source} is not a JSON IR object.` }; - } - const contract = ir as IrContract; - if (typeof contract.ir_version !== 'string') { - return { - ir: null, - code: 'contract-invalid', - error: `${source} is missing its string ir_version.`, - }; - } - if (contract.ir_version !== TYWRAP_IR_VERSION) { - return { - ir: null, - code: 'ir-version-mismatch', - error: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${source} declares ${contract.ir_version}. Regenerate the contract with a matching tywrap_ir.`, - }; - } - if (typeof contract.module !== 'string' || contract.module.length === 0) { - return { ir: null, code: 'contract-invalid', error: `${source} is missing its module name.` }; +): { ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] } { + const result = validateIrContract(ir, source); + if (result.ok) { + return { ir: result.contract }; } - const requiredArrays: ReadonlyArray = [ - ['functions', contract.functions], - ['classes', contract.classes], - ['type_aliases', contract.type_aliases], - ]; - for (const [field, value] of requiredArrays) { - if (!Array.isArray(value)) { - return { - ir: null, - code: 'contract-invalid', - error: `${source} is missing required array field ${field}.`, - }; - } - } - return { ir: contract }; + const first = result.diagnostics[0]; + return { + ir: null, + code: first?.code ?? 'contract-invalid', + error: first?.message ?? `${source} is not a JSON IR object.`, + }; } async function safeReadFile(path: string): Promise { @@ -172,7 +146,7 @@ async function fetchAndCacheIr( cacheKey: string, caching: boolean, checkMode: boolean -): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> { +): Promise<{ ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] }> { let ir: unknown | null = null; let irError: string | undefined; if (caching && fsUtils.isAvailable() && !checkMode) { @@ -217,7 +191,7 @@ async function fetchAndCacheIr( async function readContractInput( moduleKey: string, contractInput: TywrapOptions['contractInput'] -): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> { +): Promise<{ ir: ValidatedIrContract | null; error?: string; code?: GenerateFailure['code'] }> { const inputPath = typeof contractInput === 'string' ? contractInput : contractInput?.[moduleKey]; if (!inputPath) { return { @@ -228,7 +202,17 @@ async function readContractInput( } try { const parsed = JSON.parse(await fsUtils.readFile(inputPath)) as unknown; - const validated = validateIrVersion(parsed, `Contract ${inputPath}`); + const validation = validateIrContract(parsed, `Contract ${inputPath}`, { + allowOmittedMetadata: true, + }); + const validated = validation.ok + ? { ir: validation.contract } + : { + ir: null, + code: validation.diagnostics[0]?.code ?? 'contract-invalid', + error: + validation.diagnostics[0]?.message ?? `Contract ${inputPath} is not a JSON IR object.`, + }; if (!validated.ir) { return { ir: null, error: validated.error, code: validated.code }; } @@ -455,14 +439,70 @@ export async function generate( const irWarnings = Array.isArray((ir as Record).warnings) ? ((ir as Record).warnings as unknown[]) : []; + + const moduleModel = transformIrToTsModel(ir, recordUnknown); + + // Apply module-level export filtering (functions/classes + excludes). + filterModuleExports(moduleModel, moduleConfig, moduleKey, warnings); + + // Resolve callable values once before emitting declarations and wrappers. + const annotatedJSDoc = Boolean(resolvedOptions.output?.annotatedJSDoc); + const compiled = compileContract(ir, { + module: moduleModel, + generator: instance.generator, + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + annotatedJSDoc, + }); + const gen = compiled.generated; + for (const diagnostic of compiled.diagnostics) { + warnings.push(diagnostic.message); + } for (const warning of irWarnings) { if (typeof warning !== 'string') { continue; } - // Only type-honesty degrades feed --fail-on-warn. Environment capability - // notices (e.g. typing.get_overloads missing on Python < 3.11) describe - // the analyzing interpreter, not the generated types, and must not fail - // an otherwise clean build. + const external = warning.match( + /^Return annotation for (.+) resolves outside analyzed module: (.+)\.$/ + ); + if (external) { + const qualname = external[1]; + const candidate = external[2]; + const functionIndex = ir.functions.findIndex(func => func.qualname === qualname); + const classIndex = ir.classes.findIndex(cls => + cls.methods.some(method => method.qualname === qualname) + ); + const methodIndex = + classIndex < 0 + ? -1 + : ir.classes[classIndex]!.methods.findIndex(method => method.qualname === qualname); + const path = + functionIndex >= 0 + ? `$.functions[${functionIndex}]` + : methodIndex >= 0 + ? `$.classes[${classIndex}].methods[${methodIndex}]` + : ''; + const callableResult = compiled.callables.find(callable => callable.path === path)?.result; + const logicalType = callableResult?.logicalType; + const outerName = + logicalType && + (logicalType.kind === 'custom' || logicalType.kind === 'generic') && + logicalType.module + ? `${logicalType.module}.${logicalType.name}` + : undefined; + const resolution = callableResult?.resolution; + if ( + resolution?.status === 'supported' && + candidate === outerName && + ((resolution.value.kind === 'ndarray-float16' && + (candidate === 'numpy.ndarray' || candidate === 'numpy.typing.NDArray')) || + (resolution.value.kind === 'torch-float16' && + (candidate === 'torch.Tensor' || candidate === 'torch.HalfTensor'))) + ) { + continue; + } + } + // Runtime notices do not fail strict mode. Type-honesty warnings do. if (warning.startsWith('Return annotation for ')) { recordUnknown(`Python IR warning: ${warning}`); } else { @@ -470,15 +510,6 @@ export async function generate( } } - const moduleModel = transformIrToTsModel(ir); - - // Apply module-level export filtering (functions/classes + excludes). - filterModuleExports(moduleModel, moduleConfig, moduleKey, warnings); - - // Generate module code - const annotatedJSDoc = Boolean(resolvedOptions.output?.annotatedJSDoc); - const gen = instance.generator.generateModuleDefinition(moduleModel, annotatedJSDoc); - const baseName = moduleModel.name || 'module'; const filesToEmit: Array<{ path: string; content: string }> = [ { path: pathUtils.join(outputDir, `${baseName}.generated.ts`), content: gen.typescript }, @@ -627,197 +658,6 @@ async function fetchPythonIr( } } -/** - * Convert JSON IR from Python into the internal TypeScript model used by the generator. - */ -function collectModuleTypeVarNames(obj: Record): Set { - const names = new Set(); - const constants = Array.isArray(obj.constants) ? (obj.constants as unknown[]) : []; - - for (const constant of constants) { - const entry = constant as Record; - const name = typeof entry.name === 'string' ? entry.name.trim() : ''; - if (!name) { - continue; - } - - const valueRepr = typeof entry.value_repr === 'string' ? entry.value_repr.trim() : ''; - if (/^~?[A-Za-z_][A-Za-z0-9_]*$/.test(valueRepr) && valueRepr.replace(/^~/, '') === name) { - names.add(name); - continue; - } - - const annotation = typeof entry.annotation === 'string' ? entry.annotation.trim() : ''; - if (/(^|\.)(TypeVar|ParamSpec|TypeVarTuple)(\[|\(|$)/.test(annotation)) { - names.add(name); - } - } - - return names; -} - -function transformIrToTsModel(ir: unknown): TSPythonModule { - const obj: Record = - typeof ir === 'object' && ir !== null ? (ir as Record) : {}; - const functions = (obj.functions as unknown[]) ?? []; - const classes = (obj.classes as unknown[]) ?? []; - const aliases = (obj.type_aliases as unknown[]) ?? []; - const moduleTypeVarNames = collectModuleTypeVarNames(obj); - const parseType = ( - annotation: unknown, - typeParameters: readonly PythonGenericParameter[] = [] - ): PythonType => - parseAnnotationToPythonType(annotation, { - onUnknownTypeName: recordUnknown, - knownTypeVarNames: moduleTypeVarNames, - typeParameters, - }); - const mapTypeParameters = (value: Record): PythonGenericParameter[] => - Array.isArray(value.type_params) - ? (value.type_params as unknown[]).map(v => { - const param = (v ?? {}) as Record; - return { - name: String(param.name ?? ''), - kind: String(param.kind ?? 'typevar') as PythonGenericParameter['kind'], - bound: param.bound ? parseType(param.bound) : undefined, - constraints: Array.isArray(param.constraints) - ? (param.constraints as unknown[]).map(item => parseType(item)) - : undefined, - variance: - param.variance === 'covariant' || - param.variance === 'contravariant' || - param.variance === 'invariant' - ? param.variance - : undefined, - } satisfies PythonGenericParameter; - }) - : []; - const mapParam = ( - p: Record, - typeParameters: readonly PythonGenericParameter[] = [] - ): Parameter => ({ - name: String(p.name ?? ''), - type: parseType(p.annotation, typeParameters), - optional: Boolean(p.default), - varArgs: p.kind === 'VAR_POSITIONAL', - kwArgs: p.kind === 'VAR_KEYWORD', - positionalOnly: p.kind === 'POSITIONAL_ONLY', - keywordOnly: p.kind === 'KEYWORD_ONLY', - }); - - const mapMethodKind = (value: unknown): PythonFunction['methodKind'] => - value === 'class' || value === 'static' ? value : 'instance'; - - const mapFunc = ( - f: Record, - inheritedTypeParameters: readonly PythonGenericParameter[] = [] - ): PythonFunction => { - const localTypeParameters = mapTypeParameters(f); - const annotationTypeParameters = [...inheritedTypeParameters, ...localTypeParameters]; - return { - name: String(f.name ?? ''), - signature: { - parameters: Array.isArray(f.parameters) - ? (f.parameters as unknown[]).map(v => - mapParam((v ?? {}) as Record, annotationTypeParameters) - ) - : [], - returnType: parseType(f.returns, annotationTypeParameters), - isAsync: Boolean(f.is_async), - isGenerator: Boolean(f.is_generator), - }, - docstring: (f.docstring as string | undefined) ?? undefined, - decorators: [], - isAsync: Boolean(f.is_async), - isGenerator: Boolean(f.is_generator), - typeParameters: [...localTypeParameters], - returnType: parseType(f.returns, annotationTypeParameters), - parameters: Array.isArray(f.parameters) - ? (f.parameters as unknown[]).map(v => - mapParam((v ?? {}) as Record, annotationTypeParameters) - ) - : [], - methodKind: mapMethodKind(f.method_kind), - }; - }; - - const mapClass = (c: Record): PythonClass => { - const classTypeParameters = mapTypeParameters(c); - return { - name: String(c.name ?? ''), - bases: Array.isArray(c.bases) ? (c.bases as string[]) : [], - methods: Array.isArray(c.methods) - ? (c.methods as unknown[]).map(v => - mapFunc((v ?? {}) as Record, classTypeParameters) - ) - : [], - properties: Array.isArray(c.fields) - ? (c.fields as unknown[]).map(v => { - const p = (v ?? {}) as Record; - return { - name: String(p.name ?? ''), - type: parseType(p.annotation, classTypeParameters), - readonly: false, - setter: false, - getter: true, - optional: Boolean(p.default), - }; - }) - : [], - accessors: Array.isArray(c.accessors) - ? (c.accessors as unknown[]).map(v => { - const a = (v ?? {}) as Record; - return { - name: String(a.name ?? ''), - type: parseType(a.returns, classTypeParameters), - docstring: (a.docstring as string | undefined) ?? undefined, - readOnly: typeof a.read_only === 'boolean' ? a.read_only : undefined, - isCached: Boolean(a.is_cached), - }; - }) - : [], - docstring: (c.docstring as string | undefined) ?? undefined, - decorators: (c.typed_dict as boolean) ? ['__typed_dict__'] : [], - kind: (c.typed_dict as boolean) - ? 'typed_dict' - : (c.is_protocol as boolean) - ? 'protocol' - : (c.is_namedtuple as boolean) - ? 'namedtuple' - : (c.is_dataclass as boolean) - ? 'dataclass' - : (c.is_pydantic as boolean) - ? 'pydantic' - : 'class', - typeParameters: classTypeParameters, - }; - }; - - const mapTypeAlias = (alias: Record): PythonTypeAlias => { - const typeParameters = mapTypeParameters(alias); - return { - name: String(alias.name ?? ''), - type: parseType(alias.definition, typeParameters), - typeParameters, - }; - }; - - const moduleModel: TSPythonModule = { - name: (obj.module as string) ?? 'module', - path: undefined, - version: - typeof (obj.metadata as Record | undefined)?.package_version === 'string' - ? ((obj.metadata as Record | undefined)?.package_version as string) - : undefined, - functions: functions.map(v => mapFunc((v ?? {}) as Record)), - classes: classes.map(v => mapClass((v ?? {}) as Record)), - typeAliases: aliases.map(v => mapTypeAlias((v ?? {}) as Record)), - imports: [], - exports: [], - }; - return moduleModel; -} - /** * Compute a stable cache key filename for a module and options */ diff --git a/src/utils/codec.ts b/src/utils/codec.ts index 19134559..83d26a2b 100644 --- a/src/utils/codec.ts +++ b/src/utils/codec.ts @@ -10,6 +10,8 @@ */ import { tagDecodedShape } from '../runtime/validators.js'; +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; +import { isSafeJsonInteger, MAX_SAFE_JSON_INTEGER } from '../contracts/value-contract.js'; const SCIENTIFIC_MARKERS = [ 'dataframe', @@ -376,17 +378,20 @@ function isPromiseLike(value: unknown): value is PromiseLike { * @param arr - Typed array or plain array * @returns Plain JavaScript array with values converted (BigInt → Number where safe) */ -function typedArrayToPlain(arr: unknown): unknown[] | null { +function typedArrayToPlain(arr: unknown, dtype?: string): unknown[] | null { if (Array.isArray(arr)) { - return arr; + return dtype === 'float16' ? arr.map(decodeFloat16StorageWord) : arr; } // Handle typed arrays (Int32Array, Float64Array, BigInt64Array, etc.) if (ArrayBuffer.isView(arr) && 'length' in arr) { + if (dtype === 'float16') { + return Array.from(arr as unknown as ArrayLike, decodeFloat16StorageWord); + } const values = Array.from(arr as unknown as ArrayLike); return values.map(value => { // Convert BigInt to Number if within safe integer range if (typeof value === 'bigint') { - if (value >= BigInt(Number.MIN_SAFE_INTEGER) && value <= BigInt(Number.MAX_SAFE_INTEGER)) { + if (value >= BigInt(-MAX_SAFE_JSON_INTEGER) && value <= BigInt(MAX_SAFE_JSON_INTEGER)) { return Number(value); } } @@ -395,7 +400,8 @@ function typedArrayToPlain(arr: unknown): unknown[] | null { } // Fallback: check if iterable before converting if (arr !== null && arr !== undefined && typeof arr === 'object' && Symbol.iterator in arr) { - return Array.from(arr as Iterable); + const values = Array.from(arr as Iterable); + return dtype === 'float16' ? values.map(decodeFloat16StorageWord) : values; } return null; } @@ -406,16 +412,30 @@ function typedArrayToPlain(arr: unknown): unknown[] | null { * Why: Arrow decoding returns Table objects, not raw arrays. We need to extract * the column values and convert any typed arrays to plain arrays. */ -function extractArrowValues(data: unknown): unknown[] | null { +function extractArrowValues(data: unknown, dtype?: string): unknown[] | null { if (Array.isArray(data)) { - return data; + return typedArrayToPlain(data, dtype); } // Arrow table - extract values from first column - const table = data as ArrowTable & { getChildAt?: (i: number) => { toArray?: () => unknown } }; + const table = data as ArrowTable & { + getChildAt?: (i: number) => { + toArray?: () => unknown; + type?: unknown; + nullCount?: number; + }; + }; if (typeof table.getChildAt === 'function') { const column = table.getChildAt(0); if (column && typeof column.toArray === 'function') { - return typedArrayToPlain(column.toArray()); + if (dtype === 'float16') { + if (column.type !== undefined && String(column.type) !== 'Float16') { + throw new Error('float16 Arrow column type does not match envelope dtype'); + } + if (column.nullCount !== undefined && column.nullCount > 0) { + throw new Error('float16 Arrow column contains null values'); + } + } + return typedArrayToPlain(column.toArray(), dtype); } } return null; @@ -427,7 +447,7 @@ function extractNdarrayArrowValues( dtype: string | undefined ): unknown[] { try { - const values = extractArrowValues(data); + const values = extractArrowValues(data, dtype); if (values) { return values; } @@ -646,6 +666,31 @@ function assertArrowCount( } } +/** Decode one IEEE 754 binary16 storage word. Keep negative zero. */ +function float16StorageToNumber(bits: number): number { + const sign = bits & 0x8000 ? -1 : 1; + const exponent = (bits >>> 10) & 0x1f; + const fraction = bits & 0x03ff; + if (exponent === 0) { + return sign * fraction * 2 ** -24; + } + if (exponent === 0x1f) { + return fraction === 0 ? sign * Infinity : NaN; + } + return sign * (1024 + fraction) * 2 ** (exponent - 25); +} + +function decodeFloat16StorageWord(bits: unknown, index: number): number { + if (typeof bits !== 'number' || !isSafeJsonInteger(bits) || bits < 0 || bits > 0xffff) { + throw new Error(`Invalid ndarray envelope: float16 storage at b64[${index}] must be a uint16`); + } + const number = float16StorageToNumber(bits); + if (!Number.isFinite(number)) { + throw new Error(`Invalid ndarray envelope: non-finite float16 value at b64[${index}]`); + } + return number; +} + function assertCodecVersion(envelope: { codecVersion?: unknown }, typeTag: string): void { if (!('codecVersion' in envelope)) { return; @@ -723,7 +768,7 @@ function finishNdarrayDecode( ): unknown { const values = strictV1 ? extractNdarrayArrowValues(data, shape, dtype) - : extractArrowValues(data); + : extractArrowValues(data, dtype); if (!values) { return tagDecodedShape(data, metadata); } @@ -1064,6 +1109,19 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = ( if (strictV1 && 'dtype' in nested) { assertOptionalNonEmptyDtype(nested.dtype, 'torch.tensor', 'value.dtype', nestedShape); } + if (strictV1 && dtype === 'torch.float16') { + if (!isStrictV1Envelope(nested)) { + throw new Error( + 'Invalid torch.tensor envelope: float16 value must use ndarray codecVersion 1' + ); + } + if (nested.dtype !== 'float16') { + throw new Error( + `Invalid torch.tensor envelope: value.dtype at path value.dtype must be "float16" ` + + `for outer dtype "torch.float16"; got ${JSON.stringify(nested.dtype)}` + ); + } + } const deviceValue = value.device; if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) { throw new Error( @@ -1290,7 +1348,8 @@ function decodeEnvelope( async function decodeEnvelopeAsync( value: unknown, - decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise + decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise, + provenance?: DecodedProvenance ): Promise { let visitedNodes = 0; @@ -1353,6 +1412,34 @@ async function decodeEnvelopeAsync( } }; + const recordScalarProvenance = ( + original: unknown, + decoded: unknown, + target?: DecodeTarget + ): void => { + if ( + !provenance || + typeof decoded !== 'number' || + !isPlainObject(original) || + original.__tywrap__ !== 'ndarray' || + !isStrictV1Envelope(original) || + !Array.isArray(original.shape) || + original.shape.length !== 0 + ) { + return; + } + const metadata = { + marker: 'ndarray' as const, + dims: 0, + dtype: typeof original.dtype === 'string' ? original.dtype : undefined, + }; + if (target) { + provenance.recordChild(target.container, target.key, metadata); + } else { + provenance.recordRoot(metadata); + } + }; + const settleDecoded = async ( decoded: PromiseLike, original: unknown, @@ -1365,6 +1452,7 @@ async function decodeEnvelopeAsync( } catch (error) { return prefixHandlerError(error, path); } + recordScalarProvenance(original, resolved, path === 'result' ? undefined : target); assignDecoded(target, original, resolved); }; @@ -1411,8 +1499,10 @@ async function decodeEnvelopeAsync( pending.push(settleDecoded(decoded, current, { container: root, key: 'value' }, path)); } } else if (target) { + recordScalarProvenance(current, decoded, target); assignDecoded(target, current, decoded); } else if (decoded !== current) { + recordScalarProvenance(current, decoded); root.value = decoded; } return; @@ -1486,9 +1576,12 @@ async function decodeEnvelopeAsync( /** * Decode values produced by the Python bridge. */ -export async function decodeValueAsync(value: unknown): Promise { +export async function decodeValueAsync( + value: unknown, + provenance?: DecodedProvenance +): Promise { try { - return await decodeEnvelopeAsync(value, tryDecodeArrowTable); + return await decodeEnvelopeAsync(value, tryDecodeArrowTable, provenance); } catch (error) { throw asScientificDecodeError(error, 'unknown'); } diff --git a/test/annotation-parser.test.ts b/test/annotation-parser.test.ts index 9d87863e..f4fb0c99 100644 --- a/test/annotation-parser.test.ts +++ b/test/annotation-parser.test.ts @@ -68,4 +68,35 @@ describe('annotation parser', () => { expect((t as any).name).toBe('args'); expect((t as any).module).toBe('Request'); }); + + it('keeps NumPy dtype detail without warning on its shape wildcard', () => { + const unknown: string[] = []; + const type = parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', + { onUnknownTypeName: name => unknown.push(name) } + ); + expect(type).toMatchObject({ + kind: 'generic', + name: 'ndarray', + module: 'numpy', + typeArgs: [ + { kind: 'collection', name: 'tuple' }, + { + kind: 'generic', + name: 'dtype', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }, + ], + }); + expect(unknown).toEqual([]); + parseAnnotationToPythonType('numpy.ndarray[tuple[Any, ...], numpy.dtype[numpy.float16]]', { + onUnknownTypeName: name => unknown.push(name), + }); + expect(unknown).toEqual([]); + parseAnnotationToPythonType('list[Any]', { + onUnknownTypeName: name => unknown.push(name), + }); + expect(unknown).toEqual(['Any']); + }); }); diff --git a/test/bridge-codec.test.ts b/test/bridge-codec.test.ts index fe83668f..fd38e01a 100644 --- a/test/bridge-codec.test.ts +++ b/test/bridge-codec.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { BridgeCodec, type CodecOptions } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { BridgeCodecError, BridgeProtocolError, @@ -633,6 +634,71 @@ describe('decodeResponseAsync - Arrow Integration', () => { }); }); + it('records scalar ndarray proof at the root and nested array and record slots', async () => { + const scalar = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype: 'float16', + }; + const rootProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: scalar }), + rootProof + ) + ).toBe(1.5); + expect(rootProof.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + + const nestedProof = new DecodedProvenance(); + const decoded = await codec.decodeResponseAsync<{ tuple: [number, { value: number }] }>( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { tuple: [scalar, { value: scalar }] }, + }), + nestedProof + ); + expect(decoded).toEqual({ tuple: [1.5, { value: 1.5 }] }); + expect(nestedProof.atRoot()).toBeUndefined(); + expect(nestedProof.atChild(decoded.tuple, '0')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple[1], 'value')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple, 1)).toBeUndefined(); + }); + + it('does not record proof for plain numbers or malformed scalar envelopes', async () => { + const plainProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: 1.5 }), + plainProof + ) + ).toBe(1.5); + expect(plainProof.atRoot()).toBeUndefined(); + + const malformedProof = new DecodedProvenance(); + await expect( + codec.decodeResponseAsync( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + }, + }), + malformedProof + ) + ).rejects.toThrow(/must be an array at depth 0/); + expect(malformedProof.atRoot()).toBeUndefined(); + }); + it('labels v1 scientific-envelope validation failures without Arrow wording', async () => { const payload = JSON.stringify({ id: 1, diff --git a/test/callable-compiler.test.ts b/test/callable-compiler.test.ts new file mode 100644 index 00000000..e154674f --- /dev/null +++ b/test/callable-compiler.test.ts @@ -0,0 +1,1732 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import ts from 'typescript'; +import { + compileContract, + DEFAULT_CALLABLE_CAPABILITIES, + DEFAULT_VALUE_CONVERSION, +} from '../src/core/callable-compiler.js'; +import { CodeGenerator } from '../src/core/generator.js'; +import { parseAnnotationToPythonType } from '../src/core/annotation-parser.js'; +import { validateIrContract } from '../src/core/ir-contract.js'; +import { BridgeValidationError } from '../src/runtime/errors.js'; +import { HttpBridge } from '../src/runtime/http.js'; +import { clearRuntimeBridge, setRuntimeBridge } from 'tywrap/runtime'; +import type { PythonModule, PythonType } from '../src/types/index.js'; + +const rawIr = { + ir_version: '0.4.0', + module: 'fixture', + functions: [ + { + name: 'select', + qualname: 'fixture.select', + docstring: null, + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str | int', default: false }, + ], + returns: 'str | int', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [ + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: false }, + ], + returns: 'int', + }, + ], + }, + { + name: 'nested', + qualname: 'fixture.nested', + docstring: null, + parameters: [], + returns: 'dict[str, dict[str, int]]', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + { + name: 'async_value', + qualname: 'fixture.async_value', + docstring: null, + parameters: [], + returns: 'str', + is_async: true, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + { + name: 'point', + qualname: 'fixture.point', + docstring: null, + parameters: [], + returns: 'fixture.Point', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + ], + classes: [ + { + name: 'Point', + qualname: 'fixture.Point', + docstring: null, + bases: ['object'], + methods: [], + typed_dict: false, + total: null, + fields: [ + { name: 'x', kind: 'FIELD', annotation: 'int', default: false }, + { name: 'y', kind: 'FIELD', annotation: 'int', default: false }, + ], + is_protocol: false, + is_namedtuple: false, + is_dataclass: true, + is_pydantic: false, + type_params: [], + accessors: [], + }, + ], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [], +}; + +const moduleModel: PythonModule = { + name: 'fixture', + functions: [ + { + name: 'select', + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Any' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + overloads: [ + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'int' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'int' }, + }, + ], + }, + { + name: 'nested', + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Any' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [], + returnType: { + kind: 'collection', + name: 'dict', + itemTypes: [ + { kind: 'primitive', name: 'str' }, + { + kind: 'collection', + name: 'dict', + itemTypes: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + ], + }, + }, + { + name: 'async_value', + signature: { + parameters: [], + returnType: { kind: 'primitive', name: 'str' }, + isAsync: true, + isGenerator: false, + }, + decorators: [], + isAsync: true, + isGenerator: false, + parameters: [], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + name: 'point', + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Point' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [], + returnType: { kind: 'custom', name: 'Point' }, + }, + ], + classes: [ + { + name: 'Point', + bases: [], + methods: [], + properties: [], + decorators: [], + kind: 'dataclass', + }, + ], + typeAliases: [], + imports: [], + exports: [], +}; + +describe('compileContract', () => { + it('resolves overloads and nested records once before emitting both wrapper files', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + + expect(compiled.files.map(file => file.kind)).toEqual(['typescript', 'declaration']); + expect(compiled.generated.typescript).toContain( + 'export function select(key: string): Promise;' + ); + expect(compiled.generated.declaration).toContain( + 'export function select(key: number): Promise;' + ); + expect(compiled.generated.typescript).toContain( + 'export async function select(key: unknown): Promise' + ); + const nested = compiled.callables.find(callable => callable.name === 'nested'); + expect(nested?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'record', + additionalValues: { + kind: 'record', + additionalValues: { kind: 'integer', constraint: 'safe-integer' }, + }, + }, + }); + const selected = compiled.callables.find(callable => callable.name === 'select'); + expect(selected?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'union', + options: [ + { kind: 'string', wire: 'json', decodedAs: 'string' }, + { kind: 'integer', wire: 'json', decodedAs: 'number', constraint: 'safe-integer' }, + ], + }, + }); + expect(compiled.generated.typescript).toContain('selectOverloadReturnValidator('); + expect(compiled.generated.typescript).toContain('"constraint":"safe-integer"'); + }); + + it('rejects a wrong return from the generated wrapper for a selected overload', async () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-overload-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + _functionName: string, + args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = (typeof args[0] === 'string' ? 99 : 4) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + select: (key: string | number) => Promise; + }; + await expect(generated.select('key')).rejects.toThrow(BridgeValidationError); + await expect(generated.select(2)).resolves.toBe(4); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('warns when an optional parameter makes two overload inputs overlap', () => { + const source = rawIr.functions[0]!; + const key = source.overloads[0]!.parameters[0]!; + const optionalBase = { + name: 'base', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: 'int', + default: true, + }; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + parameters: [key, optionalBase], + overloads: [ + { parameters: [key], returns: 'str' }, + { parameters: [key, optionalBase], returns: 'int' }, + ], + }, + ], + classes: [], + }, + 'optional overload contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const original = moduleModel.functions[0]!; + const keyModel = original.overloads![0]!.parameters[0]!; + const baseModel = { + ...keyModel, + name: 'base', + type: { kind: 'primitive', name: 'int' } as PythonType, + optional: true, + }; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [ + { + ...original, + parameters: [keyModel, baseModel], + overloads: [ + { parameters: [keyModel], returnType: { kind: 'primitive', name: 'str' } }, + { parameters: [keyModel, baseModel], returnType: { kind: 'primitive', name: 'int' } }, + ], + }, + ], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'overload-ambiguous', + path: '$.functions[0].overloads[1]', + }), + ]) + ); + }); + + it('warns when keyword-only overloads overlap despite declaration order', () => { + const source = rawIr.functions[0]!; + const x = { name: 'x', kind: 'KEYWORD_ONLY', annotation: 'str', default: false }; + const y = { name: 'y', kind: 'KEYWORD_ONLY', annotation: 'int', default: false }; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'choose', + qualname: 'fixture.choose', + parameters: [x, y], + overloads: [ + { parameters: [x, y], returns: 'str' }, + { parameters: [y, x], returns: 'int' }, + ], + }, + ], + classes: [], + }, + 'keyword overload contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: [{ ...moduleModel.functions[0]!, name: 'choose' }], + classes: [], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'overload-ambiguous', + path: '$.functions[0].overloads[1]', + }), + ]) + ); + }); + + it('takes callable types from validated IR, not from the export selection model', () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'echo', + qualname: 'fixture.echo', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + ], + classes: [], + }, + 'IR authority contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const forged = { + ...moduleModel.functions[1]!, + name: 'echo', + parameters: [ + { + name: 'value', + type: { kind: 'primitive', name: 'int' } as PythonType, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'int' } as PythonType, + }; + const compiled = compileContract(ir.contract, { + module: { ...moduleModel, functions: [forged], classes: [] }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.module.functions[0]?.parameters[0]?.type).toEqual({ + kind: 'primitive', + name: 'str', + }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', + value: { kind: 'string' }, + }); + expect(compiled.generated.declaration).toContain('echo(value: string): Promise'); + expect(compiled.generated.declaration).not.toContain('echo(value: number)'); + }); + + it('selects a class method overload without validating its implicit cls receiver', async () => { + const source = rawIr.functions[0]!; + const receiver = { + name: 'cls', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: null, + default: false, + }; + const stringInput = source.overloads[0]!.parameters[0]!; + const integerInput = source.overloads[1]!.parameters[0]!; + const methodIr = { + ...source, + name: 'convert', + qualname: 'fixture.Converter.convert', + parameters: [receiver, source.parameters[0]!], + method_kind: 'class', + overloads: [ + { parameters: [receiver, stringInput], returns: 'str' }, + { parameters: [receiver, integerInput], returns: 'int' }, + ], + }; + const ir = validateIrContract( + { + ...rawIr, + functions: [], + classes: [ + { + ...rawIr.classes[0]!, + name: 'Converter', + qualname: 'fixture.Converter', + methods: [methodIr], + fields: [], + is_dataclass: false, + }, + ], + }, + 'class overload contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const original = moduleModel.functions[0]!; + const receiverModel = { + ...original.parameters[0]!, + name: 'cls', + type: { kind: 'custom', name: 'Any', module: 'typing' } as PythonType, + }; + const method = { + ...original, + name: 'convert', + methodKind: 'class' as const, + parameters: [receiverModel, original.parameters[0]!], + overloads: original.overloads?.map(overload => ({ + ...overload, + parameters: [receiverModel, ...overload.parameters], + })), + }; + const model: PythonModule = { + ...moduleModel, + functions: [], + classes: [ + { + ...moduleModel.classes[0]!, + name: 'Converter', + kind: 'class', + methods: [method], + properties: [], + }, + ], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics.some(item => item.path.endsWith('.parameters[0]'))).toBe(false); + expect(compiled.generated.typescript).toContain('"selectable":true'); + expect(compiled.generated.declaration).toContain( + 'static convert(key: string): Promise;' + ); + expect(compiled.generated.typescript).toContain( + 'static async convert(key: unknown): Promise' + ); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-class-overload-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + _functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = 7 as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + Converter: { convert: (key: string | number) => Promise }; + }; + await expect(generated.Converter.convert('key')).rejects.toThrow(BridgeValidationError); + await expect(generated.Converter.convert(2)).resolves.toBe(7); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('validates a generated scalar float16 wrapper from the HTTP codec proof', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'scalar_value', + qualname: 'fixture.scalar_value', + returns: 'numpy.typing.NDArray[numpy.float16]', + }, + ], + classes: [], + }, + 'scalar contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const scalarType: PythonType = { + kind: 'generic', + name: 'NDArray', + module: 'numpy.typing', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }; + const original = moduleModel.functions[1]!; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [ + { + ...original, + name: 'scalar_value', + returnType: scalarType, + signature: { ...original.signature, returnType: scalarType }, + }, + ], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.typescript).toContain('"marker":"ndarray","dtype":"float16"'); + expect(compiled.generated.declaration).toContain( + 'scalarValue(): Promise<__tywrapFloat16Value>' + ); + + let dtype = 'float16'; + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + shape: [], + dtype, + data: 1.5, + }, + }) + ); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-scalar-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + scalarValue: () => Promise; + }; + await expect(generated.scalarValue()).resolves.toBe(1.5); + dtype = 'float32'; + await expect(generated.scalarValue()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('uses revision 2 for selected unions and fixed heterogeneous tuples', () => { + const union = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + }); + const tuple = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[1].returns', + logicalType: { + kind: 'collection', + name: 'tuple', + itemTypes: [ + { kind: 'primitive', name: 'int' }, + { kind: 'primitive', name: 'str' }, + ], + }, + }); + + expect(union).toMatchObject({ + status: 'supported', + value: { kind: 'union', options: [{ kind: 'string' }, { kind: 'integer' }] }, + }); + expect(tuple).toMatchObject({ + status: 'supported', + value: { kind: 'tuple', items: [{ kind: 'integer' }, { kind: 'string' }] }, + }); + }); + + it('requires dtype evidence before selecting the float16 scientific contract', () => { + const resolve = (logicalType: PythonType) => + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType, + }); + + expect(resolve({ kind: 'custom', name: 'ndarray', module: 'numpy' })).toMatchObject({ + status: 'unresolved', + }); + expect(resolve({ kind: 'custom', name: 'Tensor', module: 'torch' })).toMatchObject({ + status: 'unresolved', + }); + expect( + resolve({ + kind: 'generic', + name: 'NDArray', + module: 'numpy', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'numpy' }], + }) + ).toMatchObject({ + status: 'supported', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]' + ) + ) + ).toMatchObject({ + status: 'supported', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float32]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); + expect( + resolve( + parseAnnotationToPythonType( + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[torch.float16]]' + ) + ) + ).toMatchObject({ status: 'unresolved' }); + }); + + it('keeps the existing bytes RPC shape in the value contract', () => { + const resolution = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { kind: 'primitive', name: 'bytes' }, + }); + expect(resolution).toMatchObject({ + status: 'supported', + value: { kind: 'bytes', wire: 'base64-envelope', decodedAs: 'Uint8Array' }, + }); + }); + + it('matches the outer Torch dtype recorded by the decoder', () => { + const tensorType: PythonType = { + kind: 'generic', + name: 'Tensor', + module: 'torch', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], + }; + const resolution = DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: tensorType, + }); + expect(resolution.status).toBe('supported'); + if (resolution.status !== 'supported') { + return; + } + const generated = new CodeGenerator().generateModuleDefinition({ + ...moduleModel, + classes: [], + functions: [ + { + ...moduleModel.functions[1]!, + name: 'tensorValue', + returnType: tensorType, + callableContract: { + parameterValues: [], + returnValue: resolution.value, + overloads: [], + }, + }, + ], + }); + expect(generated.typescript).toContain('"marker":"torch.tensor","dtype":"torch.float16"'); + expect(generated.declaration).toContain('tensorValue(): Promise<__tywrapFloat16Tensor>'); + expect( + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: { + ...tensorType, + typeArgs: [{ kind: 'custom', name: 'float16', module: 'external' }], + }, + }) + ).toMatchObject({ status: 'unresolved' }); + }); + + it('accepts only the evaluated torch.HalfTensor class as an exact float16 annotation', () => { + const resolve = (annotation: string) => + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType: parseAnnotationToPythonType(annotation), + }); + expect(resolve('torch.HalfTensor')).toMatchObject({ + status: 'supported', + value: { + kind: 'torch-float16', + dtype: 'torch.float16', + value: { kind: 'ndarray-float16', dtype: 'float16' }, + }, + }); + expect(resolve('torch.FloatTensor')).toMatchObject({ status: 'unresolved' }); + expect(resolve('foreign.HalfTensor')).toMatchObject({ status: 'unresolved' }); + expect(resolve('HalfTensor')).toMatchObject({ status: 'unresolved' }); + }); + + it('rejects a Torch float16 response with a mismatched nested dtype', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'tensor_value', + qualname: 'fixture.tensor_value', + returns: 'torch.Tensor[torch.float16]', + }, + ], + classes: [], + }, + 'Torch contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const tensorType: PythonType = { + kind: 'generic', + name: 'Tensor', + module: 'torch', + typeArgs: [{ kind: 'custom', name: 'float16', module: 'torch' }], + }; + const original = moduleModel.functions[1]!; + const model: PythonModule = { + ...moduleModel, + classes: [], + functions: [ + { + ...original, + name: 'tensor_value', + returnType: tensorType, + signature: { ...original.signature, returnType: tensorType }, + }, + ], + }; + const compiled = compileContract(ir.contract, { + module: model, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.typescript).toContain( + '"marker":"torch.tensor","dtype":"torch.float16"' + ); + + let nestedDtype = 'float16'; + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + id: ++requestId, + result: { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + shape: [], + dtype: 'torch.float16', + device: 'cpu', + value: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + shape: [], + dtype: nestedDtype, + data: 1.5, + }, + }, + }) + ); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-torch-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + tensorValue: () => Promise<{ data: number; dtype: string }>; + }; + await expect(generated.tensorValue()).resolves.toMatchObject({ + data: 1.5, + dtype: 'torch.float16', + }); + nestedDtype = 'float32'; + await expect(generated.tensorValue()).rejects.toThrow(/value\.dtype.*must be "float16"/); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('keeps supported coroutine results and degrades unadapted dataclass outputs', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'dataclass-unsupported', path: '$.functions[3].returns' }), + ]) + ); + expect( + compiled.diagnostics.some(diagnostic => diagnostic.code === 'coroutine-unsupported') + ).toBe(false); + expect(compiled.module.functions[2]?.returnType).toEqual({ kind: 'primitive', name: 'str' }); + expect(compiled.generated.declaration).toContain('asyncValue(): Promise'); + expect(compiled.callables[2]?.requiredCapabilities).toContain('coroutine-execution'); + expect(compiled.module.functions[3]?.returnType).toEqual({ + kind: 'custom', + name: 'Any', + module: 'typing', + }); + }); + + it('degrades coroutine results when the execution capability is unavailable', () => { + const validation = validateIrContract(rawIr, 'fixture contract'); + expect(validation.ok).toBe(true); + if (!validation.ok) { + return; + } + + const compiled = compileContract(validation.contract, { + module: moduleModel, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES.map(capability => + capability.name === 'coroutine-execution' ? { ...capability, available: false } : capability + ), + }); + + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'coroutine-unsupported', path: '$.functions[2].returns' }), + ]) + ); + expect(compiled.module.functions[2]?.returnType).toEqual({ + kind: 'custom', + name: 'Any', + module: 'typing', + }); + expect(compiled.generated.declaration).toContain('asyncValue(): Promise'); + }); + + it('does not promise a TypeScript object for an unconstrained Python object result', async () => { + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'object_value', + qualname: 'fixture.object_value', + returns: 'object', + }, + ], + classes: [], + }, + 'object result contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: [{ ...moduleModel.functions[1]!, name: 'object_value' }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[0].returns' }), + ]) + ); + expect(compiled.generated.declaration).toContain('objectValue(): Promise'); + expect(compiled.generated.typescript).toContain('createReturnValidator({"kind":"any"}'); + + let nestedList: ReturnType | undefined; + for (const annotation of ['list[object]', 'dict[str, object]']) { + const nested = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'object_value', + qualname: 'fixture.object_value', + returns: annotation, + }, + ], + classes: [], + }, + 'nested object result contract' + ); + expect(nested.ok).toBe(true); + if (!nested.ok) { + continue; + } + const nestedCompiled = compileContract(nested.contract, { + module: { + ...moduleModel, + classes: [], + functions: [{ ...moduleModel.functions[1]!, name: 'object_value' }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(nestedCompiled.generated.declaration).toContain('objectValue(): Promise'); + expect(nestedCompiled.generated.typescript).toContain( + annotation.startsWith('list') + ? 'createReturnValidator({"kind":"array","element":{"kind":"any"}}' + : 'createReturnValidator({"kind":"record","values":{"kind":"any"}}' + ); + if (annotation.startsWith('list')) { + nestedList = nestedCompiled; + } + } + + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ id: ++requestId, result: 'a valid Python object' })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-object-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + objectValue: () => Promise; + }; + await expect(generated.objectValue()).resolves.toBe('a valid Python object'); + if (nestedList) { + const nestedPath = join(temporary, 'nested.generated.mjs'); + await writeFile( + nestedPath, + ts.transpileModule(nestedList.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText, + 'utf8' + ); + const nestedGenerated = (await import(pathToFileURL(nestedPath).href)) as { + objectValue: () => Promise; + }; + await expect(nestedGenerated.objectValue()).rejects.toThrow(BridgeValidationError); + } + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('keeps generic inputs but uses unknown for unvalidated type-variable results', async () => { + const source = rawIr.functions[1]!; + const typeVar = { + name: 'T', + kind: 'typevar' as const, + bound: null, + constraints: null, + variance: null, + }; + const generic = { + ...source, + name: 'generic_identity', + qualname: 'fixture.generic_identity', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: '~T', default: false }, + ], + returns: '~T', + type_params: [typeVar], + }; + const nested = { + ...generic, + name: 'generic_nested', + qualname: 'fixture.generic_nested', + parameters: [ + { name: 'value', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'list[~T]', default: false }, + ], + returns: 'list[~T]', + }; + const paramspec = { + ...generic, + name: 'paramspec_apply', + qualname: 'fixture.paramspec_apply', + parameters: [ + { + name: 'callback', + kind: 'POSITIONAL_OR_KEYWORD', + annotation: 'typing.Callable[~P, ~T]', + default: false, + }, + ], + type_params: [ + { name: 'P', kind: 'paramspec' as const, bound: null, constraints: null, variance: null }, + typeVar, + ], + }; + const ir = validateIrContract( + { ...rawIr, functions: [generic, nested, paramspec], classes: [] }, + 'generic result contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: [generic, nested, paramspec].map(func => ({ + ...moduleModel.functions[1]!, + name: func.name, + })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[0].returns' }), + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[1].returns' }), + expect.objectContaining({ code: 'conversion-unresolved', path: '$.functions[2].returns' }), + ]) + ); + expect(compiled.generated.declaration).toContain( + 'genericIdentity(value: T): Promise' + ); + expect(compiled.generated.declaration).toContain( + 'genericNested(value: T[]): Promise' + ); + expect(compiled.generated.declaration).toMatch( + /paramspecApply<[^>]*>\([^\n]*\): Promise/ + ); + + let requestId = 0; + const server = createServer((request, response) => { + request.resume(); + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ id: ++requestId, result: 'not the numeric input' })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const bridge = new HttpBridge({ baseURL: `http://127.0.0.1:${address.port}` }); + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-generic-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + const declarationPath = join(temporary, 'fixture.generated.d.ts'); + const consumerPath = join(temporary, 'consumer.ts'); + await writeFile(declarationPath, compiled.generated.declaration, 'utf8'); + await writeFile( + consumerPath, + "import { genericIdentity } from './fixture.generated.js';\nconst numeric: Promise = genericIdentity(42);\nvoid numeric;\n", + 'utf8' + ); + const program = ts.createProgram([consumerPath], { + noEmit: true, + strict: true, + skipLibCheck: true, + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + types: [], + }); + expect(ts.getPreEmitDiagnostics(program).map(diagnostic => diagnostic.code)).toContain(2322); + setRuntimeBridge(bridge); + const generated = (await import(pathToFileURL(outputPath).href)) as { + genericIdentity: (value: T) => Promise; + }; + await expect(generated.genericIdentity(42)).resolves.toBe('not the numeric input'); + } finally { + clearRuntimeBridge(); + await bridge.dispose(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('keeps existing marker and literal checks when the declaration falls back to unknown', async () => { + const source = rawIr.functions[1]!; + const annotations = [ + { name: 'dataframe_value', returns: 'pandas.DataFrame' }, + { name: 'literal_value', returns: "typing.Literal['x']" }, + ]; + const ir = validateIrContract( + { + ...rawIr, + functions: annotations.map(({ name, returns }) => ({ + ...source, + name, + qualname: `fixture.${name}`, + returns, + })), + classes: [], + }, + 'unresolved validation contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + classes: [], + functions: annotations.map(({ name }) => ({ ...moduleModel.functions[1]!, name })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.declaration).toContain('dataframeValue(): Promise'); + expect(compiled.generated.declaration).toContain('literalValue(): Promise'); + expect(compiled.generated.typescript).toContain('"marker":"dataframe"'); + expect(compiled.generated.typescript).toContain('"kind":"literal","value":"x"'); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-fallback-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = (functionName === 'literal_value' ? 'y' : {}) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + dataframeValue: () => Promise; + literalValue: () => Promise; + }; + await expect(generated.dataframeValue()).rejects.toThrow(BridgeValidationError); + await expect(generated.literalValue()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('resolves selected local TypedDict fields and simple aliases through revision 2', async () => { + const source = rawIr.functions[1]!; + const point = rawIr.classes[0]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { + ...source, + name: 'payload_value', + qualname: 'fixture.payload_value', + returns: 'Payload', + }, + { ...source, name: 'user_id', qualname: 'fixture.user_id', returns: 'UserId' }, + ], + classes: [ + { + ...point, + name: 'Payload', + qualname: 'fixture.Payload', + typed_dict: true, + is_dataclass: false, + fields: [ + { name: 'x', kind: 'FIELD', annotation: 'int', default: false }, + { name: 'label', kind: 'FIELD', annotation: 'str', default: true }, + ], + }, + ], + type_aliases: [{ name: 'UserId', definition: 'int', is_generic: false, type_params: [] }], + }, + 'selected named values contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: ['payload_value', 'user_id'].map(name => ({ + ...moduleModel.functions[1]!, + name, + })), + classes: [{ ...moduleModel.classes[0]!, name: 'Payload' }], + typeAliases: [{ name: 'UserId', type: { kind: 'primitive', name: 'int' } }], + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.diagnostics).toEqual([]); + expect(compiled.generated.declaration).toContain('payloadValue(): Promise'); + expect(compiled.generated.declaration).toContain('userId(): Promise'); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'record', + fields: [ + { name: 'x', required: true, value: { kind: 'integer' } }, + { name: 'label', required: false, value: { kind: 'string' } }, + ], + }, + }); + expect(compiled.callables[1]?.result.resolution).toMatchObject({ + status: 'supported', + value: { kind: 'integer', constraint: 'safe-integer' }, + }); + + const temporary = await mkdtemp(join(process.cwd(), 'test', '.tywrap-named-proof-')); + try { + const outputPath = join(temporary, 'fixture.generated.mjs'); + const javascript = ts.transpileModule(compiled.generated.typescript, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + await writeFile(outputPath, javascript, 'utf8'); + setRuntimeBridge({ + async call( + _module: string, + functionName: string, + _args: unknown[], + _kwargs?: Record, + validate?: (result: T) => void + ): Promise { + const result = ( + functionName === 'user_id' ? Number.MAX_SAFE_INTEGER + 1 : { x: 'bad' } + ) as T; + validate?.(result); + return result; + }, + async dispose(): Promise {}, + }); + const generated = (await import(pathToFileURL(outputPath).href)) as { + payloadValue: () => Promise<{ x: number; label?: string }>; + userId: () => Promise; + }; + await expect(generated.payloadValue()).rejects.toThrow(BridgeValidationError); + await expect(generated.userId()).rejects.toThrow(BridgeValidationError); + } finally { + clearRuntimeBridge(); + await rm(temporary, { recursive: true, force: true }); + } + }); + + it('composes nested local records and aliases, but stops cycles and foreign names', () => { + const source = rawIr.functions[1]!; + const point = rawIr.classes[0]!; + const typedDict = (name: string, annotation: string) => ({ + ...point, + name, + qualname: `fixture.${name}`, + typed_dict: true, + is_dataclass: false, + fields: [{ name: 'value', kind: 'FIELD', annotation, default: false }], + }); + const returns = [ + ['nested_values', 'list[Outer]'], + ['alias_values', 'AliasAlias'], + ['cycle_values', 'Loop'], + ['foreign_values', 'foreign.Outer'], + ['self_values', 'SelfRef'], + ['qualified_values', 'fixture.Outer'], + ] as const; + const ir = validateIrContract( + { + ...rawIr, + functions: returns.map(([name, annotation]) => ({ + ...source, + name, + qualname: `fixture.${name}`, + returns: annotation, + })), + classes: [ + typedDict('Child', 'int'), + typedDict('Outer', 'Child'), + typedDict('SelfRef', 'SelfRef'), + ], + type_aliases: [ + { name: 'UserId', definition: 'int', is_generic: false, type_params: [] }, + { name: 'AliasAlias', definition: 'UserId', is_generic: false, type_params: [] }, + { name: 'Loop', definition: 'Loop', is_generic: false, type_params: [] }, + ], + }, + 'composed named values contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: returns.map(([name]) => ({ ...moduleModel.functions[1]!, name })), + classes: ['Child', 'Outer', 'SelfRef'].map(name => ({ + ...moduleModel.classes[0]!, + name, + })), + typeAliases: ['UserId', 'AliasAlias', 'Loop'].map(name => ({ + name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'supported', + value: { + kind: 'sequence', + item: { + kind: 'record', + fields: [ + { + name: 'value', + value: { + kind: 'record', + fields: [{ name: 'value', value: { kind: 'integer' } }], + }, + }, + ], + }, + }, + }); + expect(compiled.callables[1]?.result.resolution).toMatchObject({ + status: 'supported', + value: { kind: 'integer', constraint: 'safe-integer' }, + }); + expect(compiled.callables[2]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[3]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[4]?.result.resolution.status).toBe('unresolved'); + expect(compiled.callables[5]?.result.resolution.status).toBe('supported'); + expect(compiled.generated.declaration).toContain('nestedValues(): Promise'); + expect(compiled.generated.declaration).toContain('aliasValues(): Promise'); + expect(compiled.generated.declaration).toContain('cycleValues(): Promise'); + expect(compiled.generated.declaration).toContain('foreignValues(): Promise'); + expect(compiled.generated.declaration).toContain('selfValues(): Promise'); + expect(compiled.generated.declaration).toContain('qualifiedValues(): Promise'); + expect(compiled.generated.typescript).toContain( + 'const __validateforeign_valuesResult = createReturnValidator({"kind":"any"}' + ); + + const rejectIntegers: typeof DEFAULT_VALUE_CONVERSION = { + revision: DEFAULT_VALUE_CONVERSION.revision, + resolve(request) { + if (request.logicalType.kind === 'primitive' && request.logicalType.name === 'int') { + return { + status: 'unsupported', + reason: 'This configured conversion declines integers.', + guidance: 'Use a different conversion.', + }; + } + return DEFAULT_VALUE_CONVERSION.resolve(request); + }, + }; + const declined = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: returns.map(([name]) => ({ ...moduleModel.functions[1]!, name })), + classes: ['Child', 'Outer', 'SelfRef'].map(name => ({ + ...moduleModel.classes[0]!, + name, + })), + typeAliases: ['UserId', 'AliasAlias', 'Loop'].map(name => ({ + name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: rejectIntegers, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(declined.callables[0]?.result.resolution.status).toBe('unsupported'); + expect(declined.callables[1]?.result.resolution.status).toBe('unsupported'); + expect(declined.generated.declaration).toContain('nestedValues(): Promise'); + }); + + it('bounds local alias expansion when a converter leaves custom names unresolved', () => { + const aliases = Array.from({ length: 70 }, (_, index) => ({ + name: `Alias${index}`, + definition: index === 69 ? 'External' : `Alias${index + 1}`, + is_generic: false, + type_params: [], + })); + const source = rawIr.functions[1]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [ + { ...source, name: 'alias_chain', qualname: 'fixture.alias_chain', returns: 'Alias0' }, + ], + type_aliases: aliases, + }, + 'bounded alias chain contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const unresolvedCustom: typeof DEFAULT_VALUE_CONVERSION = { + revision: DEFAULT_VALUE_CONVERSION.revision, + resolve(request) { + if (request.logicalType.kind === 'custom') { + return { status: 'unresolved', annotation: request.logicalType.name }; + } + return DEFAULT_VALUE_CONVERSION.resolve(request); + }, + }; + const compiled = compileContract(ir.contract, { + module: { + ...moduleModel, + functions: [{ ...moduleModel.functions[1]!, name: 'alias_chain' }], + classes: [], + typeAliases: aliases.map(alias => ({ + name: alias.name, + type: { kind: 'custom' as const, name: 'Any' }, + })), + }, + generator: new CodeGenerator(), + conversion: unresolvedCustom, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.callables[0]?.result.resolution).toMatchObject({ + status: 'unsupported', + reason: 'Revision 2 limits value contracts to 64 nested nodes.', + }); + }); + + it('treats a variadic integer tuple as the supported sequence contract', () => { + const logicalType = parseAnnotationToPythonType('tuple[int, ...]'); + expect( + DEFAULT_VALUE_CONVERSION.resolve({ + direction: 'output', + path: '$.functions[0].returns', + logicalType, + }) + ).toMatchObject({ + status: 'supported', + value: { + kind: 'sequence', + item: { kind: 'integer', constraint: 'safe-integer' }, + }, + }); + }); + + it('requires returned dataclass fields even when their constructor has defaults', () => { + const point = rawIr.classes[0]!; + const ir = validateIrContract( + { + ...rawIr, + functions: [], + classes: [ + { + ...point, + fields: [{ ...point.fields[0]!, default: true }, point.fields[1]!], + }, + ], + }, + 'Point default contract' + ); + expect(ir.ok).toBe(true); + if (!ir.ok) { + return; + } + const compiled = compileContract(ir.contract, { + module: { ...moduleModel, functions: [] }, + generator: new CodeGenerator(), + conversion: DEFAULT_VALUE_CONVERSION, + capabilities: DEFAULT_CALLABLE_CAPABILITIES, + }); + expect(compiled.generated.declaration).toContain('x: number;'); + expect(compiled.generated.declaration).not.toContain('x?: number;'); + }); +}); diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts new file mode 100644 index 00000000..55f756ef --- /dev/null +++ b/test/decoded_provenance.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; + +describe('decoded provenance', () => { + it('keeps equal scalar values separate by call and nested location', () => { + const firstParent = { one: 1.5, two: 1.5 }; + const secondParent = { one: 1.5 }; + const first = new DecodedProvenance(); + const second = new DecodedProvenance(); + + first.recordChild(firstParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + first.recordChild(firstParent, 'two', { marker: 'ndarray', dims: 0, dtype: 'float32' }); + second.recordChild(secondParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float64' }); + + expect(first.atChild(firstParent, 'one')?.dtype).toBe('float16'); + expect(first.atChild(firstParent, 'two')?.dtype).toBe('float32'); + expect(first.atChild(secondParent, 'one')).toBeUndefined(); + expect(second.atChild(firstParent, 'one')).toBeUndefined(); + expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); + }); + + it('uses the same array slot for a numeric or string index', () => { + const parent = [1.5]; + const provenance = new DecodedProvenance(); + const metadata = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + provenance.recordChild(parent, 0, metadata); + expect(provenance.atChild(parent, '0')).toEqual(metadata); + expect(() => provenance.recordChild(parent, '0', metadata)).toThrow(/Duplicate child/); + }); + + it('has one root slot and a bounded number of entries', () => { + const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + const provenance = new DecodedProvenance(2); + const parent = [1.5]; + provenance.recordRoot(evidence); + provenance.recordChild(parent, 0, evidence); + + expect(provenance.atRoot()).toEqual(evidence); + expect(provenance.atChild(parent, 0)).toEqual(evidence); + expect(() => provenance.recordRoot(evidence)).toThrow(/Duplicate root/); + expect(() => provenance.recordChild(parent, 0, evidence)).toThrow(/Duplicate child/); + expect(() => provenance.recordChild(parent, 1, evidence)).toThrow(/exceeds 2 entries/); + }); +}); diff --git a/test/generator.test.ts b/test/generator.test.ts index e23e4dfd..571ec3af 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -59,6 +59,76 @@ describe('CodeGenerator', () => { expect(code.typescript).toContain('createReturnValidator'); }); + it('uses one opt-in runtime provider for free and static wrappers', () => { + const ping = { + name: 'ping', + signature: { + parameters: [], + returnType: { kind: 'primitive', name: 'int' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType: { kind: 'primitive', name: 'int' }, + parameters: [], + } as any; + const module = { + name: 'binding_fixture', + functions: [ping], + classes: [ + { + name: 'Box', + kind: 'class', + bases: [], + properties: [], + decorators: [], + methods: [{ ...ping, name: 'pong', methodKind: 'static' }], + }, + ], + typeAliases: [], + imports: [], + exports: [], + } as any; + const normal = gen.generateModuleDefinition(module); + const bound = gen.generateModuleBindingTemplate(module); + expect(normal.typescript).toContain( + "getRuntimeBridge().call('binding_fixture', 'ping'" + ); + expect(normal.typescript).toContain( + "getRuntimeBridge().call('binding_fixture', 'Box.pong'" + ); + expect(bound.typescript).toContain( + "__tywrapRuntimeProvider().call('binding_fixture', 'ping'" + ); + expect(bound.typescript).toContain( + "__tywrapRuntimeProvider().call('binding_fixture', 'Box.pong'" + ); + expect(bound.typescript).not.toContain('getRuntimeBridge'); + expect(bound.declaration).toBe(normal.declaration); + for (const identifier of [ + 'ping', + 'provider', + 'createReturnValidator', + '__tywrapReturnDefinitions', + '__args', + 'for', + 'super', + 'this', + 'arguments', + 'undefined', + 'provider;inject', + ]) { + expect(() => gen.generateModuleBindingTemplate(module, false, identifier)).toThrow( + /Invalid runtime getter identifier/ + ); + } + expect( + gen.generateModuleBindingTemplate(module, false, '__tywrapRuntimeProvider1').typescript + ).toContain('__tywrapRuntimeProvider1().call'); + }); + it('emits a null union member for X | None returns, not an accept-everything any', () => { const code = gen.generateFunctionWrapper( { @@ -561,6 +631,153 @@ describe('CodeGenerator', () => { ); }); + it('preserves declared overload input-to-return relationships', () => { + const code = gen.generateFunctionWrapper( + { + name: 'get_value', + signature: { + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + parameters: [ + { + name: 'key', + type: { + kind: 'union', + types: [ + { kind: 'primitive', name: 'str' }, + { kind: 'primitive', name: 'int' }, + ], + }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + overloads: [ + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'str' }, + }, + { + parameters: [ + { + name: 'key', + type: { kind: 'primitive', name: 'int' }, + optional: false, + varArgs: false, + kwArgs: false, + }, + ], + returnType: { kind: 'primitive', name: 'int' }, + }, + ], + } as any, + 'advanced_types' + ); + + expect(code.typescript).toContain('export function getValue(key: string): Promise;'); + expect(code.typescript).toContain('export function getValue(key: number): Promise;'); + expect(code.typescript).toContain( + 'export async function getValue(key: unknown): Promise' + ); + expect(code.declaration).toContain('export function getValue(key: string): Promise;'); + expect(code.declaration).toContain('export function getValue(key: number): Promise;'); + }); + + it('does not expose the implementation signature beside declared overloads', () => { + const key = { + name: 'key', + type: { kind: 'primitive', name: 'str' }, + optional: false, + varArgs: false, + kwArgs: false, + }; + const fallback = { + name: 'fallback', + type: { kind: 'primitive', name: 'int' }, + optional: true, + varArgs: false, + kwArgs: false, + }; + const method = { + name: 'get_value', + signature: { + parameters: [key, fallback], + returnType: { kind: 'primitive', name: 'str' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + parameters: [key, fallback], + returnType: { kind: 'primitive', name: 'str' }, + methodKind: 'static', + overloads: [{ parameters: [key], returnType: { kind: 'primitive', name: 'str' } }], + } as any; + + const functionCode = gen.generateFunctionWrapper(method, 'fixture'); + expect(functionCode.declaration).toContain( + 'export function getValue(key: string): Promise;' + ); + expect(functionCode.declaration).not.toContain('fallback'); + + const classCode = gen.generateClassWrapper( + { + name: 'Catalog', + bases: [], + methods: [method], + properties: [], + decorators: [], + kind: 'class', + }, + 'fixture' + ); + expect(classCode.declaration).toContain('static getValue(key: string): Promise;'); + expect(classCode.declaration).not.toContain('fallback'); + }); + it('models *args as an array parameter when kwargs are present', () => { const code = gen.generateFunctionWrapper( { diff --git a/test/integration.test.ts b/test/integration.test.ts index 659f67c2..e97d85e6 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -224,6 +224,78 @@ describe('IR-only integration', () => { { cwd: process.cwd(), timeoutMs: 30_000 } ); + expect(compile.code, `${compile.stdout}\n${compile.stderr}`).toBe(0); + expect(compile.stderr).toBe(''); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); + + it('preserves Python overload input-to-return relationships in generated declarations', async () => { + const support = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (support.code !== 0) { + return; + } + const tempDir = await mkdtemp(join(process.cwd(), 'test', '.tywrap-overload-contract-')); + try { + const outDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { advanced_types: { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: ['test/fixtures/python'], + output: { dir: outDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as any); + + expect(result.failures).toEqual([]); + const generated = await fsUtils.readFile(join(outDir, 'advanced_types.generated.d.ts')); + expect(generated).toContain('export function getValue(key: string): Promise;'); + expect(generated).toContain('export function getValue(key: number): Promise;'); + + const consumerPath = join(tempDir, 'consumer.ts'); + await writeFile( + consumerPath, + `import { getValue } from './generated/advanced_types.generated.js'; + +const text: Promise = getValue('key'); +const integer: Promise = getValue(1); +// @ts-expect-error boolean has no declared Python overload +void getValue(true); +// @ts-expect-error string overload cannot return Promise +const wrong: Promise = getValue('key'); + +void text; +void integer; +void wrong; +`, + 'utf8' + ); + const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); + const compile = await processUtils.exec( + process.execPath, + [ + tscPath, + '--ignoreConfig', + '--noEmit', + '--pretty', + 'false', + '--target', + 'ES2022', + '--lib', + 'ES2022,DOM,DOM.Iterable', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck', + consumerPath, + ], + { cwd: process.cwd(), timeoutMs: 30_000 } + ); + expect(compile.code).toBe(0); expect(compile.stderr).toBe(''); } finally { @@ -231,6 +303,90 @@ describe('IR-only integration', () => { } }, 30_000); + it('preserves type variables declared only by overload signatures', async () => { + const support = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (support.code !== 0) { + return; + } + const tempDir = await mkdtemp(join(process.cwd(), 'test', '.tywrap-generic-overload-')); + try { + const importDir = join(tempDir, 'py'); + await mkdir(importDir, { recursive: true }); + await writeFile( + join(importDir, 'generic_overload.py'), + `from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def choose(value: T) -> T: ... + +@overload +def choose(value: None) -> None: ... + +def choose(value: object) -> object: + return value +`, + 'utf8' + ); + const outDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { generic_overload: { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: [importDir], + output: { dir: outDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as any); + expect(result.failures).toEqual([]); + const declaration = await fsUtils.readFile(join(outDir, 'generic_overload.generated.d.ts')); + expect(declaration).toContain('export function choose(value: T): Promise;'); + const consumerPath = join(tempDir, 'consumer.ts'); + await writeFile( + consumerPath, + `import { choose } from './generated/generic_overload.generated.js'; + +const text: Promise = choose('key'); +const integer: Promise = choose(2); +// @ts-expect-error An unresolved TypeVar does not validate the decoded result. +const unsafe: Promise = choose('key'); +void text; +void integer; +void unsafe; +`, + 'utf8' + ); + const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); + const compile = await processUtils.exec( + process.execPath, + [ + tscPath, + '--ignoreConfig', + '--noEmit', + '--pretty', + 'false', + '--target', + 'ES2022', + '--lib', + 'ES2022,DOM,DOM.Iterable', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck', + consumerPath, + ], + { cwd: process.cwd(), timeoutMs: 30_000 } + ); + expect(compile.code, `${compile.stdout}\n${compile.stderr}`).toBe(0); + expect(compile.stderr).toBe(''); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); + it('generates safe generic wrappers and declaration files that typecheck', async () => { if (!(await supportsVariadicTypingFeatures())) { return; @@ -302,9 +458,9 @@ class Container(Generic[T]): const typescript = await fsUtils.readFile(join(outDir, 'generic_module.generated.ts')); const declaration = await fsUtils.readFile(join(outDir, 'generic_module.generated.d.ts')); - expect(typescript).toContain('export function identity(x: T): Promise;'); + expect(typescript).toContain('export function identity(x: T): Promise;'); expect(typescript).toContain( - 'export async function forward(container: Container): Promise>' + 'export async function forward(container: Container): Promise' ); expect(typescript).toContain( 'export async function acceptTransform

(' @@ -354,9 +510,11 @@ export declare function createReturnValidator(schema: ReturnSchema, const pair: Pair = ['a', 'b']; const transform: Transform<[number], string> = (...args) => String(args[0]); const container = {} as Container; -const accepted: Promise> = acceptTransform<[number], string>(transform); -const forwarded: Promise> = forward(container); -const resolved: Promise = identity(1); +const accepted: Promise = acceptTransform<[number], string>(transform); +const forwarded: Promise = forward(container); +const resolved: Promise = identity(1); +// @ts-expect-error The decoded generic result has no supported value contract. +const unsafe: Promise = identity(1); const passthroughResult: Promise = passthrough([1, 2], { flag: true }); void pair; @@ -364,6 +522,7 @@ void transform; void accepted; void forwarded; void resolved; +void unsafe; void passthroughResult; `, 'utf-8' diff --git a/test/ir-contract-validation.test.ts b/test/ir-contract-validation.test.ts new file mode 100644 index 00000000..2913b432 --- /dev/null +++ b/test/ir-contract-validation.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest'; +import { TYWRAP_IR_VERSION, validateIrContract } from '../src/core/ir-contract.js'; + +const validContract = { + ir_version: TYWRAP_IR_VERSION, + module: 'fixture', + functions: [ + { + name: 'get_value', + qualname: 'fixture.get_value', + docstring: null, + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str | int', default: false }, + ], + returns: 'str | int', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [ + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'str', default: false }, + ], + returns: 'str', + }, + { + parameters: [ + { name: 'key', kind: 'POSITIONAL_OR_KEYWORD', annotation: 'int', default: false }, + ], + returns: 'int', + }, + ], + }, + ], + classes: [], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [], +}; + +describe('validateIrContract', () => { + it('accepts the complete IR 0.4 callable shape, including overloads', () => { + const result = validateIrContract(validContract, 'fixture contract'); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.contract.functions[0]?.overloads).toHaveLength(2); + } + }); + + it('reports each malformed nested overload field at its JSON path', () => { + const invalid = structuredClone(validContract); + invalid.functions[0]!.overloads[1]!.parameters[0]!.kind = 'MALFORMED'; + invalid.functions[0]!.overloads[1]!.returns = 42 as unknown as string; + + const result = validateIrContract(invalid, 'fixture contract'); + + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'contract-invalid', + path: '$.functions[0].overloads[1].parameters[0].kind', + }), + expect.objectContaining({ + code: 'contract-invalid', + path: '$.functions[0].overloads[1].returns', + }), + ]) + ); + } + }); + + it('keeps IR version mismatch separate from structural diagnostics', () => { + const invalid = { ...validContract, ir_version: '0.3.0' }; + const result = validateIrContract(invalid, 'fixture contract'); + + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics).toContainEqual( + expect.objectContaining({ code: 'ir-version-mismatch', path: '$.ir_version' }) + ); + } + }); + + it('reads stable offline contracts without extractor metadata', () => { + const offline = Object.fromEntries( + Object.entries(validContract).filter(([key]) => key !== 'metadata') + ); + const live = validateIrContract(offline, 'live IR'); + expect(live).toMatchObject({ ok: false }); + + const saved = validateIrContract(offline, 'saved contract', { + allowOmittedMetadata: true, + }); + expect(saved.ok).toBe(true); + if (saved.ok) { + expect(saved.contract.metadata).toEqual({}); + } + }); + + it('bounds malformed-contract diagnostics', () => { + const invalid = { + ...validContract, + functions: Array.from({ length: 120 }, () => ({ name: 2 })), + }; + const result = validateIrContract(invalid, 'bad contract'); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect(result.diagnostics.length).toBeLessThanOrEqual(101); + expect(result.diagnostics.at(-1)?.message).toContain('more than 100 contract errors'); + } + }); + + it('rejects contracts that exceed the shared entry traversal limit', () => { + const oversized = { ...validContract, warnings: Array(100_001).fill('warning') }; + const result = validateIrContract(oversized, 'large contract'); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) { + expect( + result.diagnostics.some(diagnostic => + diagnostic.message.includes('100000 entry validation limit') + ) + ).toBe(true); + } + }); +}); diff --git a/test/ir_contract.test.ts b/test/ir_contract.test.ts index cf57e5da..0f005bdf 100644 --- a/test/ir_contract.test.ts +++ b/test/ir_contract.test.ts @@ -96,6 +96,93 @@ describe('pinned IR contracts', () => { } }); + it('keeps a supported NumPy float16 contract strict in offline and check modes', async () => { + const tempDir = await mkdtemp(join(tmpdir(), 'tywrap-ir-float16-')); + try { + const contractPath = join(tempDir, 'scientific.contract.json'); + await writeFile( + contractPath, + JSON.stringify({ + ir_version: '0.4.0', + module: 'scientific_fixture', + functions: [ + { + name: 'value', + qualname: 'scientific_fixture.value', + docstring: null, + parameters: [], + returns: 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]', + is_async: false, + is_generator: false, + type_params: [], + method_kind: 'instance', + overloads: [], + }, + ], + classes: [], + constants: [], + type_aliases: [], + metadata: {}, + warnings: [ + 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.ndarray.', + ], + }), + 'utf8' + ); + const configured = { + ...options(join(tempDir, 'generated')), + pythonModules: { scientific_fixture: { typeHints: 'strict' as const } }, + contractInput: contractPath, + }; + const first = await generate(configured); + expect(first.failures).toEqual([]); + expect(first.warnings).toEqual([]); + const emitted = await readFile( + join(tempDir, 'generated', 'scientific_fixture.generated.ts'), + 'utf8' + ); + expect(emitted).toContain('value(): Promise<__tywrapFloat16Value>'); + expect(emitted).toContain('"marker":"ndarray","dtype":"float16"'); + const check = await generate(configured, { check: true }); + expect(check.failures).toEqual([]); + expect(check.warnings).toEqual([]); + expect(check.outOfDate).toEqual([]); + + const foreign = JSON.parse(await readFile(contractPath, 'utf8')) as { + functions: Array<{ returns: string }>; + warnings: string[]; + }; + foreign.functions[0]!.returns = + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[external.float16]]'; + await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); + const rejected = await generate(configured); + expect( + rejected.warnings.some(warning => + warning.includes('no value conversion is defined for numpy.ndarray') + ) + ).toBe(true); + expect( + rejected.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.ndarray') + ) + ).toBe(true); + + foreign.functions[0]!.returns = + 'numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.float16]]'; + foreign.warnings[0] = + 'Return annotation for scientific_fixture.value resolves outside analyzed module: numpy.Opaque.'; + await writeFile(contractPath, JSON.stringify(foreign), 'utf8'); + const unrelated = await generate(configured); + expect( + unrelated.warnings.some(warning => + warning.includes('resolves outside analyzed module: numpy.Opaque') + ) + ).toBe(true); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + // Windows cannot run the fake-interpreter shim: extensionless sh scripts // fail with ENOENT and .cmd files fail with EINVAL (Node's batch-file spawn // mitigation; the production spawn rightly never sets shell:true). The diff --git a/test/ir_extraction_advanced.test.ts b/test/ir_extraction_advanced.test.ts index a3ddb785..0e2804f5 100644 --- a/test/ir_extraction_advanced.test.ts +++ b/test/ir_extraction_advanced.test.ts @@ -209,6 +209,39 @@ describe('IR Extraction - Complex Fixture Files', () => { }); describe('IR Extraction - Generic Metadata', () => { + it('keeps type parameters declared only in overload signatures', async () => { + const version = await processUtils.exec(PYTHON_EXECUTABLE, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + if (version.code !== 0) { + return; + } + + createTempModule( + 'overload_only_generic', + `from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def choose(value: T) -> T: ... + +@overload +def choose(value: None) -> None: ... + +def choose(value: object) -> object: + return value +` + ); + + const ir = await extractIR('overload_only_generic'); + const choose = ir.functions.find((item: { name: string }) => item.name === 'choose'); + expect(choose.type_params.map((param: { name: string }) => param.name)).toEqual(['T']); + expect(choose.overloads).toHaveLength(2); + expect(choose.overloads[0].parameters[0].annotation).toMatch(/T$/); + }); + it('extracts ordered type parameters for functions, classes, and type aliases', async () => { if (!(await supportsVariadicTypingFeatures())) { return; diff --git a/test/ir_version_drift.test.ts b/test/ir_version_drift.test.ts index a929532b..00f96f21 100644 --- a/test/ir_version_drift.test.ts +++ b/test/ir_version_drift.test.ts @@ -9,7 +9,7 @@ import { describe, expect, it } from 'vitest'; * so this test runs without a Python interpreter and without pulling tywrap.ts' * dependency graph into the test process. The single sources are: * - Python: tywrap_ir/tywrap_ir/__init__.py (IR_VERSION = "...") - * - TypeScript: src/tywrap.ts (const TYWRAP_IR_VERSION = '...') + * - TypeScript: src/core/ir-contract.ts (const TYWRAP_IR_VERSION = '...') */ const repoRoot = process.cwd(); @@ -32,7 +32,7 @@ describe('IR_VERSION drift', () => { 'IR_VERSION' ); const tsVersion = readVersion( - join('src', 'tywrap.ts'), + join('src', 'core', 'ir-contract.ts'), /\bTYWRAP_IR_VERSION\s*=\s*["']([^"']+)["']/, 'TYWRAP_IR_VERSION' ); diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 36295109..6b2979da 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -10,7 +10,7 @@ import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywr const __tywrapReturnDefinitions: Record = {"Movie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false}}},"RatedMovie":{"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number"},"optional":true}}}}; -const __validateannotated_echoResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.typing_torture.annotated_echo", __tywrapReturnDefinitions); +const __validateannotated_echoResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.typing_torture.annotated_echo", __tywrapReturnDefinitions); export async function annotatedEcho(value: number): Promise { const __args: unknown[] = [value]; @@ -47,16 +47,16 @@ export async function generatorValues(): Promise { const __validategeneric_identityResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.generic_identity", __tywrapReturnDefinitions); -export async function genericIdentity(value: T): Promise { +export async function genericIdentity(value: T): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'generic_identity', __args, undefined, __validategeneric_identityResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'generic_identity', __args, undefined, __validategeneric_identityResult); } -const __validateint_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.int_key_dict", __tywrapReturnDefinitions); +const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.int_key_dict", __tywrapReturnDefinitions); -export async function intKeyDict(value: { [key: number]: string; }): Promise<{ [key: number]: string; }> { +export async function intKeyDict(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call<{ [key: number]: string; }>('fixtures.typing_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); } const __validateiterator_valuesResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.iterator_values", __tywrapReturnDefinitions); @@ -68,42 +68,39 @@ export async function iteratorValues(): Promise { const __validateliteral_echoResult = createReturnValidator({"kind":"union","options":[{"kind":"literal","value":"red"},{"kind":"literal","value":"blue"},{"kind":"literal","value":7}]}, "fixtures.typing_torture.literal_echo", __tywrapReturnDefinitions); -export async function literalEcho(value: "red" | "blue" | 7): Promise<"red" | "blue" | 7> { +export async function literalEcho(value: "red" | "blue" | 7): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call<"red" | "blue" | 7>('fixtures.typing_torture', 'literal_echo', __args, undefined, __validateliteral_echoResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'literal_echo', __args, undefined, __validateliteral_echoResult); } -const __validatenested_containersResult = createReturnValidator({"kind":"array","element":{"kind":"record","values":{"kind":"array","element":{"kind":"tuple","elements":[{"kind":"primitive","type":"number"},{"kind":"primitive","type":"string"}]}}}}, "fixtures.typing_torture.nested_containers", __tywrapReturnDefinitions); +const __validatenested_containersResult = createReturnValidator({"kind":"array","element":{"kind":"record","fields":{},"values":{"kind":"array","element":{"kind":"tuple","elements":[{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"string"}]}}}}, "fixtures.typing_torture.nested_containers", __tywrapReturnDefinitions); export async function nestedContainers(value: Record[]): Promise[]> { const __args: unknown[] = [value]; return getRuntimeBridge().call[]>('fixtures.typing_torture', 'nested_containers', __args, undefined, __validatenested_containersResult); } -const __validatenone_returnResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.none_return", __tywrapReturnDefinitions); +const __validatenone_returnResult = createReturnValidator({"kind":"primitive","type":"null"}, "fixtures.typing_torture.none_return", __tywrapReturnDefinitions); export async function noneReturn(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.typing_torture', 'none_return', __args, undefined, __validatenone_returnResult); } -const __validateoptional_unionResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"string"},{"kind":"primitive","type":"number"},{"kind":"primitive","type":"null"}]}, "fixtures.typing_torture.optional_union", __tywrapReturnDefinitions); +const __validateoptional_unionResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"string"},{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"null"}]}, "fixtures.typing_torture.optional_union", __tywrapReturnDefinitions); export async function optionalUnion(value: string | number | null): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'optional_union', __args, undefined, __validateoptional_unionResult); } -const __validateoverloadedResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"number"},{"kind":"primitive","type":"string"}]}, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); +const __validateoverloadedResult = createReturnValidator({"kind":"union","options":[{"kind":"primitive","type":"number","constraint":"safe-integer"},{"kind":"primitive","type":"string"}]}, "fixtures.typing_torture.overloaded", __tywrapReturnDefinitions); -export async function overloaded(value: number | string): Promise { - const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'overloaded', __args, undefined, __validateoverloadedResult); -} +/* overload implementation checked separately */ const __validateparamspec_applyResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.paramspec_apply", __tywrapReturnDefinitions); -export async function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise { +export async function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise { let __kwargs = kwargs; const __args: unknown[] = [callback]; let __varargs: unknown[] = []; @@ -117,52 +114,52 @@ export async function paramspecApply

(callback: (...args: } } __args.push(...__varargs); - return getRuntimeBridge().call('fixtures.typing_torture', 'paramspec_apply', __args, __kwargs, __validateparamspec_applyResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'paramspec_apply', __args, __kwargs, __validateparamspec_applyResult); } const __validateprotocol_identityResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.protocol_identity", __tywrapReturnDefinitions); -export async function protocolIdentity(value: SupportsClose): Promise { +export async function protocolIdentity(value: SupportsClose): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call('fixtures.typing_torture', 'protocol_identity', __args, undefined, __validateprotocol_identityResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'protocol_identity', __args, undefined, __validateprotocol_identityResult); } -const __validaterated_movieResult = createReturnValidator({"kind":"ref","name":"RatedMovie"}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); +const __validaterated_movieResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false},"rating":{"schema":{"kind":"primitive","type":"number","constraint":"finite"},"optional":true}}}, "fixtures.typing_torture.rated_movie", __tywrapReturnDefinitions); export async function ratedMovie(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.typing_torture', 'rated_movie', __args, undefined, __validaterated_movieResult); } -const __validateset_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.typing_torture.set_values", __tywrapReturnDefinitions); +const __validateset_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.typing_torture.set_values", __tywrapReturnDefinitions); export async function setValues(value: number[]): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'set_values', __args, undefined, __validateset_valuesResult); } -const __validatetuple_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.typing_torture.tuple_key_dict", __tywrapReturnDefinitions); +const __validatetuple_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.tuple_key_dict", __tywrapReturnDefinitions); -export async function tupleKeyDict(value: Record): Promise> { +export async function tupleKeyDict(value: unknown): Promise { const __args: unknown[] = [value]; - return getRuntimeBridge().call>('fixtures.typing_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); + return getRuntimeBridge().call('fixtures.typing_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); } -const __validatetyped_dict_echoResult = createReturnValidator({"kind":"ref","name":"Movie"}, "fixtures.typing_torture.typed_dict_echo", __tywrapReturnDefinitions); +const __validatetyped_dict_echoResult = createReturnValidator({"kind":"record","fields":{"title":{"schema":{"kind":"primitive","type":"string"},"optional":false},"year":{"schema":{"kind":"primitive","type":"number","constraint":"safe-integer"},"optional":false}}}, "fixtures.typing_torture.typed_dict_echo", __tywrapReturnDefinitions); export async function typedDictEcho(value: Movie): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'typed_dict_echo', __args, undefined, __validatetyped_dict_echoResult); } -const __validateuser_id_echoResult = createReturnValidator({"kind":"any"}, "fixtures.typing_torture.user_id_echo", __tywrapReturnDefinitions); +const __validateuser_id_echoResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.typing_torture.user_id_echo", __tywrapReturnDefinitions); export async function userIdEcho(value: UserId): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'user_id_echo', __args, undefined, __validateuser_id_echoResult); } -const __validatevariadic_tupleResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.typing_torture.variadic_tuple", __tywrapReturnDefinitions); +const __validatevariadic_tupleResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.typing_torture.variadic_tuple", __tywrapReturnDefinitions); export async function variadicTuple(value: number[]): Promise { const __args: unknown[] = [value]; @@ -197,13 +194,13 @@ export function frozenValues(value: string[]): Promise; export function generatorValues(): Promise; -export function genericIdentity(value: T): Promise; +export function genericIdentity(value: T): Promise; -export function intKeyDict(value: { [key: number]: string; }): Promise<{ [key: number]: string; }>; +export function intKeyDict(value: unknown): Promise; export function iteratorValues(): Promise; -export function literalEcho(value: "red" | "blue" | 7): Promise<"red" | "blue" | 7>; +export function literalEcho(value: "red" | "blue" | 7): Promise; export function nestedContainers(value: Record[]): Promise[]>; @@ -211,17 +208,17 @@ export function noneReturn(): Promise; export function optionalUnion(value: string | number | null): Promise; -export function overloaded(value: number | string): Promise; +/* overload declarations checked separately */ -export function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; +export function paramspecApply

(callback: (...args: P) => T, args?: unknown[], kwargs?: Record): Promise; -export function protocolIdentity(value: SupportsClose): Promise; +export function protocolIdentity(value: SupportsClose): Promise; export function ratedMovie(): Promise; export function setValues(value: number[]): Promise; -export function tupleKeyDict(value: Record): Promise>; +export function tupleKeyDict(value: unknown): Promise; export function typedDictEcho(value: Movie): Promise; @@ -256,7 +253,7 @@ import { createReturnValidator, getRuntimeBridge, type ReturnSchema } from 'tywr const __tywrapReturnDefinitions: Record = {}; -const __validatebools_and_intsResult = createReturnValidator({"kind":"array","element":{"kind":"union","options":[{"kind":"primitive","type":"boolean"},{"kind":"primitive","type":"number"}]}}, "fixtures.values_torture.bools_and_ints", __tywrapReturnDefinitions); +const __validatebools_and_intsResult = createReturnValidator({"kind":"array","element":{"kind":"union","options":[{"kind":"primitive","type":"boolean"},{"kind":"primitive","type":"number","constraint":"safe-integer"}]}}, "fixtures.values_torture.bools_and_ints", __tywrapReturnDefinitions); export async function boolsAndInts(): Promise { const __args: unknown[] = []; @@ -286,26 +283,26 @@ export async function coroutineValue(): Promise { const __validatedataclass_instanceResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.dataclass_instance", __tywrapReturnDefinitions); -export async function dataclassInstance(): Promise { +export async function dataclassInstance(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'dataclass_instance', __args, undefined, __validatedataclass_instanceResult); + return getRuntimeBridge().call('fixtures.values_torture', 'dataclass_instance', __args, undefined, __validatedataclass_instanceResult); } const __validatedecimal_valuesResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.decimal_values", __tywrapReturnDefinitions); -export async function decimalValues(): Promise { +export async function decimalValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args, undefined, __validatedecimal_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args, undefined, __validatedecimal_valuesResult); } const __validatedeeply_nestedResult = createReturnValidator({"kind":"array","element":{"kind":"any"}}, "fixtures.values_torture.deeply_nested", __tywrapReturnDefinitions); -export async function deeplyNested(): Promise { +export async function deeplyNested(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'deeply_nested', __args, undefined, __validatedeeply_nestedResult); + return getRuntimeBridge().call('fixtures.values_torture', 'deeply_nested', __args, undefined, __validatedeeply_nestedResult); } -const __validateecho_intResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.echo_int", __tywrapReturnDefinitions); +const __validateecho_intResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.echo_int", __tywrapReturnDefinitions); export async function echoInt(value: number): Promise { const __args: unknown[] = [value]; @@ -314,9 +311,9 @@ export async function echoInt(value: number): Promise { const __validateempty_valuesResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"tuple","elements":[{"kind":"any"}]},{"kind":"array","element":{"kind":"any"}},{"kind":"record","values":{"kind":"any"}},{"kind":"array","element":{"kind":"any"}}]}, "fixtures.values_torture.empty_values", __tywrapReturnDefinitions); -export async function emptyValues(): Promise<[[unknown], object[], Record, object[]]> { +export async function emptyValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call<[[unknown], object[], Record, object[]]>('fixtures.values_torture', 'empty_values', __args, undefined, __validateempty_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'empty_values', __args, undefined, __validateempty_valuesResult); } const __validateenum_memberResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.enum_member", __tywrapReturnDefinitions); @@ -326,7 +323,7 @@ export async function enumMember(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'enum_member', __args, undefined, __validateenum_memberResult); } -const __validatefinite_float_edgesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.finite_float_edges", __tywrapReturnDefinitions); +const __validatefinite_float_edgesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"finite"}}, "fixtures.values_torture.finite_float_edges", __tywrapReturnDefinitions); export async function finiteFloatEdges(): Promise { const __args: unknown[] = []; @@ -335,61 +332,61 @@ export async function finiteFloatEdges(): Promise { const __validategenerator_valueResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.generator_value", __tywrapReturnDefinitions); -export async function generatorValue(): Promise { +export async function generatorValue(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call('fixtures.values_torture', 'generator_value', __args, undefined, __validategenerator_valueResult); + return getRuntimeBridge().call('fixtures.values_torture', 'generator_value', __args, undefined, __validategenerator_valueResult); } -const __validateint_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.values_torture.int_key_dict", __tywrapReturnDefinitions); +const __validateint_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.int_key_dict", __tywrapReturnDefinitions); -export async function intKeyDict(): Promise<{ [key: number]: string; }> { +export async function intKeyDict(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call<{ [key: number]: string; }>('fixtures.values_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); + return getRuntimeBridge().call('fixtures.values_torture', 'int_key_dict', __args, undefined, __validateint_key_dictResult); } -const __validateinteger_boundariesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.integer_boundaries", __tywrapReturnDefinitions); +const __validateinteger_boundariesResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}}, "fixtures.values_torture.integer_boundaries", __tywrapReturnDefinitions); export async function integerBoundaries(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_boundaries', __args, undefined, __validateinteger_boundariesResult); } -const __validateinteger_factorial_30Result = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_factorial_30", __tywrapReturnDefinitions); +const __validateinteger_factorial_30Result = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_factorial_30", __tywrapReturnDefinitions); export async function integerFactorial_30(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_factorial_30', __args, undefined, __validateinteger_factorial_30Result); } -const __validateinteger_first_roundedResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_first_rounded", __tywrapReturnDefinitions); +const __validateinteger_first_roundedResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_first_rounded", __tywrapReturnDefinitions); export async function integerFirstRounded(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_first_rounded', __args, undefined, __validateinteger_first_roundedResult); } -const __validateinteger_first_unsafeResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_first_unsafe", __tywrapReturnDefinitions); +const __validateinteger_first_unsafeResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_first_unsafe", __tywrapReturnDefinitions); export async function integerFirstUnsafe(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_first_unsafe', __args, undefined, __validateinteger_first_unsafeResult); } -const __validateinteger_int64_maxResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_int64_max", __tywrapReturnDefinitions); +const __validateinteger_int64_maxResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_int64_max", __tywrapReturnDefinitions); export async function integerInt64Max(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_int64_max', __args, undefined, __validateinteger_int64_maxResult); } -const __validateinteger_int64_minResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_int64_min", __tywrapReturnDefinitions); +const __validateinteger_int64_minResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_int64_min", __tywrapReturnDefinitions); export async function integerInt64Min(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'integer_int64_min', __args, undefined, __validateinteger_int64_minResult); } -const __validateinteger_safe_maxResult = createReturnValidator({"kind":"primitive","type":"number"}, "fixtures.values_torture.integer_safe_max", __tywrapReturnDefinitions); +const __validateinteger_safe_maxResult = createReturnValidator({"kind":"primitive","type":"number","constraint":"safe-integer"}, "fixtures.values_torture.integer_safe_max", __tywrapReturnDefinitions); export async function integerSafeMax(): Promise { const __args: unknown[] = []; @@ -410,14 +407,14 @@ export async function megabyteText(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'megabyte_text', __args, undefined, __validatemegabyte_textResult); } -const __validateset_and_frozensetResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"array","element":{"kind":"primitive","type":"number"}},{"kind":"array","element":{"kind":"primitive","type":"string"}}]}, "fixtures.values_torture.set_and_frozenset", __tywrapReturnDefinitions); +const __validateset_and_frozensetResult = createReturnValidator({"kind":"tuple","elements":[{"kind":"array","element":{"kind":"primitive","type":"number","constraint":"safe-integer"}},{"kind":"array","element":{"kind":"primitive","type":"string"}}]}, "fixtures.values_torture.set_and_frozenset", __tywrapReturnDefinitions); export async function setAndFrozenset(): Promise<[number[], string[]]> { const __args: unknown[] = []; return getRuntimeBridge().call<[number[], string[]]>('fixtures.values_torture', 'set_and_frozenset', __args, undefined, __validateset_and_frozensetResult); } -const __validatespecial_floatsResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number"}}, "fixtures.values_torture.special_floats", __tywrapReturnDefinitions); +const __validatespecial_floatsResult = createReturnValidator({"kind":"array","element":{"kind":"primitive","type":"number","constraint":"finite"}}, "fixtures.values_torture.special_floats", __tywrapReturnDefinitions); export function specialFloats(): Promise; export function specialFloats(include?: boolean): Promise; @@ -431,16 +428,16 @@ export async function specialFloats(include?: boolean): Promise { const __validatetemporal_valuesResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.temporal_values", __tywrapReturnDefinitions); -export async function temporalValues(): Promise> { +export async function temporalValues(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'temporal_values', __args, undefined, __validatetemporal_valuesResult); + return getRuntimeBridge().call('fixtures.values_torture', 'temporal_values', __args, undefined, __validatetemporal_valuesResult); } -const __validatetuple_key_dictResult = createReturnValidator({"kind":"record","values":{"kind":"primitive","type":"string"}}, "fixtures.values_torture.tuple_key_dict", __tywrapReturnDefinitions); +const __validatetuple_key_dictResult = createReturnValidator({"kind":"any"}, "fixtures.values_torture.tuple_key_dict", __tywrapReturnDefinitions); -export async function tupleKeyDict(): Promise> { +export async function tupleKeyDict(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); + return getRuntimeBridge().call('fixtures.values_torture', 'tuple_key_dict', __args, undefined, __validatetuple_key_dictResult); } const __validateunicode_textResult = createReturnValidator({"kind":"primitive","type":"string"}, "fixtures.values_torture.unicode_text", __tywrapReturnDefinitions); @@ -452,9 +449,9 @@ export async function unicodeText(): Promise { const __validateuuid_and_pathResult = createReturnValidator({"kind":"record","values":{"kind":"any"}}, "fixtures.values_torture.uuid_and_path", __tywrapReturnDefinitions); -export async function uuidAndPath(): Promise> { +export async function uuidAndPath(): Promise { const __args: unknown[] = []; - return getRuntimeBridge().call>('fixtures.values_torture', 'uuid_and_path', __args, undefined, __validateuuid_and_pathResult); + return getRuntimeBridge().call('fixtures.values_torture', 'uuid_and_path', __args, undefined, __validateuuid_and_pathResult); } /** @@ -482,23 +479,23 @@ export function complexValue(): Promise; export function coroutineValue(): Promise; -export function dataclassInstance(): Promise; +export function dataclassInstance(): Promise; -export function decimalValues(): Promise; +export function decimalValues(): Promise; -export function deeplyNested(): Promise; +export function deeplyNested(): Promise; export function echoInt(value: number): Promise; -export function emptyValues(): Promise<[[unknown], object[], Record, object[]]>; +export function emptyValues(): Promise; export function enumMember(): Promise; export function finiteFloatEdges(): Promise; -export function generatorValue(): Promise; +export function generatorValue(): Promise; -export function intKeyDict(): Promise<{ [key: number]: string; }>; +export function intKeyDict(): Promise; export function integerBoundaries(): Promise; @@ -523,13 +520,13 @@ export function setAndFrozenset(): Promise<[number[], string[]]>; export function specialFloats(): Promise; export function specialFloats(include?: boolean): Promise; -export function temporalValues(): Promise>; +export function temporalValues(): Promise; -export function tupleKeyDict(): Promise>; +export function tupleKeyDict(): Promise; export function unicodeText(): Promise; -export function uuidAndPath(): Promise>; +export function uuidAndPath(): Promise; /** * Point(x: 'int', y: 'int') diff --git a/test/menagerie/fixtures/library_torture.py b/test/menagerie/fixtures/library_torture.py index 502bcd1d..7d705d56 100644 --- a/test/menagerie/fixtures/library_torture.py +++ b/test/menagerie/fixtures/library_torture.py @@ -23,6 +23,16 @@ def numpy_adversarial() -> object: } +def numpy_adversarial_safe_scalar() -> object: + import numpy as np + + return { + "array": np.array([2**53 + 1, 2**63 - 1], dtype=np.int64), + "scalar": np.int64(2**53 - 1), + "float_column": np.array([1.0, 2.5]), + } + + def pandas_adversarial() -> object: import pandas as pd @@ -345,6 +355,16 @@ def sklearn_simple_estimator() -> object: return LinearRegression(fit_intercept=False, positive=True) +def sklearn_unsafe_nested_estimator() -> object: + from sklearn.base import BaseEstimator + + class NestedParams(BaseEstimator): + def __init__(self, config: object = None): + self.config = config + + return NestedParams(config={'nested': {'unsafe': 2**53 + 1}}) + + def sklearn_pipeline() -> object: from sklearn.linear_model import LogisticRegression from sklearn.pipeline import Pipeline diff --git a/test/menagerie/gen.test.ts b/test/menagerie/gen.test.ts index 204495be..d40160ca 100644 --- a/test/menagerie/gen.test.ts +++ b/test/menagerie/gen.test.ts @@ -9,6 +9,14 @@ import { TIER_ONE_MODULES } from './manifest.js'; const defaultPythonPath = getDefaultPythonPath(); const fixtureImportPath = 'test/menagerie'; +async function supportsPythonOverloads(): Promise { + const result = await processUtils.exec(defaultPythonPath, [ + '-c', + 'import typing; raise SystemExit(0 if hasattr(typing, "get_overloads") else 1)', + ]); + return result.code === 0; +} + async function compileGeneratedFile(generatedPath: string): Promise { const tscPath = join(process.cwd(), 'node_modules', 'typescript', 'lib', 'tsc.js'); const compile = await processUtils.exec( @@ -60,12 +68,40 @@ describe('menagerie generation gate', () => { expect(generatedTs).toBeDefined(); expect(generatedDeclaration).toBeDefined(); - await expect( - Promise.all([ - fsUtils.readFile(generatedTs as string), - fsUtils.readFile(generatedDeclaration as string), - ]) - ).resolves.toMatchSnapshot(); + let generated = await Promise.all([ + fsUtils.readFile(generatedTs as string), + fsUtils.readFile(generatedDeclaration as string), + ]); + if (moduleName === 'fixtures.typing_torture') { + const supportsOverloads = await supportsPythonOverloads(); + if (supportsOverloads) { + expect(generated[0]).toContain('selectOverloadReturnValidator('); + expect(generated[1]).toContain('export function overloaded(value: number)'); + expect(generated[1]).toContain('export function overloaded(value: string)'); + } else { + expect(generated[0]).toContain( + 'export async function overloaded(value: number | string)' + ); + expect(generated[1]).toContain('export function overloaded(value: number | string)'); + await compileGeneratedFile(generatedTs as string); + } + generated = [ + generated[0] + .replace( + 'createReturnValidator, selectOverloadReturnValidator, getRuntimeBridge', + 'createReturnValidator, getRuntimeBridge' + ) + .replace( + /^(?:export function overloaded[^\n]*\n)*export async function overloaded[^\n]*\n[\s\S]*?^\}/m, + '/* overload implementation checked separately */' + ), + generated[1].replace( + /^export function overloaded[^\n]*(?:\nexport function overloaded[^\n]*)*/m, + '/* overload declarations checked separately */' + ), + ]; + } + expect(generated).toMatchSnapshot(); } finally { await rm(tempDir, { recursive: true, force: true }); } diff --git a/test/menagerie/libs.test.ts b/test/menagerie/libs.test.ts index 1a8a3bc8..25e51f19 100644 --- a/test/menagerie/libs.test.ts +++ b/test/menagerie/libs.test.ts @@ -63,17 +63,20 @@ describe.skipIf(!bridgeAvailable)('menagerie optional-library gate', () => { ); it.skipIf(!hasPythonModule('numpy') || !hasPythonModule('pyarrow'))( - 'round-trips ndarrays nested in a mapping while retaining scalar behavior', + 'preserves nested Arrow arrays and rejects unsafe NumPy scalars', async () => { const bridge = makeBridge(); try { await expect( - bridge.call('fixtures.library_torture', 'numpy_adversarial', []) + bridge.call('fixtures.library_torture', 'numpy_adversarial_safe_scalar', []) ).resolves.toEqual({ array: [9007199254740993n, 9223372036854775807n], - scalar: 9007199254740992, + scalar: 9007199254740991, float_column: [1, 2.5], }); + await expect( + bridge.call('fixtures.library_torture', 'numpy_adversarial', []) + ).rejects.toThrow(/Unsafe Python integer at result\.scalar/); } finally { await bridge.dispose(); } diff --git a/test/menagerie/manifest.ts b/test/menagerie/manifest.ts index b40368fa..6bd76e79 100644 --- a/test/menagerie/manifest.ts +++ b/test/menagerie/manifest.ts @@ -134,42 +134,37 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ valuesRow({ id: 'integer-first-unsafe', call: 'integer_first_unsafe()', - status: 'KNOWN_LIE', - currentBehavior: 'The first unsafe integer is delivered as an untagged JavaScript number.', - expected: equal(2 ** 53), - expectedFix: 'Future bigint or tagged-integer transport.', + status: 'LOUD_FAIL', + currentBehavior: 'The bridge rejects the first unsafe integer before JSON encoding.', + expected: error(/Unsafe Python integer at result:.*outside the JavaScript safe integer range/i), }), valuesRow({ id: 'integer-first-rounded', call: 'integer_first_rounded()', - status: 'KNOWN_LIE', - currentBehavior: '2^53 + 1 rounds down to 2^53 in JavaScript.', - expected: equal(2 ** 53), - expectedFix: 'Future bigint or tagged-integer transport.', + status: 'LOUD_FAIL', + currentBehavior: 'The bridge rejects 2^53 + 1 before JSON encoding.', + expected: error(/Unsafe Python integer at result:.*outside the JavaScript safe integer range/i), }), valuesRow({ id: 'integer-int64-max', call: 'integer_int64_max()', - status: 'KNOWN_LIE', - currentBehavior: 'The signed int64 maximum rounds in JavaScript.', - expected: equal(2 ** 63), - expectedFix: 'Future bigint or tagged-integer transport.', + status: 'LOUD_FAIL', + currentBehavior: 'The bridge rejects the signed int64 maximum before JSON encoding.', + expected: error(/Unsafe Python integer at result:.*outside the JavaScript safe integer range/i), }), valuesRow({ id: 'integer-int64-min', call: 'integer_int64_min()', - status: 'KNOWN_LIE', - currentBehavior: 'The signed int64 minimum is delivered as an unsafe JavaScript number.', - expected: equal(-(2 ** 63)), - expectedFix: 'Future bigint or tagged-integer transport.', + status: 'LOUD_FAIL', + currentBehavior: 'The bridge rejects the signed int64 minimum before JSON encoding.', + expected: error(/Unsafe Python integer at result:.*outside the JavaScript safe integer range/i), }), valuesRow({ id: 'integer-factorial-30', call: 'integer_factorial_30()', - status: 'KNOWN_LIE', - currentBehavior: 'factorial(30) is delivered as a rounded JavaScript number.', - expected: equal(2.6525285981219107e32), - expectedFix: 'Future bigint or tagged-integer transport.', + status: 'LOUD_FAIL', + currentBehavior: 'The bridge rejects factorial(30) before JSON encoding.', + expected: error(/Unsafe Python integer at result:.*outside the JavaScript safe integer range/i), }), valuesRow({ id: 'empty-values', @@ -286,9 +281,9 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ call: 'numpy_float16()', codec: 'arrow', requires: ['numpy', 'pyarrow'], - status: 'KNOWN_LIE', - currentBehavior: 'Arrow JS exposes float16 storage bits as numbers.', - expected: equal([15872, 49280]), + status: 'EXPECTED_OK', + currentBehavior: 'Arrow float16 storage bits decode to numeric values.', + expected: { kind: 'ndarray', value: [1.5, -2.25], dtype: 'float16' }, }), libraryRow({ id: 'numpy-float16-json', @@ -745,13 +740,12 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ call: 'torch_float16()', codec: 'arrow', requires: ['torch', 'pyarrow'], - status: 'KNOWN_LIE', - currentBehavior: 'Arrow JS exposes float16 tensor storage bits as numbers.', + status: 'EXPECTED_OK', + currentBehavior: 'The nested ndarray converts float16 tensor storage to numeric values.', expected: { kind: 'match', - value: { data: [15872, 49280], shape: [2], dtype: 'torch.float16', device: 'cpu' }, + value: { data: [1.5, -2.25], shape: [2], dtype: 'torch.float16', device: 'cpu' }, }, - expectedFix: 'Future Arrow float16 decoding that preserves numeric values.', }), libraryRow({ id: 'torch-float16-json', @@ -876,6 +870,14 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ value: { className: 'LinearRegression', params: { fit_intercept: false, positive: true } }, }, }), + libraryRow({ + id: 'sklearn-unsafe-nested-estimator', + call: 'sklearn_unsafe_nested_estimator()', + requires: ['sklearn'], + status: 'LOUD_FAIL', + currentBehavior: 'Unsafe nested estimator parameters reject before JSON encoding.', + expected: error(/Unsafe Python integer at result\.params\.config\.nested\.unsafe/), + }), libraryRow({ id: 'sklearn-pipeline', call: 'sklearn_pipeline()', diff --git a/test/python/test_bridge_core.py b/test/python/test_bridge_core.py index 38d5c164..e60b4594 100644 --- a/test/python/test_bridge_core.py +++ b/test/python/test_bridge_core.py @@ -6,12 +6,15 @@ import subprocess import sys import tracemalloc +import types from pathlib import Path import pytest RUNTIME_DIR = Path(__file__).parent.parent.parent / 'runtime' +VALUE_CONTRACT_PATH = RUNTIME_DIR.parent / 'docs' / 'maintainers' / 'value-contracts.v2.json' +VALUE_FIXTURES_PATH = RUNTIME_DIR.parent / 'docs' / 'maintainers' / 'value-contract-fixtures.v2.json' sys.path.insert(0, str(RUNTIME_DIR)) @@ -28,10 +31,264 @@ serialize, serialize_dataframe_json, serialize_ndarray_json, + serialize_series, serialize_series_json, + serialize_sparse_matrix, ) +def test_safe_integer_policy_matches_frozen_specification() -> None: + specification = json.loads(VALUE_CONTRACT_PATH.read_text()) + assert specification['revision'] == 2 + assert specification['codecVersion'] == bridge_core.CODEC_VERSION + assert specification['rules']['integer']['maximum'] == bridge_core.JS_SAFE_INTEGER_MAX + assert specification['rules']['integer']['minimum'] == -bridge_core.JS_SAFE_INTEGER_MAX + + +def test_fixed_integer_proof_fixtures() -> None: + fixtures = json.loads(VALUE_FIXTURES_PATH.read_text()) + assert fixtures['revision'] == 2 + for case in fixtures['integerCases']: + value = int(case['decimal']) + if case['outcome'] == 'reject': + with pytest.raises(RuntimeError, match='Unsafe Python integer at result:'): + serialize(value, force_json_markers=True) + else: + assert serialize(value, force_json_markers=True) == value + + +@pytest.mark.parametrize('value', [2**53, -(2**53), 2**63 - 1, -(2**63)]) +def test_unsafe_integer_rejection_names_root_and_nested_paths(value: int) -> None: + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result:.*return an explicit string'): + serialize(value, force_json_markers=True) + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.items\[0\]:'): + serialize({'items': [value]}, force_json_markers=True) + + +def test_safe_integer_edges_and_finite_floats_still_encode() -> None: + value = [-(2**53 - 1), 2**53 - 1, True, False, -0.0, 1.5] + serialized = serialize(value, force_json_markers=True) + assert json.loads(encode_value(serialized, allow_nan=False)) == value + + +def test_model_dump_values_follow_nested_integer_policy() -> None: + class Model: + def model_dump(self, **_kwargs: object) -> dict[str, object]: + return {'nested': {'value': 2**53 + 1}} + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.nested\.value:'): + serialize(Model(), force_json_markers=True) + + class ScalarModel: + def model_dump(self, **_kwargs: object) -> int: + return 2**53 + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result:'): + serialize(ScalarModel(), force_json_markers=True) + + +def test_mapping_model_dump_keeps_alias_precedence() -> None: + class ModelMap(dict[str, int]): + def model_dump(self, **_kwargs: object) -> dict[str, int]: + return {'from_alias': self['internal']} + + value = ModelMap(internal=7) + assert serialize(value, force_json_markers=True) == {'from_alias': 7} + assert serialize({'item': value}, force_json_markers=True) == { + 'item': {'from_alias': 7} + } + + +def test_model_dump_cycle_rejects_without_recursing_forever() -> None: + class Model: + other: object + + def model_dump(self, **_kwargs: object) -> object: + return self.other + + first = Model() + second = Model() + first.other = second + second.other = first + + with pytest.raises(RuntimeError, match='Circular model conversion detected at result'): + serialize(first, force_json_markers=True) + + +def test_long_model_dump_chain_obeys_depth_and_node_limits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Model: + def __init__(self, other: object): + self.other = other + + def model_dump(self, **_kwargs: object) -> object: + return self.other + + value: object = 1 + for _ in range(10): + value = Model(value) + + monkeypatch.setattr(bridge_core, 'MAX_SERIALIZE_DEPTH', 4) + with pytest.raises(RuntimeError, match='maximum depth 4 exceeded at result'): + serialize(value, force_json_markers=True) + + monkeypatch.setattr(bridge_core, 'MAX_SERIALIZE_DEPTH', MAX_SERIALIZE_DEPTH) + monkeypatch.setattr(bridge_core, 'MAX_SERIALIZE_NODES', 4) + with pytest.raises(RuntimeError, match='maximum visited nodes 4 exceeded at result'): + serialize(value, force_json_markers=True) + + +def test_nested_pydantic_models_keep_value_semantics() -> None: + pydantic = pytest.importorskip('pydantic') + if not hasattr(pydantic.BaseModel, 'model_dump'): + pytest.skip('Pydantic v2 is required') + + class Child(pydantic.BaseModel): + count: int + + class Parent(pydantic.BaseModel): + child: Child + items: list[int] + + value = Parent(child=Child(count=3), items=[4, 5]) + assert serialize(value, force_json_markers=True) == { + 'child': {'count': 3}, + 'items': [4, 5], + } + + +def test_numpy_scalar_follows_nested_integer_policy() -> None: + np = pytest.importorskip('numpy') + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.nested\[0\]:'): + serialize({'nested': [np.int64(2**53)]}, force_json_markers=True) + + +def test_integer_subclass_cannot_bypass_policy() -> None: + class IntegerSubclass(int): + pass + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.value:'): + serialize({'value': IntegerSubclass(2**53)}, force_json_markers=True) + + class RecordSubclass(dict[str, int]): + pass + + class ListSubclass(list[int]): + pass + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.value:'): + serialize(RecordSubclass(value=2**53), force_json_markers=True) + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\[0\]:'): + serialize(ListSubclass([2**53]), force_json_markers=True) + + class MisleadingInteger(int): + def __lt__(self, _other: object) -> bool: + return False + + def __gt__(self, _other: object) -> bool: + return False + + def __int__(self) -> int: + return 0 + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.value:'): + serialize({'value': MisleadingInteger(2**53 + 1)}, force_json_markers=True) + + +@pytest.mark.parametrize('container_type', [set, frozenset]) +def test_set_subclass_values_follow_nested_integer_policy(container_type: type) -> None: + class SetSubclass(container_type): + pass + + for force_json_markers in (False, True): + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.items\[0\]:'): + serialize({'items': SetSubclass([2**53 + 1])}, force_json_markers=force_json_markers) + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\[0\]:'): + serialize(SetSubclass([2**53 + 1]), force_json_markers=force_json_markers) + + safe = serialize({'items': SetSubclass([2**53 - 1])}, force_json_markers=force_json_markers) + assert json.loads(encode_value(safe, allow_nan=False)) == { + 'items': [2**53 - 1] + } + + +def test_estimator_metadata_rejects_nested_unsafe_integer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sklearn = types.ModuleType('sklearn') + sklearn.__version__ = 'stub' + sklearn_base = types.ModuleType('sklearn.base') + + class BaseEstimator: + pass + + sklearn_base.BaseEstimator = BaseEstimator + sklearn.base = sklearn_base + monkeypatch.setitem(sys.modules, 'sklearn', sklearn) + monkeypatch.setitem(sys.modules, 'sklearn.base', sklearn_base) + + class Estimator(BaseEstimator): + def __init__(self, value: int): + self.value = value + + def get_params(self, *, deep: bool) -> dict[str, object]: + assert deep is False + return {'config': {'nested': {'unsafe': self.value}}} + + for force_json_markers in (False, True): + with pytest.raises( + RuntimeError, + match=r'Unsafe Python integer at result\.params\.config\.nested\.unsafe:', + ): + serialize(Estimator(2**53 + 1), force_json_markers=force_json_markers) + + with pytest.raises( + RuntimeError, + match=r'Unsafe Python integer at result\.fit\.params\.config\.nested\.unsafe:', + ): + serialize({'fit': Estimator(2**53)}, force_json_markers=True) + + safe = serialize(Estimator(2**53 - 1), force_json_markers=True) + assert json.loads(encode_value(safe, allow_nan=False))['params']['config']['nested'][ + 'unsafe' + ] == 2**53 - 1 + + +def test_terminal_scientific_metadata_rejects_unsafe_integer() -> None: + class Series: + name = 2**53 + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.name:'): + serialize_series(Series(), force_json_markers=False) + + class EmptyData: + size = 0 + + class Sparse: + dtype = types.SimpleNamespace(kind='i') + data = EmptyData() + shape = (2**53, 1) + + def getformat(self) -> str: + return 'coo' + + with pytest.raises(RuntimeError, match=r'Unsafe Python integer at result\.shape\[0\]:'): + serialize_sparse_matrix(Sparse()) + + +def test_json_scientific_values_report_unsafe_integer_path() -> None: + np = pytest.importorskip('numpy') + sparse = pytest.importorskip('scipy.sparse') + with pytest.raises(RuntimeError, match=r'failed at result:.*safe integer range'): + serialize(np.array([2**53], dtype=np.int64), force_json_markers=True) + with pytest.raises(RuntimeError, match=r'failed at result\.matrix:.*safe integer range'): + serialize( + {'matrix': sparse.csr_matrix([[2**53]])}, + force_json_markers=True, + ) + + def test_plain_values_do_not_import_scientific_codecs() -> None: """Plain JSON values must not cold-import optional scientific packages.""" script = f""" diff --git a/test/rpc-client.test.ts b/test/rpc-client.test.ts index b9aaa1eb..1bbc4255 100644 --- a/test/rpc-client.test.ts +++ b/test/rpc-client.test.ts @@ -503,6 +503,71 @@ describe('BridgeProtocol', () => { const parsed = JSON.parse(transport.lastMessage!); expect(parsed.params.kwargs).toEqual({ key: 'value' }); }); + + it('keeps equal scalar proof with each concurrent call', async () => { + transport.send = async message => { + const request = JSON.parse(message); + const dtype = request.params.functionName === 'half' ? 'float16' : 'float32'; + await new Promise(resolve => setTimeout(resolve, dtype === 'float16' ? 5 : 0)); + return JSON.stringify({ + id: request.id, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype, + }, + }); + }; + + const seen: string[] = []; + const [half, full] = await Promise.all([ + protocol.call('module', 'half', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float16'); + seen.push('half'); + }), + protocol.call('module', 'full', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float32'); + seen.push('full'); + }), + ]); + expect([half, full]).toEqual([1.5, 1.5]); + expect(seen).toHaveLength(2); + }); + + it('does not pass rejected decode proof to validation or a later call', async () => { + let bad = true; + transport.setDynamicResponse(() => + bad + ? { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + } + : 1.5 + ); + const validate = vi.fn((_value: number, provenance?: { atRoot(): unknown }) => { + expect(provenance?.atRoot()).toBeUndefined(); + }); + + await expect(protocol.call('module', 'bad', [], undefined, validate)).rejects.toThrow( + /must be an array at depth 0/ + ); + expect(validate).not.toHaveBeenCalled(); + bad = false; + await expect(protocol.call('module', 'plain', [], undefined, validate)).resolves.toBe( + 1.5 + ); + expect(validate).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 033bd349..ca35a82c 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -3,11 +3,13 @@ import type { AddressInfo } from 'node:net'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { BridgeValidationError } from '../src/runtime/errors.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { HttpBridge } from '../src/runtime/http.js'; import { NodeBridge } from '../src/runtime/node.js'; import { createReturnValidator, describeReceivedShape, + selectOverloadReturnValidator, tagDecodedShape, type ReturnSchema, } from '../src/runtime/validators.js'; @@ -26,6 +28,112 @@ describe('generated return validators', () => { expect(validator(42)).toBe(42); }); + it('checks the value contract number constraints at nested paths', () => { + const validator = createReturnValidator( + { + kind: 'record', + values: { kind: 'primitive', type: 'number', constraint: 'safe-integer' }, + }, + 'fixture.counts' + ); + expect(validator({ count: 42 })).toEqual({ count: 42 }); + expect(() => validator({ count: 1.5 })).toThrow(BridgeValidationError); + expect(() => validator({ count: Number.MAX_SAFE_INTEGER + 1 })).toThrow(BridgeValidationError); + }); + + it('requires per-call provenance for a scalar float16 ndarray', () => { + const validate = createReturnValidator( + { kind: 'marker', marker: 'ndarray', dims: 0, dtype: 'float16' }, + 'fixture.scalar' + ); + const valid = new DecodedProvenance(); + valid.recordRoot({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + const wrongDtype = new DecodedProvenance(); + wrongDtype.recordRoot({ marker: 'ndarray', dims: 0, dtype: 'float32' }); + const wrongShape = new DecodedProvenance(); + wrongShape.recordRoot({ marker: 'ndarray', dims: 1, dtype: 'float16' }); + + expect(() => validate(1.5)).toThrow(BridgeValidationError); + expect(validate(1.5, valid)).toBe(1.5); + expect(() => validate(1.5, wrongDtype)).toThrow(BridgeValidationError); + expect(() => validate(1.5, wrongShape)).toThrow(BridgeValidationError); + expect(validate(1.5, valid)).toBe(1.5); + }); + + it('checks nested scalar proofs at array indices and record keys', () => { + const scalar = { kind: 'marker', marker: 'ndarray', dims: 0, dtype: 'float16' } as const; + const rows = [[1.5]]; + const result = { rows, scalar: 1.5 }; + const proof = new DecodedProvenance(); + proof.recordChild(rows[0]!, 0, { marker: 'ndarray', dims: 0, dtype: 'float16' }); + proof.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + const validate = createReturnValidator( + { + kind: 'record', + fields: { + rows: { schema: { kind: 'array', element: { kind: 'tuple', elements: [scalar] } } }, + scalar: { schema: scalar }, + }, + }, + 'fixture.nestedScalar' + ); + expect(validate(result, proof)).toBe(result); + expect(() => validate(result)).toThrow(BridgeValidationError); + + const stringIndex = new DecodedProvenance(); + stringIndex.recordChild(rows[0]!, '0', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + stringIndex.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + expect(validate(result, stringIndex)).toBe(result); + + const wrongIndex = new DecodedProvenance(); + wrongIndex.recordChild(rows[0]!, 1, { marker: 'ndarray', dims: 0, dtype: 'float16' }); + wrongIndex.recordChild(result, 'scalar', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + expect(() => validate(result, wrongIndex)).toThrow(BridgeValidationError); + + const other = { rows: [[1.5]], scalar: 1.5 }; + expect(() => validate(other, proof)).toThrow(BridgeValidationError); + }); + + it('uses declaration order when an optional-arity overload also matches', () => { + const stringValue = { kind: 'primitive', type: 'string' } as const; + const integerValue = { kind: 'primitive', type: 'number', constraint: 'safe-integer' } as const; + const overloads = [ + { + parameters: [ + { name: 'value', kind: 'positional-or-keyword', optional: false, value: stringValue }, + ], + result: stringValue, + selectable: true, + }, + { + parameters: [ + { name: 'value', kind: 'positional-or-keyword', optional: false, value: stringValue }, + { name: 'base', kind: 'positional-or-keyword', optional: true, value: integerValue }, + ], + result: integerValue, + selectable: true, + }, + ] as const; + const fallback = createReturnValidator({ kind: 'any' }, 'fixture.ambiguous'); + const oneArg = selectOverloadReturnValidator( + overloads, + ['key'], + undefined, + fallback, + 'fixture.ambiguous' + ); + const twoArgs = selectOverloadReturnValidator( + overloads, + ['key', 2], + undefined, + fallback, + 'fixture.ambiguous' + ); + expect(() => oneArg(4)).toThrow(BridgeValidationError); + expect(oneArg('text')).toBe('text'); + expect(twoArgs(4)).toBe(4); + }); + it('checks unions, optionals, tuples, TypedDict records, and no-op schemas', () => { const schema: ReturnSchema = { kind: 'tuple', @@ -220,7 +328,17 @@ describe('return validator bridge propagation', () => { try { await expect(bridge.call('math', 'sqrt', [16], undefined, validate)).resolves.toBe(4); expect(validate).toHaveBeenCalledOnce(); - expect(validate).toHaveBeenCalledWith(4); + expect(validate).toHaveBeenCalledWith(4, expect.any(DecodedProvenance)); + const [, provenance] = validate.mock.calls[0] as unknown as [number, DecodedProvenance]; + expect(provenance.atRoot()).toBeUndefined(); + + const reject = vi.fn((_value: number) => { + throw new Error('one-argument validator rejected the value'); + }); + await expect(bridge.call('math', 'sqrt', [16], undefined, reject)).rejects.toThrow( + 'one-argument validator rejected the value' + ); + expect(reject).toHaveBeenCalledOnce(); } finally { await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve())) diff --git a/test/value_contracts.test.ts b/test/value_contracts.test.ts new file mode 100644 index 00000000..71a7eaed --- /dev/null +++ b/test/value_contracts.test.ts @@ -0,0 +1,307 @@ +import { readFileSync } from 'node:fs'; +import { afterEach, describe, expect, it } from 'vitest'; +import { BridgeCodec } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; +import { + MAX_SAFE_JSON_INTEGER, + VALUE_CONTRACT_REVISION, + isSafeJsonInteger, +} from '../src/contracts/value-contract.js'; +import { + clearArrowDecoder, + decodeValue, + decodeValueAsync, + registerArrowDecoder, + type ArrowTable, +} from '../src/utils/codec.js'; + +const specification = JSON.parse( + readFileSync(new URL('../docs/maintainers/value-contracts.v2.json', import.meta.url), 'utf8') +) as { + revision: number; + rules: { + integer: { minimum: number; maximum: number }; + bytes: { decodedAs: string }; + }; +}; +const fixtures = JSON.parse( + readFileSync( + new URL('../docs/maintainers/value-contract-fixtures.v2.json', import.meta.url), + 'utf8' + ) +) as { + revision: number; + bytesCases: readonly { + logicalHex: string; + requestEnvelope: object; + responseEnvelope: object; + decodedAs: string; + }[]; + binary16Cases: readonly { wordHex: string; outcome: string }[]; +}; + +function referenceBinary16(bits: number): number { + const sign = (bits & 0x8000) << 16; + let exponent = (bits >>> 10) & 0x1f; + let fraction = bits & 0x03ff; + let floatBits: number; + + if (exponent === 0 && fraction !== 0) { + exponent = -14; + while ((fraction & 0x0400) === 0) { + fraction <<= 1; + exponent -= 1; + } + fraction &= 0x03ff; + floatBits = sign | ((exponent + 127) << 23) | (fraction << 13); + } else if (exponent === 0) { + floatBits = sign; + } else { + floatBits = sign | ((exponent + 112) << 23) | (fraction << 13); + } + + const view = new DataView(new ArrayBuffer(4)); + view.setUint32(0, floatBits, true); + return view.getFloat32(0, true); +} + +function envelope(shape: number[]): object { + return { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'arrow', + b64: 'AA==', + shape, + dtype: 'float16', + }; +} + +function registerWords(words: readonly number[]): void { + registerArrowDecoder( + () => + ({ + getChildAt: () => ({ toArray: () => Uint16Array.from(words) }), + }) as ArrowTable + ); +} + +afterEach(() => clearArrowDecoder()); + +describe('frozen value policy', () => { + it('keeps the TypeScript safe-integer range equal to the JSON specification', () => { + expect(VALUE_CONTRACT_REVISION).toBe(specification.revision); + expect(fixtures.revision).toBe(specification.revision); + expect(MAX_SAFE_JSON_INTEGER).toBe(specification.rules.integer.maximum); + expect(-MAX_SAFE_JSON_INTEGER).toBe(specification.rules.integer.minimum); + expect(isSafeJsonInteger(MAX_SAFE_JSON_INTEGER)).toBe(true); + expect(isSafeJsonInteger(-MAX_SAFE_JSON_INTEGER)).toBe(true); + expect(isSafeJsonInteger(2 ** 53)).toBe(false); + expect(isSafeJsonInteger(-(2 ** 53))).toBe(false); + expect(isSafeJsonInteger(1.5)).toBe(false); + }); + + it('matches the shared bytes request and response fixtures', async () => { + const codec = new BridgeCodec(); + expect(specification.rules.bytes.decodedAs).toBe('JavaScript Uint8Array'); + for (const fixture of fixtures.bytesCases) { + const bytes = Uint8Array.from(Buffer.from(fixture.logicalHex, 'hex')); + const request = JSON.parse(codec.encodeRequest({ value: bytes })); + expect(request.value).toEqual(fixture.requestEnvelope); + + const decoded = await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: fixture.responseEnvelope }) + ); + expect(decoded).toBeInstanceOf(Uint8Array); + expect([...decoded]).toEqual([...bytes]); + expect(`Uint8Array[${[...decoded].join(',')}]`).toBe(fixture.decodedAs); + } + }); +}); + +describe('Arrow float16 value contract', () => { + it('matches the fixed proof fixtures', () => { + for (const fixture of fixtures.binary16Cases) { + registerWords([Number.parseInt(fixture.wordHex, 16)]); + if (fixture.outcome === 'reject') { + expect(() => decodeValue(envelope([1]))).toThrow(/non-finite float16/); + } else { + const decoded = decodeValue(envelope([1])) as number[]; + expect(Object.is(decoded[0], Number(fixture.outcome)), fixture.wordHex).toBe(true); + } + } + }); + + it('matches an independent float32 bit reference for every finite binary16 word', () => { + const words = Array.from({ length: 0x10000 }, (_, word) => word).filter( + word => ((word >>> 10) & 0x1f) !== 0x1f + ); + registerWords(words); + const decoded = decodeValue(envelope([words.length])) as number[]; + expect(decoded).toHaveLength(words.length); + for (let index = 0; index < words.length; index += 1) { + if (!Object.is(decoded[index], referenceBinary16(words[index] as number))) { + throw new Error(`binary16 word ${words[index]} decoded incorrectly`); + } + } + expect(Object.is(decoded[0], 0)).toBe(true); + expect(Object.is(decoded[words.indexOf(0x8000)], -0)).toBe(true); + expect(decoded[words.indexOf(0x0001)]).toBe(2 ** -24); + expect(decoded[words.indexOf(0x7bff)]).toBe(65504); + expect(decoded[words.indexOf(0xfbff)]).toBe(-65504); + }); + + it('decodes scalar and multidimensional arrays before reshaping', () => { + registerWords([0x3e00]); + expect(decodeValue(envelope([]))).toBe(1.5); + registerWords([0x3e00, 0xc080, 0x0001, 0x8000]); + expect(decodeValue(envelope([2, 2]))).toEqual([ + [1.5, -2.25], + [2 ** -24, -0], + ]); + }); + + it('carries Arrow scalar float16 proof with the decoded number', async () => { + registerWords([0x3e00]); + const provenance = new DecodedProvenance(); + expect(await decodeValueAsync(envelope([]), provenance)).toBe(1.5); + expect(provenance.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + }); + + it('uses the ndarray rule inside nested Torch and record values', async () => { + registerWords([0x3e00, 0xc080]); + const value = { + outer: [ + { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + value: envelope([2]), + shape: [2], + dtype: 'torch.float16', + device: 'cpu', + }, + ], + }; + await expect(decodeValueAsync(value)).resolves.toEqual({ + outer: [{ data: [1.5, -2.25], shape: [2], dtype: 'torch.float16', device: 'cpu' }], + }); + }); + + it.each([ + { shape: [], encoding: 'json' }, + { shape: [1], encoding: 'json' }, + { shape: [], encoding: 'arrow' }, + { shape: [1], encoding: 'arrow' }, + ] as const)( + 'requires a matching nested dtype for Torch float16 shape $shape with $encoding data', + async ({ shape, encoding }) => { + registerWords([0x3e00]); + const nested = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding, + shape, + dtype: 'float16', + ...(encoding === 'arrow' ? { b64: 'AA==' } : { data: shape.length ? [1.5] : 1.5 }), + }; + const tensor = { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + value: nested, + shape, + dtype: 'torch.float16', + device: 'cpu', + }; + + await expect(decodeValueAsync(tensor)).resolves.toMatchObject({ + data: shape.length ? [1.5] : 1.5, + dtype: 'torch.float16', + }); + await expect( + decodeValueAsync({ ...tensor, value: { ...nested, dtype: 'float32' } }) + ).rejects.toThrow(/value\.dtype.*must be "float16".*"torch\.float16"/); + } + ); + + it('requires the nested version 1 envelope for Torch float16', async () => { + await expect( + decodeValueAsync({ + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + shape: [], + dtype: 'torch.float16', + value: { + __tywrap__: 'ndarray', + encoding: 'json', + shape: [], + dtype: 'float16', + data: 1.5, + }, + }) + ).rejects.toThrow(/float16 value must use ndarray codecVersion 1/); + }); + + it.each([0x7c00, 0xfc00, 0x7e00])('rejects non-finite word %s', word => { + registerWords([word]); + expect(() => decodeValue(envelope([1]))).toThrow(/non-finite float16 value at b64\[0\]/); + }); + + it('rejects a mismatched Arrow type or null storage', () => { + registerArrowDecoder( + () => + ({ + getChildAt: () => ({ + type: 'Int16', + nullCount: 0, + toArray: () => Uint16Array.of(0x3e00), + }), + }) as ArrowTable + ); + expect(() => decodeValue(envelope([1]))).toThrow(/column type does not match/); + registerArrowDecoder( + () => + ({ + getChildAt: () => ({ + type: 'Float16', + nullCount: 1, + toArray: () => Uint16Array.of(0), + }), + }) as ArrowTable + ); + expect(() => decodeValue(envelope([1]))).toThrow(/column contains null values/); + }); + + it('keeps JSON fallback numeric and Arrow int64 exact', () => { + expect( + decodeValue({ + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: [1.5, -2.25], + shape: [2], + dtype: 'float16', + }) + ).toEqual([1.5, -2.25]); + + registerArrowDecoder( + () => + ({ + getChildAt: () => ({ + toArray: () => BigInt64Array.of(-9223372036854775808n, 9223372036854775807n), + }), + }) as ArrowTable + ); + expect( + decodeValue({ + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'arrow', + b64: 'AA==', + shape: [2], + dtype: 'int64', + }) + ).toEqual([-9223372036854775808n, 9223372036854775807n]); + }); +}); diff --git a/tywrap_ir/tywrap_ir/ir.py b/tywrap_ir/tywrap_ir/ir.py index 95c4f5bc..d8198719 100644 --- a/tywrap_ir/tywrap_ir/ir.py +++ b/tywrap_ir/tywrap_ir/ir.py @@ -493,11 +493,13 @@ def _extract_function( # Async generators must be marked as generators so callers can distinguish # them from plain coroutines. is_generator = inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) + overloads, overload_annotations = _extract_overloads(obj) type_params = ( _collect_scoped_type_params( obj, *annotations_for_params, returns, + *overload_annotations, inherited_type_params=inherited_type_params, ) if include_type_params @@ -514,7 +516,7 @@ def _extract_function( is_generator=is_generator, type_params=type_params, method_kind=method_kind, - overloads=_extract_overloads(obj), + overloads=overloads, ) @@ -525,8 +527,8 @@ def _extract_function( _overload_warning_recorded = False -def _extract_overloads(obj: Any) -> List[IROverload]: - """Capture @typing.overload signatures for ``obj`` (Python 3.11+). +def _extract_overloads(obj: Any) -> tuple[List[IROverload], List[Any]]: + """Capture overload signatures and their raw annotations (Python 3.11+). On interpreters without ``typing.get_overloads`` (3.10 and earlier) this degrades to an empty list and records a single structured IR warning @@ -541,14 +543,15 @@ def _extract_overloads(obj: Any) -> List[IROverload]: "typing.get_overloads is unavailable (Python < 3.11); " "@overload signatures will not be captured in the IR." ) - return [] + return [], [] try: registered = get_overloads(obj) except Exception: - return [] + return [], [] out: List[IROverload] = [] + annotations: List[Any] = [] for ov in registered or (): try: sig = inspect.signature(ov) @@ -564,6 +567,7 @@ def _extract_overloads(obj: Any) -> List[IROverload]: params: List[IRParam] = [] for pname, p in sig.parameters.items(): ann = hints.get(pname, p.annotation) + annotations.append(ann) params.append( IRParam( name=pname, @@ -573,8 +577,9 @@ def _extract_overloads(obj: Any) -> List[IROverload]: ) ) ret = hints.get("return", sig.return_annotation) + annotations.append(ret) out.append(IROverload(parameters=params, returns=_stringify_annotation(ret))) - return out + return out, annotations def _accessor_return_annotation(getter: Any) -> Optional[str]: