From 8ec57da6026aee76c8c736303517c9fdcb7704fb Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:12:43 -0700 Subject: [PATCH 1/5] feat(codec): define bytes and per-call provenance contracts --- .../value-contract-fixtures.v2.json | 14 ++ docs/maintainers/value-contracts.md | 134 ++++++++++++++---- docs/maintainers/value-contracts.v2.json | 9 ++ src/contracts/value-contract.ts | 2 + src/runtime/decoded-provenance.ts | 59 ++++++++ test/decoded_provenance.test.ts | 35 +++++ 6 files changed, 223 insertions(+), 30 deletions(-) create mode 100644 src/runtime/decoded-provenance.ts create mode 100644 test/decoded_provenance.test.ts diff --git a/docs/maintainers/value-contract-fixtures.v2.json b/docs/maintainers/value-contract-fixtures.v2.json index abe3a5d8..78337545 100644 --- a/docs/maintainers/value-contract-fixtures.v2.json +++ b/docs/maintainers/value-contract-fixtures.v2.json @@ -10,6 +10,20 @@ { "decimal": "9223372036854775807", "outcome": "reject" }, { "decimal": "265252859812191058636308480000000", "outcome": "reject" } ], + "bytesCases": [ + { + "logicalHex": "00ff80", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "AP+A" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "AP+A" }, + "decodedAs": "Uint8Array[0,255,128]" + }, + { + "logicalHex": "", + "requestEnvelope": { "__tywrap_bytes__": true, "b64": "" }, + "responseEnvelope": { "__type__": "bytes", "encoding": "base64", "data": "" }, + "decodedAs": "Uint8Array[]" + } + ], "binary16Cases": [ { "wordHex": "0000", "outcome": "0" }, { "wordHex": "8000", "outcome": "-0" }, diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index 3229566b..dbc55a31 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -1,38 +1,112 @@ # Value contracts -The frozen internal prototype is [value-contracts.v2.json](value-contracts.v2.json). -That JSON file owns the policy. The TypeScript interface mirrors it and has revision 2. - -Each contract names a logical value, its wire encoding, its decoded JavaScript value, and its constraints. Sequences, fixed tuples, records, and unions compose child contracts. A union has two to 32 alternatives. The compiler rejects a union if one wire value could decode two ways. Contract depth stops at 64. Named records can inline within that limit. Recursive aliases stay unsupported until a reviewed reference graph exists. The prototype covers float16 ndarrays and Torch tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 may decode as `bigint`. Unsupported conversions carry a reason and guidance. - -The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow envelopes, JSON fallback, and transports keep their shape. The producer rejects unsafe Python integers before it writes JSON. A generated wrapper that consumes this interface must keep `number` for Python `int` and add a safe-integer return constraint. Regenerate wrappers after that compiler change. Do not return `bigint` under an existing `Promise`. - -Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow tables. JSON fallback for ndarray and pandas values still rejects unsafe integers. This is a behavior change from silent rounding to a call error. - -The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused tests also compare every finite binary16 storage word with an independent float32 bit reference. They test signed zero with `Object.is`. - -The Python serializer checks plain integers, NumPy scalar integers, and model dumps during its existing value walk. It also checks terminal estimator parameters and Series name metadata before JSON output. A JSON ndarray or sparse matrix checks integer data before it creates an envelope. Sparse metadata checks shapes and indices. The TypeScript decoder converts float16 Arrow storage during extraction, before it reshapes the ndarray. Torch uses that same ndarray path. These checks do not walk Arrow payloads again for schema validation. +The frozen internal prototype is +[value-contracts.v2.json](value-contracts.v2.json). That JSON file owns the +policy. The TypeScript interface mirrors it and has revision 2. + +Each contract names a logical value, its wire encoding, its decoded JavaScript +value, and its constraints. Sequences, fixed tuples, records, and unions compose +child contracts. A union has two to 32 alternatives. The compiler rejects a +union if one wire value could decode two ways. Contract depth stops at 64. Named +records can inline within that limit. Recursive aliases stay unsupported until a +reviewed reference graph exists. The prototype covers float16 ndarrays and Torch +tensors. Other Arrow types keep their existing rules. In particular, Arrow int64 +may decode as `bigint`. Unsupported conversions carry a reason and guidance. + +The wire stays at `tywrap/1` and scientific `codecVersion: 1`. Existing Arrow +envelopes, JSON fallback, and transports keep their shape. The producer rejects +unsafe Python integers before it writes JSON. A generated wrapper that consumes +this interface must keep `number` for Python `int` and add a safe-integer return +constraint. Regenerate wrappers after that compiler change. Do not return +`bigint` under an existing `Promise`. + +The bytes leaf keeps the existing base64 envelopes. Requests use +`__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to +`Uint8Array`. The Torch float16 contract names `torch.float16` on the outer +tensor and `float16` on its nested ndarray. + +Migration for users with large Python integers: convert them to strings in +Python with an explicit return annotation, or keep integer columns in Arrow +tables. JSON fallback for ndarray and pandas values still rejects unsafe +integers. This is a behavior change from silent rounding to a call error. + +The [fixed proof fixtures](value-contract-fixtures.v2.json) cover integer +boundaries, binary16 values, unions, fixed tuples, and nested paths. The focused +tests also compare every finite binary16 storage word with an independent +float32 bit reference. They test signed zero with `Object.is`. + +The Python serializer checks plain integers, NumPy scalar integers, and model +dumps during its existing value walk. It also checks terminal estimator +parameters and Series name metadata before JSON output. A JSON ndarray or sparse +matrix checks integer data before it creates an envelope. Sparse metadata checks +shapes and indices. The TypeScript decoder converts float16 Arrow storage during +extraction, before it reshapes the ndarray. Torch uses that same ndarray path. +These checks do not walk Arrow payloads again for schema validation. ## Exact bigint extension for #337 -This extension is a design sketch. The 0.11 policy still rejects unsafe plain Python integers. - -Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map Python `int` to TypeScript `bigint` for both parameters and returns. The return validator must require `typeof value === 'bigint'`, including every record and sequence member. Keep `bool` separate. Calls without the option keep `number` and the safe range. - -Add a version 2 integer value envelope with a canonical signed decimal string. A sample value is `{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. The request encoder and Python request decoder must validate the same grammar and digit limit. The response encoder and TypeScript decoder must do the same. Use `BigInt` only after validation. Keep Arrow int64 columns on their current Arrow path. - -The bridge must advertise exact integer support during capability negotiation. A generated bigint wrapper must reject an older bridge before its first call. The new bridge must emit version 2 integer envelopes only when the call opts in. An older client therefore continues to receive the 0.11 safe integer policy. Unknown integer envelopes, mixed versions, oversized strings, and malformed decimal values must fail with a path. Do not guess a conversion. - -Acceptance evidence still needed: a bounded prototype must round trip positive and negative values beyond 64 bits through nested records and arrays. It must test bigint inputs, malformed tags, payload limits, and both mixed-version directions. An independent review must approve the wire grammar, capability rule, generated types, and migration cost before #337 can close. +This extension is a design sketch. The 0.11 policy still rejects unsafe plain +Python integers. + +Choose an explicit callable option, `integerMode: 'bigint'`. When enabled, map +Python `int` to TypeScript `bigint` for both parameters and returns. The return +validator must require `typeof value === 'bigint'`, including every record and +sequence member. Keep `bool` separate. Calls without the option keep `number` +and the safe range. + +Add a version 2 integer value envelope with a canonical signed decimal string. A +sample value is +`{"__tywrap__":"integer","codecVersion":2,"encoding":"decimal","value":"18446744073709551617"}`. +The request encoder and Python request decoder must validate the same grammar +and digit limit. The response encoder and TypeScript decoder must do the same. +Use `BigInt` only after validation. Keep Arrow int64 columns on their current +Arrow path. + +The bridge must advertise exact integer support during capability negotiation. A +generated bigint wrapper must reject an older bridge before its first call. The +new bridge must emit version 2 integer envelopes only when the call opts in. An +older client therefore continues to receive the 0.11 safe integer policy. +Unknown integer envelopes, mixed versions, oversized strings, and malformed +decimal values must fail with a path. Do not guess a conversion. + +Acceptance evidence still needed: a bounded prototype must round trip positive +and negative values beyond 64 bits through nested records and arrays. It must +test bigint inputs, malformed tags, payload limits, and both mixed-version +directions. An independent review must approve the wire grammar, capability +rule, generated types, and migration cost before #337 can close. ## Dataclass return extension for #339 -This extension is a design sketch. Dataclass inputs remain unsupported and need a separate input contract. Users must pass a plain record through an explicitly typed function if they need that behavior now. - -For an annotated dataclass return, compile a structural TypeScript record from `dataclasses.fields`. Include each instance field in the output, including fields with defaults or `init=False`. A default does not make a returned field optional. A field annotated `Optional[T]` permits `null`. Compose nested dataclasses, records, and sequences only when every field has a supported value contract. Reject ordinary class instances and unsupported fields with their result paths. - -Use a version 2 dataclass envelope with a declared type identity and a `fields` record. For example, `Point(1, 2)` would carry `{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must check the type identity, exact field names, field values, and required fields. It must reject extra fields unless the compiled contract allows them. The Python encoder must use the current bounded value walk. It must detect cycles and enforce depth, node, and payload limits without calling `dataclasses.asdict` as a shortcut. - -The bridge must advertise dataclass value support. A generated dataclass wrapper must reject an older bridge before calling it. A new bridge must send the envelope only when the wrapper requests it. This leaves existing value RPC and ordinary class behavior unchanged. - -Acceptance evidence still needed: a generated `Promise` wrapper must return the exact plain record and reject missing, extra, or wrong typed fields. Tests must cover optional and defaulted fields, nested values, cycles, unsupported fields, depth, and payload limits. An independent review must approve the type identity, version rule, and migration cost before #339 can close. +This extension is a design sketch. Dataclass inputs remain unsupported and need +a separate input contract. Users must pass a plain record through an explicitly +typed function if they need that behavior now. + +For an annotated dataclass return, compile a structural TypeScript record from +`dataclasses.fields`. Include each instance field in the output, including +fields with defaults or `init=False`. A default does not make a returned field +optional. A field annotated `Optional[T]` permits `null`. Compose nested +dataclasses, records, and sequences only when every field has a supported value +contract. Reject ordinary class instances and unsupported fields with their +result paths. + +Use a version 2 dataclass envelope with a declared type identity and a `fields` +record. For example, `Point(1, 2)` would carry +`{"__tywrap__":"dataclass","codecVersion":2,"encoding":"fields","type":"fixtures.values_torture.Point","fields":{"x":1,"y":2}}`. +The TypeScript decoder returns `{x: 1, y: 2}`. The generated validator must +check the type identity, exact field names, field values, and required fields. +It must reject extra fields unless the compiled contract allows them. The Python +encoder must use the current bounded value walk. It must detect cycles and +enforce depth, node, and payload limits without calling `dataclasses.asdict` as +a shortcut. + +The bridge must advertise dataclass value support. A generated dataclass wrapper +must reject an older bridge before calling it. A new bridge must send the +envelope only when the wrapper requests it. This leaves existing value RPC and +ordinary class behavior unchanged. + +Acceptance evidence still needed: a generated `Promise` wrapper must +return the exact plain record and reject missing, extra, or wrong typed fields. +Tests must cover optional and defaulted fields, nested values, cycles, +unsupported fields, depth, and payload limits. An independent review must +approve the type identity, version rule, and migration cost before #339 can +close. diff --git a/docs/maintainers/value-contracts.v2.json b/docs/maintainers/value-contracts.v2.json index bfebfa32..827a766c 100644 --- a/docs/maintainers/value-contracts.v2.json +++ b/docs/maintainers/value-contracts.v2.json @@ -19,6 +19,13 @@ "decodedAs": "JavaScript number", "failure": "reject NaN and Infinity" }, + "bytes": { + "logicalKind": "Python bytes or bytearray", + "requestWire": "existing __tywrap_bytes__ base64 envelope", + "responseWire": "existing __type__: bytes base64 envelope", + "decodedAs": "JavaScript Uint8Array", + "failure": "reject malformed base64 or payloads above the existing byte limit" + }, "sequence": { "logicalKind": "list or variadic tuple of one contracted item kind", "wire": "JSON array", @@ -57,6 +64,8 @@ "logicalKind": "Torch float16 tensor", "wire": "codecVersion 1 torch.tensor envelope containing ndarray envelope", "decodedAs": "tensor record with numeric data", + "outerDtype": "torch.float16", + "nestedDtype": "float16", "conversion": "reuse ndarray-float16 rule for nested value" } }, diff --git a/src/contracts/value-contract.ts b/src/contracts/value-contract.ts index 52c9351c..d8bc033b 100644 --- a/src/contracts/value-contract.ts +++ b/src/contracts/value-contract.ts @@ -20,6 +20,7 @@ export type ValueContract = readonly constraint: 'finite'; } | { readonly kind: 'string'; readonly wire: 'json'; readonly decodedAs: 'string' } + | { readonly kind: 'bytes'; readonly wire: 'base64-envelope'; readonly decodedAs: 'Uint8Array' } | { readonly kind: 'sequence'; readonly wire: 'json'; @@ -57,6 +58,7 @@ export type ValueContract = readonly kind: 'torch-float16'; readonly wire: 'ndarray-envelope'; readonly decodedAs: 'tensor-record'; + readonly dtype: 'torch.float16'; readonly value: Extract; } | { diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts new file mode 100644 index 00000000..a983406c --- /dev/null +++ b/src/runtime/decoded-provenance.ts @@ -0,0 +1,59 @@ +import type { ScientificMarker } from '../utils/codec.js'; + +/** Metadata from one successfully decoded scientific envelope. */ +export interface DecodedShapeMetadata { + readonly marker: ScientificMarker; + readonly dims?: number; + readonly dtype?: string; +} + +/** Scalar provenance stays with one decoded call, including nested values. */ +export class DecodedProvenance { + private rootScalar?: DecodedShapeMetadata; + private readonly children = new WeakMap>(); + private entries = 0; + private readonly maxEntries: number; + + constructor(maxEntries = 1_000_000) { + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + throw new Error('Decoded provenance entry limit must be a positive safe integer'); + } + this.maxEntries = maxEntries; + } + + recordRoot(metadata: DecodedShapeMetadata): void { + if (this.rootScalar !== undefined) { + throw new Error('Duplicate root scalar provenance'); + } + this.claimEntry(); + this.rootScalar = Object.freeze({ ...metadata }); + } + + recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + let entries = this.children.get(parent); + if (!entries) { + entries = new Map(); + this.children.set(parent, entries); + } + if (entries.has(key)) { + throw new Error('Duplicate child scalar provenance'); + } + this.claimEntry(); + entries.set(key, Object.freeze({ ...metadata })); + } + + atRoot(): DecodedShapeMetadata | undefined { + return this.rootScalar; + } + + atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { + return this.children.get(parent)?.get(key); + } + + private claimEntry(): void { + if (this.entries >= this.maxEntries) { + throw new Error(`Decoded scalar provenance exceeds ${this.maxEntries} entries`); + } + this.entries += 1; + } +} diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts new file mode 100644 index 00000000..424890ad --- /dev/null +++ b/test/decoded_provenance.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; + +describe('decoded provenance', () => { + it('keeps equal scalar values separate by call and nested location', () => { + const firstParent = { one: 1.5, two: 1.5 }; + const secondParent = { one: 1.5 }; + const first = new DecodedProvenance(); + const second = new DecodedProvenance(); + + first.recordChild(firstParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float16' }); + first.recordChild(firstParent, 'two', { marker: 'ndarray', dims: 0, dtype: 'float32' }); + second.recordChild(secondParent, 'one', { marker: 'ndarray', dims: 0, dtype: 'float64' }); + + expect(first.atChild(firstParent, 'one')?.dtype).toBe('float16'); + expect(first.atChild(firstParent, 'two')?.dtype).toBe('float32'); + expect(first.atChild(secondParent, 'one')).toBeUndefined(); + expect(second.atChild(firstParent, 'one')).toBeUndefined(); + expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); + }); + + it('has one root slot and a bounded number of entries', () => { + const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + const provenance = new DecodedProvenance(2); + const parent = [1.5]; + provenance.recordRoot(evidence); + provenance.recordChild(parent, 0, evidence); + + expect(provenance.atRoot()).toEqual(evidence); + expect(provenance.atChild(parent, 0)).toEqual(evidence); + expect(() => provenance.recordRoot(evidence)).toThrow(/Duplicate root/); + expect(() => provenance.recordChild(parent, 0, evidence)).toThrow(/Duplicate child/); + expect(() => provenance.recordChild(parent, 1, evidence)).toThrow(/exceeds 2 entries/); + }); +}); From a2e2773c580d2b0b74ec1e7e80dbd7a66dd62b75 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:17:02 -0700 Subject: [PATCH 2/5] feat(codec): carry scalar ndarray proof per RPC call --- src/runtime/base-bridge.ts | 3 +- src/runtime/bridge-codec.ts | 5 ++- src/runtime/decoded-provenance.ts | 15 ++++++- src/runtime/rpc-client.ts | 11 ++++-- src/types/index.ts | 4 +- src/utils/codec.ts | 42 ++++++++++++++++++-- test/bridge-codec.test.ts | 66 +++++++++++++++++++++++++++++++ test/decoded_provenance.test.ts | 9 +++++ test/rpc-client.test.ts | 65 ++++++++++++++++++++++++++++++ test/value_contracts.test.ts | 40 ++++++++++++++++++- 10 files changed, 247 insertions(+), 13 deletions(-) diff --git a/src/runtime/base-bridge.ts b/src/runtime/base-bridge.ts index 0749a9d2..e3fd049d 100644 --- a/src/runtime/base-bridge.ts +++ b/src/runtime/base-bridge.ts @@ -18,6 +18,7 @@ import type { PythonRuntime, BridgeInfo } from '../types/index.js'; import { DisposableBase } from './bounded-context.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; import type { RpcClient, GetBridgeInfoOptions } from './rpc-client.js'; import type { TransportCapabilities } from './transport.js'; @@ -44,7 +45,7 @@ export abstract class BasePythonBridge extends DisposableBase implements PythonR functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { await this.ensureReady(); return this.getRpcClient().call(module, functionName, args, kwargs, validate); diff --git a/src/runtime/bridge-codec.ts b/src/runtime/bridge-codec.ts index 2edc44bf..f0fc56f8 100644 --- a/src/runtime/bridge-codec.ts +++ b/src/runtime/bridge-codec.ts @@ -16,6 +16,7 @@ import { ScientificDecodeError, } from '../utils/codec.js'; import { PROTOCOL_ID } from './transport.js'; +import type { DecodedProvenance } from './decoded-provenance.js'; // ═══════════════════════════════════════════════════════════════════════════ // TYPES @@ -662,13 +663,13 @@ export class BridgeCodec { * @throws BridgeProtocolError if the RPC response is invalid * @throws BridgeExecutionError if response contains a Python error */ - async decodeResponseAsync(payload: string): Promise { + async decodeResponseAsync(payload: string, provenance?: DecodedProvenance): Promise { const result = this.parseResponseResult(payload); // Decode scientific envelopes in the result. let decoded: unknown; try { - decoded = await decodeScientificValue(result); + decoded = await decodeScientificValue(result, provenance); } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); const decodeError = err instanceof ScientificDecodeError ? err : undefined; diff --git a/src/runtime/decoded-provenance.ts b/src/runtime/decoded-provenance.ts index a983406c..945936af 100644 --- a/src/runtime/decoded-provenance.ts +++ b/src/runtime/decoded-provenance.ts @@ -30,6 +30,7 @@ export class DecodedProvenance { } recordChild(parent: object, key: string | number, metadata: DecodedShapeMetadata): void { + key = this.normalizeKey(parent, key); let entries = this.children.get(parent); if (!entries) { entries = new Map(); @@ -47,7 +48,19 @@ export class DecodedProvenance { } atChild(parent: object, key: string | number): DecodedShapeMetadata | undefined { - return this.children.get(parent)?.get(key); + return this.children.get(parent)?.get(this.normalizeKey(parent, key)); + } + + private normalizeKey(parent: object, key: string | number): string | number { + if ( + Array.isArray(parent) && + typeof key === 'string' && + /^(0|[1-9]\d*)$/.test(key) && + Number.isSafeInteger(Number(key)) + ) { + return Number(key); + } + return key; } private claimEntry(): void { diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index 3eee208f..e475d10a 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -23,6 +23,7 @@ import type { BridgeBackend, BridgeInfo } from '../types/index.js'; import { DisposableBase, type ExecuteOptions } from './bounded-context.js'; import { BridgeProtocolError } from './errors.js'; import { BridgeCodec, type CodecOptions } from './bridge-codec.js'; +import { DecodedProvenance } from './decoded-provenance.js'; import { PROTOCOL_ID, TYWRAP_PROTOCOL_VERSION, @@ -431,9 +432,10 @@ export class RpcClient extends DisposableBase { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise { - return this.sendMessageAsync( + const provenance = validate ? new DecodedProvenance() : undefined; + return this.sendVia( { method: 'call', params: { @@ -446,11 +448,12 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - validate(result); + validate(result, provenance); return result; }, } - : undefined + : undefined, + responseStr => this.codec.decodeResponseAsync(responseStr, provenance) ); } diff --git a/src/types/index.ts b/src/types/index.ts index c5b0d408..7ff3dd42 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -2,6 +2,8 @@ * Core type definitions for tywrap */ +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; + export interface PythonModule { name: string; path?: string; @@ -503,7 +505,7 @@ export interface PythonRuntime { functionName: string, args: unknown[], kwargs?: Record, - validate?: (result: T) => void + validate?: (result: T, provenance?: DecodedProvenance) => void ): Promise; } diff --git a/src/utils/codec.ts b/src/utils/codec.ts index 01d86601..dec055a9 100644 --- a/src/utils/codec.ts +++ b/src/utils/codec.ts @@ -10,6 +10,7 @@ */ import { tagDecodedShape } from '../runtime/validators.js'; +import type { DecodedProvenance } from '../runtime/decoded-provenance.js'; import { isSafeJsonInteger, MAX_SAFE_JSON_INTEGER } from '../contracts/value-contract.js'; const SCIENTIFIC_MARKERS = [ @@ -1334,7 +1335,8 @@ function decodeEnvelope( async function decodeEnvelopeAsync( value: unknown, - decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise + decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise, + provenance?: DecodedProvenance ): Promise { let visitedNodes = 0; @@ -1397,6 +1399,34 @@ async function decodeEnvelopeAsync( } }; + const recordScalarProvenance = ( + original: unknown, + decoded: unknown, + target?: DecodeTarget + ): void => { + if ( + !provenance || + typeof decoded !== 'number' || + !isPlainObject(original) || + original.__tywrap__ !== 'ndarray' || + !isStrictV1Envelope(original) || + !Array.isArray(original.shape) || + original.shape.length !== 0 + ) { + return; + } + const metadata = { + marker: 'ndarray' as const, + dims: 0, + dtype: typeof original.dtype === 'string' ? original.dtype : undefined, + }; + if (target) { + provenance.recordChild(target.container, target.key, metadata); + } else { + provenance.recordRoot(metadata); + } + }; + const settleDecoded = async ( decoded: PromiseLike, original: unknown, @@ -1409,6 +1439,7 @@ async function decodeEnvelopeAsync( } catch (error) { return prefixHandlerError(error, path); } + recordScalarProvenance(original, resolved, path === 'result' ? undefined : target); assignDecoded(target, original, resolved); }; @@ -1455,8 +1486,10 @@ async function decodeEnvelopeAsync( pending.push(settleDecoded(decoded, current, { container: root, key: 'value' }, path)); } } else if (target) { + recordScalarProvenance(current, decoded, target); assignDecoded(target, current, decoded); } else if (decoded !== current) { + recordScalarProvenance(current, decoded); root.value = decoded; } return; @@ -1530,9 +1563,12 @@ async function decodeEnvelopeAsync( /** * Decode values produced by the Python bridge. */ -export async function decodeValueAsync(value: unknown): Promise { +export async function decodeValueAsync( + value: unknown, + provenance?: DecodedProvenance +): Promise { try { - return await decodeEnvelopeAsync(value, tryDecodeArrowTable); + return await decodeEnvelopeAsync(value, tryDecodeArrowTable, provenance); } catch (error) { throw asScientificDecodeError(error, 'unknown'); } diff --git a/test/bridge-codec.test.ts b/test/bridge-codec.test.ts index fe83668f..fd38e01a 100644 --- a/test/bridge-codec.test.ts +++ b/test/bridge-codec.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { BridgeCodec, type CodecOptions } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { BridgeCodecError, BridgeProtocolError, @@ -633,6 +634,71 @@ describe('decodeResponseAsync - Arrow Integration', () => { }); }); + it('records scalar ndarray proof at the root and nested array and record slots', async () => { + const scalar = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype: 'float16', + }; + const rootProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: scalar }), + rootProof + ) + ).toBe(1.5); + expect(rootProof.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + + const nestedProof = new DecodedProvenance(); + const decoded = await codec.decodeResponseAsync<{ tuple: [number, { value: number }] }>( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { tuple: [scalar, { value: scalar }] }, + }), + nestedProof + ); + expect(decoded).toEqual({ tuple: [1.5, { value: 1.5 }] }); + expect(nestedProof.atRoot()).toBeUndefined(); + expect(nestedProof.atChild(decoded.tuple, '0')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple[1], 'value')?.dtype).toBe('float16'); + expect(nestedProof.atChild(decoded.tuple, 1)).toBeUndefined(); + }); + + it('does not record proof for plain numbers or malformed scalar envelopes', async () => { + const plainProof = new DecodedProvenance(); + expect( + await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: 1.5 }), + plainProof + ) + ).toBe(1.5); + expect(plainProof.atRoot()).toBeUndefined(); + + const malformedProof = new DecodedProvenance(); + await expect( + codec.decodeResponseAsync( + JSON.stringify({ + id: 1, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + }, + }), + malformedProof + ) + ).rejects.toThrow(/must be an array at depth 0/); + expect(malformedProof.atRoot()).toBeUndefined(); + }); + it('labels v1 scientific-envelope validation failures without Arrow wording', async () => { const payload = JSON.stringify({ id: 1, diff --git a/test/decoded_provenance.test.ts b/test/decoded_provenance.test.ts index 424890ad..55f756ef 100644 --- a/test/decoded_provenance.test.ts +++ b/test/decoded_provenance.test.ts @@ -19,6 +19,15 @@ describe('decoded provenance', () => { expect(second.atChild(secondParent, 'one')?.dtype).toBe('float64'); }); + it('uses the same array slot for a numeric or string index', () => { + const parent = [1.5]; + const provenance = new DecodedProvenance(); + const metadata = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; + provenance.recordChild(parent, 0, metadata); + expect(provenance.atChild(parent, '0')).toEqual(metadata); + expect(() => provenance.recordChild(parent, '0', metadata)).toThrow(/Duplicate child/); + }); + it('has one root slot and a bounded number of entries', () => { const evidence = { marker: 'ndarray' as const, dims: 0, dtype: 'float16' }; const provenance = new DecodedProvenance(2); diff --git a/test/rpc-client.test.ts b/test/rpc-client.test.ts index b9aaa1eb..1bbc4255 100644 --- a/test/rpc-client.test.ts +++ b/test/rpc-client.test.ts @@ -503,6 +503,71 @@ describe('BridgeProtocol', () => { const parsed = JSON.parse(transport.lastMessage!); expect(parsed.params.kwargs).toEqual({ key: 'value' }); }); + + it('keeps equal scalar proof with each concurrent call', async () => { + transport.send = async message => { + const request = JSON.parse(message); + const dtype = request.params.functionName === 'half' ? 'float16' : 'float32'; + await new Promise(resolve => setTimeout(resolve, dtype === 'float16' ? 5 : 0)); + return JSON.stringify({ + id: request.id, + protocol: 'tywrap/1', + result: { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [], + dtype, + }, + }); + }; + + const seen: string[] = []; + const [half, full] = await Promise.all([ + protocol.call('module', 'half', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float16'); + seen.push('half'); + }), + protocol.call('module', 'full', [], undefined, (value, provenance) => { + expect(value).toBe(1.5); + expect(provenance?.atRoot()?.dtype).toBe('float32'); + seen.push('full'); + }), + ]); + expect([half, full]).toEqual([1.5, 1.5]); + expect(seen).toHaveLength(2); + }); + + it('does not pass rejected decode proof to validation or a later call', async () => { + let bad = true; + transport.setDynamicResponse(() => + bad + ? { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding: 'json', + data: 1.5, + shape: [1], + dtype: 'float16', + } + : 1.5 + ); + const validate = vi.fn((_value: number, provenance?: { atRoot(): unknown }) => { + expect(provenance?.atRoot()).toBeUndefined(); + }); + + await expect(protocol.call('module', 'bad', [], undefined, validate)).rejects.toThrow( + /must be an array at depth 0/ + ); + expect(validate).not.toHaveBeenCalled(); + bad = false; + await expect(protocol.call('module', 'plain', [], undefined, validate)).resolves.toBe( + 1.5 + ); + expect(validate).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/test/value_contracts.test.ts b/test/value_contracts.test.ts index 264ee5d2..db366cc2 100644 --- a/test/value_contracts.test.ts +++ b/test/value_contracts.test.ts @@ -1,5 +1,7 @@ import { readFileSync } from 'node:fs'; import { afterEach, describe, expect, it } from 'vitest'; +import { BridgeCodec } from '../src/runtime/bridge-codec.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { MAX_SAFE_JSON_INTEGER, VALUE_CONTRACT_REVISION, @@ -15,7 +17,13 @@ import { const specification = JSON.parse( readFileSync(new URL('../docs/maintainers/value-contracts.v2.json', import.meta.url), 'utf8') -) as { revision: number; rules: { integer: { minimum: number; maximum: number } } }; +) as { + revision: number; + rules: { + integer: { minimum: number; maximum: number }; + bytes: { decodedAs: string }; + }; +}; const fixtures = JSON.parse( readFileSync( new URL('../docs/maintainers/value-contract-fixtures.v2.json', import.meta.url), @@ -23,6 +31,12 @@ const fixtures = JSON.parse( ) ) as { revision: number; + bytesCases: readonly { + logicalHex: string; + requestEnvelope: object; + responseEnvelope: object; + decodedAs: string; + }[]; binary16Cases: readonly { wordHex: string; outcome: string }[]; }; @@ -85,6 +99,23 @@ describe('frozen value policy', () => { expect(isSafeJsonInteger(-(2 ** 53))).toBe(false); expect(isSafeJsonInteger(1.5)).toBe(false); }); + + it('matches the shared bytes request and response fixtures', async () => { + const codec = new BridgeCodec(); + expect(specification.rules.bytes.decodedAs).toBe('JavaScript Uint8Array'); + for (const fixture of fixtures.bytesCases) { + const bytes = Uint8Array.from(Buffer.from(fixture.logicalHex, 'hex')); + const request = JSON.parse(codec.encodeRequest({ value: bytes })); + expect(request.value).toEqual(fixture.requestEnvelope); + + const decoded = await codec.decodeResponseAsync( + JSON.stringify({ id: 1, protocol: 'tywrap/1', result: fixture.responseEnvelope }) + ); + expect(decoded).toBeInstanceOf(Uint8Array); + expect([...decoded]).toEqual([...bytes]); + expect(`Uint8Array[${[...decoded].join(',')}]`).toBe(fixture.decodedAs); + } + }); }); describe('Arrow float16 value contract', () => { @@ -129,6 +160,13 @@ describe('Arrow float16 value contract', () => { ]); }); + it('carries Arrow scalar float16 proof with the decoded number', async () => { + registerWords([0x3e00]); + const provenance = new DecodedProvenance(); + expect(await decodeValueAsync(envelope([]), provenance)).toBe(1.5); + expect(provenance.atRoot()).toEqual({ marker: 'ndarray', dims: 0, dtype: 'float16' }); + }); + it('uses the ndarray rule inside nested Torch and record values', async () => { registerWords([0x3e00, 0xc080]); const value = { From 7e2301cdd6878a1423cf6787c219ac81925e7d47 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:20:22 -0700 Subject: [PATCH 3/5] fix(rpc): keep one-argument validator call shape --- src/runtime/rpc-client.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index e475d10a..aa1ad6a6 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -448,7 +448,12 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - validate(result, provenance); + // Keep the original call shape for validators that accept one argument. + if (validate.length > 1) { + validate(result, provenance); + } else { + validate(result); + } return result; }, } From a642527bc2acd74387aa81b7bc8d6c80fb7ffea5 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:21:30 -0700 Subject: [PATCH 4/5] test(rpc): assert additive validator provenance --- src/runtime/rpc-client.ts | 7 +------ test/runtime_return_validation.test.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/runtime/rpc-client.ts b/src/runtime/rpc-client.ts index aa1ad6a6..e475d10a 100644 --- a/src/runtime/rpc-client.ts +++ b/src/runtime/rpc-client.ts @@ -448,12 +448,7 @@ export class RpcClient extends DisposableBase { validate ? { validate: result => { - // Keep the original call shape for validators that accept one argument. - if (validate.length > 1) { - validate(result, provenance); - } else { - validate(result); - } + validate(result, provenance); return result; }, } diff --git a/test/runtime_return_validation.test.ts b/test/runtime_return_validation.test.ts index 033bd349..e4a78b18 100644 --- a/test/runtime_return_validation.test.ts +++ b/test/runtime_return_validation.test.ts @@ -3,6 +3,7 @@ import type { AddressInfo } from 'node:net'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { BridgeValidationError } from '../src/runtime/errors.js'; +import { DecodedProvenance } from '../src/runtime/decoded-provenance.js'; import { HttpBridge } from '../src/runtime/http.js'; import { NodeBridge } from '../src/runtime/node.js'; import { @@ -220,7 +221,17 @@ describe('return validator bridge propagation', () => { try { await expect(bridge.call('math', 'sqrt', [16], undefined, validate)).resolves.toBe(4); expect(validate).toHaveBeenCalledOnce(); - expect(validate).toHaveBeenCalledWith(4); + expect(validate).toHaveBeenCalledWith(4, expect.any(DecodedProvenance)); + const [, provenance] = validate.mock.calls[0] as unknown as [number, DecodedProvenance]; + expect(provenance.atRoot()).toBeUndefined(); + + const reject = vi.fn((_value: number) => { + throw new Error('one-argument validator rejected the value'); + }); + await expect(bridge.call('math', 'sqrt', [16], undefined, reject)).rejects.toThrow( + 'one-argument validator rejected the value' + ); + expect(reject).toHaveBeenCalledOnce(); } finally { await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve())) From e14d2f31c93c66a22ffa6695e97355f2c799cae5 Mon Sep 17 00:00:00 2001 From: Brett Bonner Date: Sun, 20 Sep 2026 01:26:31 -0700 Subject: [PATCH 5/5] fix(codec): bind Torch float16 to nested ndarray dtype --- docs/maintainers/value-contracts.md | 3 +- src/utils/codec.ts | 13 +++++++ test/value_contracts.test.ts | 56 +++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/maintainers/value-contracts.md b/docs/maintainers/value-contracts.md index dbc55a31..20c8293f 100644 --- a/docs/maintainers/value-contracts.md +++ b/docs/maintainers/value-contracts.md @@ -23,7 +23,8 @@ constraint. Regenerate wrappers after that compiler change. Do not return The bytes leaf keeps the existing base64 envelopes. Requests use `__tywrap_bytes__`; responses use `__type__: bytes`. Both decode to `Uint8Array`. The Torch float16 contract names `torch.float16` on the outer -tensor and `float16` on its nested ndarray. +tensor and `float16` on its nested version 1 ndarray. The decoder rejects a +different nested dtype before it decodes the data. Migration for users with large Python integers: convert them to strings in Python with an explicit return annotation, or keep integer columns in Arrow diff --git a/src/utils/codec.ts b/src/utils/codec.ts index dec055a9..83d26a2b 100644 --- a/src/utils/codec.ts +++ b/src/utils/codec.ts @@ -1109,6 +1109,19 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = ( if (strictV1 && 'dtype' in nested) { assertOptionalNonEmptyDtype(nested.dtype, 'torch.tensor', 'value.dtype', nestedShape); } + if (strictV1 && dtype === 'torch.float16') { + if (!isStrictV1Envelope(nested)) { + throw new Error( + 'Invalid torch.tensor envelope: float16 value must use ndarray codecVersion 1' + ); + } + if (nested.dtype !== 'float16') { + throw new Error( + `Invalid torch.tensor envelope: value.dtype at path value.dtype must be "float16" ` + + `for outer dtype "torch.float16"; got ${JSON.stringify(nested.dtype)}` + ); + } + } const deviceValue = value.device; if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) { throw new Error( diff --git a/test/value_contracts.test.ts b/test/value_contracts.test.ts index db366cc2..71a7eaed 100644 --- a/test/value_contracts.test.ts +++ b/test/value_contracts.test.ts @@ -187,6 +187,62 @@ describe('Arrow float16 value contract', () => { }); }); + it.each([ + { shape: [], encoding: 'json' }, + { shape: [1], encoding: 'json' }, + { shape: [], encoding: 'arrow' }, + { shape: [1], encoding: 'arrow' }, + ] as const)( + 'requires a matching nested dtype for Torch float16 shape $shape with $encoding data', + async ({ shape, encoding }) => { + registerWords([0x3e00]); + const nested = { + __tywrap__: 'ndarray', + codecVersion: 1, + encoding, + shape, + dtype: 'float16', + ...(encoding === 'arrow' ? { b64: 'AA==' } : { data: shape.length ? [1.5] : 1.5 }), + }; + const tensor = { + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + value: nested, + shape, + dtype: 'torch.float16', + device: 'cpu', + }; + + await expect(decodeValueAsync(tensor)).resolves.toMatchObject({ + data: shape.length ? [1.5] : 1.5, + dtype: 'torch.float16', + }); + await expect( + decodeValueAsync({ ...tensor, value: { ...nested, dtype: 'float32' } }) + ).rejects.toThrow(/value\.dtype.*must be "float16".*"torch\.float16"/); + } + ); + + it('requires the nested version 1 envelope for Torch float16', async () => { + await expect( + decodeValueAsync({ + __tywrap__: 'torch.tensor', + codecVersion: 1, + encoding: 'ndarray', + shape: [], + dtype: 'torch.float16', + value: { + __tywrap__: 'ndarray', + encoding: 'json', + shape: [], + dtype: 'float16', + data: 1.5, + }, + }) + ).rejects.toThrow(/float16 value must use ndarray codecVersion 1/); + }); + it.each([0x7c00, 0xfc00, 0x7e00])('rejects non-finite word %s', word => { registerWords([word]); expect(() => decodeValue(envelope([1]))).toThrow(/non-finite float16 value at b64\[0\]/);