NumpyCodec.cs

using System;
using Python.Runtime;

namespace NumSharp.Interop.PythonNet
{
    /// <summary>
    ///     How <see cref="NumpyCodec"/> materializes a conversion in one direction — encode
    ///     (<see cref="NDArray"/> → Python) or decode (Python → <see cref="NDArray"/>). One enum serves
    ///     both directions.
    /// </summary>
    public enum NumpyCodecMode
    {
        /// <summary>
        ///     <b>Zero-copy VIEW when the dtype and layout permit it; an independent COPY only when a
        ///     view is impossible.</b> Shared memory whenever it is achievable, a safe copy as the
        ///     fallback — never a blanket copy.
        ///
        ///     <para>On <b>decode</b> the fallback is real: complex64 (widened), UCS-4 text (narrowed to
        ///     Char, BMP only), and sub-item strides (linearized) have no zero-copy NumSharp
        ///     representation and transparently become copies, while contiguous / strided / non-numpy
        ///     exporter sources stay zero-copy views (read-only sources become NON-WRITEABLE views);
        ///     big-endian multi-byte data is refused by BOTH paths rather than silently byte-swapped.
        ///     On <b>encode</b> a view and a copy have identical dtype coverage — both need a
        ///     numpy-expressible dtype — so Auto always yields a view, and the only unrepresentable
        ///     dtype (<see cref="NPTypeCode.Decimal"/>) falls through to pythonnet's CLR wrapping
        ///     either way.</para>
        /// </summary>
        Auto = 0,

        /// <summary>
        ///     <b>Always a zero-copy VIEW</b> (shared memory, live mutation both ways). A conversion that
        ///     cannot be expressed as a view is DECLINED (the encode/decode fails) rather than silently
        ///     copying — for callers that depend on the shared-memory contract and want a loud failure
        ///     otherwise. Read-only decode sources become NON-WRITEABLE views.
        /// </summary>
        View = 1,

        /// <summary>
        ///     <b>Always an independent COPY</b> — no shared memory, no lifetime coupling, no Py_buffer
        ///     lock on the source, total dtype/layout coverage. The most predictable mode.
        /// </summary>
        Copy = 2,
    }

    /// <summary>
    ///     Policies for <see cref="NumpyCodec"/> / <see cref="NDArrayPythonInterop.RegisterCodec()"/>.
    /// </summary>
    public sealed class NumpyCodecOptions
    {
        /// <summary>The defaults: <see cref="NumpyCodecMode.Auto"/> both ways, decode registered
        /// adapters, any buffer exporter, and numeric builtin array-likes.</summary>
        public static readonly NumpyCodecOptions Default = new();

        /// <summary>
        ///     How <see cref="NDArray"/> values crossing INTO Python are materialized. Default
        ///     <see cref="NumpyCodecMode.Auto"/> — a zero-copy numpy view
        ///     (<see cref="NDArrayPythonInterop.ToNumpy(NDArray, bool?)"/> — shared mutation, source
        ///     rooted). On encode a view and a copy have the same dtype coverage, so
        ///     <see cref="NumpyCodecMode.Auto"/> and <see cref="NumpyCodecMode.View"/> behave identically
        ///     and only <see cref="NumpyCodecMode.Copy"/> forces
        ///     <see cref="NDArrayPythonInterop.ToNumpyCopy"/>.
        /// </summary>
        public NumpyCodecMode EncodeMode { get; init; } = NumpyCodecMode.Auto;

        /// <summary>
        ///     How Python objects crossing INTO .NET (<c>PyObject.As&lt;NDArray&gt;()</c>) are
        ///     materialized. Default <see cref="NumpyCodecMode.Auto"/> — a zero-copy view
        ///     (<see cref="NDArrayPythonInterop.ToNDArrayView"/> with <c>allowReadonly:true</c>) when the
        ///     source's dtype/layout permits, otherwise an independent copy
        ///     (<see cref="NDArrayPythonInterop.ToNDArray"/>). <see cref="NumpyCodecMode.View"/> declines
        ///     the decode when a view is impossible; <see cref="NumpyCodecMode.Copy"/> always copies.
        ///
        ///     <para><b>Auto/View share memory:</b> mutations flow both ways, read-only sources decode as
        ///     NON-WRITEABLE views (guarded writes through them throw instead of corrupting immutable
        ///     Python objects), and a live view holds a Py_buffer lock on the source (a <c>bytearray</c>
        ///     cannot be resized while the view lives). Choose <see cref="NumpyCodecMode.Copy"/> for a
        ///     detached snapshot that never touches the Python object.</para>
        /// </summary>
        public NumpyCodecMode DecodeMode { get; init; } = NumpyCodecMode.Auto;

        /// <summary>
        ///     <c>true</c> (default): besides <c>numpy.ndarray</c> (and subclasses), ANY object exporting
        ///     the PEP 3118 buffer protocol decodes to <see cref="NDArray"/> — the built-ins
        ///     <c>memoryview</c>, <c>bytes</c>, <c>bytearray</c>, <c>array.array</c> by name, plus every
        ///     other exporter (<c>ctypes</c> arrays, C-extension buffers, ...) via the PEP 688
        ///     <c>__buffer__</c> capability check on Python 3.12+. <c>false</c>: numpy arrays only.
        /// </summary>
        /// <remarks>
        ///     The capability check exists because exporters like <c>ctypes</c> arrays have a tp_name
        ///     generated per element type AND length (<c>c_long_Array_4</c>), so they can never be
        ///     allowlisted. On Python ≤ 3.11 there is no <c>__buffer__</c> dunder, so acceptance falls
        ///     back to the four built-in names above.
        /// </remarks>
        public bool DecodeAnyBuffer { get; init; } = true;

        /// <summary>
        ///     <c>true</c> (default): objects recognized by the process-wide
        ///     <see cref="IPythonArrayAdapter"/> registry participate in implicit
        ///     <c>PyObject.As&lt;NDArray&gt;()</c> / <c>AsManagedObject(typeof(NDArray))</c> conversion.
        ///     Built-in adapters cover <c>torch.Tensor</c> and Pandas <c>DataFrame</c>, <c>Series</c>,
        ///     <c>Index</c>, and <c>ExtensionArray</c> objects (including subclasses).
        ///     <c>false</c>: registered adapters are ignored by this codec; NumPy/buffer/builtin options
        ///     remain independent.
        /// </summary>
        /// <remarks>
        ///     Adapter conversion obeys <see cref="DecodeMode"/>. View invokes only a share-preserving
        ///     adapter route; Copy permits detach/device transfer/materialization; Auto tries the view
        ///     and then the copy route. Thus default Auto can copy a CUDA or gradient-tracking Torch
        ///     tensor to CPU when an implicit NDArray conversion is explicitly requested.
        /// </remarks>
        public bool DecodeArrayAdapters { get; init; } = true;

        /// <summary>
        ///     <c>true</c> (default): besides registered adapters and PEP 3118 buffer exporters,
        ///     everyday array-like Python objects that export NO buffer — a <c>list</c>, <c>tuple</c>,
        ///     nested sequence, or a Python scalar
        ///     (<c>int</c>/<c>float</c>/<c>bool</c>/<c>complex</c>) — also decode to <see cref="NDArray"/>,
        ///     materialized through <c>numpy.asarray</c> (<see cref="NDArrayPythonInterop.FromArrayLike"/>),
        ///     so a numpy call or plain Python code that hands back a list crosses without a manual
        ///     conversion. <c>false</c>: builtin array-likes are disabled; registered adapters and
        ///     buffer exporters remain controlled by their own independent options.
        /// </summary>
        /// <remarks>
        ///     Two consequences worth knowing (both benign, neither a reason to turn it off in practice):
        ///     an array-like decode makes that ONE conversion numpy-dependent (the buffer path stays
        ///     numpy-agnostic — and numpy is already required for every encode), and it is ALWAYS an
        ///     independent copy regardless of <see cref="DecodeMode"/> (a <c>list</c> has no shared memory
        ///     to view). It deliberately covers only the array-like builtins numpy can express as a numeric
        ///     ndarray; <c>str</c> (a char sequence), <c>dict</c>, and <c>range</c> are not accepted.
        /// </remarks>
        public bool DecodeArrayLike { get; init; } = true;

        /// <summary>
        ///     <c>true</c> (default): <see cref="NDArrayPythonInterop.RegisterCodec()"/> also registers
        ///     <see cref="TupleCodec"/>, so a C# <see cref="ValueTuple"/> / <see cref="Tuple"/> crosses into
        ///     Python as a <c>tuple</c> (<c>numpy.zeros((2, 3))</c>, <c>a.item((1, 2))</c>, a shape bound
        ///     into a scope) and a Python <c>tuple</c> decodes into a C# tuple of the same arity
        ///     (<c>(long, long) shape = a.shape</c>). pythonnet itself has no tuple conversion — a C# tuple
        ///     would otherwise reach Python as an opaque wrapped CLR object. <c>false</c>: tuples are left
        ///     to pythonnet's default handling (or to a tuple codec of your own).
        /// </summary>
        public bool ConvertTuples { get; init; } = true;
    }

    /// <summary>
    ///     pythonnet auto-marshaling codec: once registered (see
    ///     <see cref="NDArrayPythonInterop.RegisterCodec()"/>), <see cref="NDArray"/> → numpy and
    ///     supported Python array → <see cref="NDArray"/> conversion happens automatically at every
    ///     pythonnet boundary — <c>nd.ToPython()</c>, <c>scope.Set("x", nd)</c>, passing an
    ///     <see cref="NDArray"/> to a Python callable, and <c>pyObj.As&lt;NDArray&gt;()</c> /
    ///     <c>AsManagedObject(typeof(NDArray))</c> on the way back. Registered adapters make the reverse
    ///     path include library arrays such as <c>torch.Tensor</c> and Pandas containers without
    ///     duplicating memory code.
    ///
    ///     <para>Encoding falls back to pythonnet's default CLR-object wrapping for arrays it cannot
    ///     express as numpy (<see cref="NPTypeCode.Decimal"/>), instead of failing the conversion.</para>
    /// </summary>
    public sealed class NumpyCodec : IPyObjectEncoder, IPyObjectDecoder
    {
        private readonly NumpyCodecOptions _options;

        /// <summary>Create a codec with <see cref="NumpyCodecOptions.Default"/>.</summary>
        public NumpyCodec() : this(NumpyCodecOptions.Default) { }

        /// <summary>Create a codec with explicit <paramref name="options"/>.</summary>
        public NumpyCodec(NumpyCodecOptions options)
        {
            _options = options ?? throw new ArgumentNullException(nameof(options));
        }

        // ---- CLR -> Python -----------------------------------------------------------------------

        /// <inheritdoc/>
        public bool CanEncode(Type type) => typeof(NDArray).IsAssignableFrom(type);

        /// <inheritdoc/>
        /// <remarks>
        ///     The returned wrapper goes through <see cref="EncoderHandoff"/>: pythonnet takes its own reference
        ///     from it and never disposes it, so without the handoff every implicit encode would keep the
        ///     export pinned until the CLR finalizer ran.
        /// </remarks>
        public PyObject TryEncode(object value)
        {
            var nd = (NDArray)value;
            try
            {
                PyObject encoded;
                switch (_options.EncodeMode)
                {
                    case NumpyCodecMode.Copy:
                        encoded = NDArrayPythonInterop.ToNumpyCopy(nd);
                        break;
                    case NumpyCodecMode.View:
                        encoded = NDArrayPythonInterop.ToNumpy(nd);
                        break;
                    default: // Auto: view-first, copy-fallback. On encode both need a numpy dtype, so the
                             // fallback is a no-op today (every dtype encodes — Decimal converts to
                             // float64); kept for a uniform contract and any future dtype divergence.
                        try { encoded = NDArrayPythonInterop.ToNumpy(nd); }
                        catch (NotSupportedException) { encoded = NDArrayPythonInterop.ToNumpyCopy(nd); }
                        break;
                }

                return EncoderHandoff.Hand(encoded);
            }
            catch (NotSupportedException)
            {
                // Defensive: every NumSharp dtype now encodes (Decimal converts to float64 in ToNumpy),
                // so this is unreachable today. Should a future dtype have no numpy expression at all, we
                // let pythonnet wrap the NDArray as a plain CLR object, still usable from Python.
                return null;
            }
        }

        // ---- Python -> CLR -----------------------------------------------------------------------

        /// <inheritdoc/>
        public bool CanDecode(PyType objectType, Type targetType)
        {
            if (targetType != typeof(NDArray))
                return false;

            try
            {
                string name = objectType.Name;   // tp_name: "numpy.ndarray", "memoryview", "array.array", ...
                if (name == "numpy.ndarray")
                    return true;
                if (_options.DecodeArrayAdapters && PythonArrayAdapterRegistry.CanAdapt(objectType))
                    return true;
                if (_options.DecodeAnyBuffer)
                {
                    if (name == "memoryview" || name == "bytes" || name == "bytearray" || name == "array.array")
                        return true;
                    // Capability check instead of a name allowlist: PEP 688 (Python 3.12+) puts
                    // __buffer__ on EVERY type implementing the C buffer protocol, so any exporter
                    // decodes — including ones whose tp_name cannot be enumerated at all (a ctypes
                    // array is named per element type AND length: "c_long_Array_4", "c_double_Array_3",
                    // ...). Non-buffer builtins (dict/int/str/list/float) do not have it, so this never
                    // widens beyond real exporters. On Python <= 3.11 the dunder does not exist, the
                    // check is simply false, and the allowlist above still covers the common types.
                    if (objectType.HasAttr("__buffer__"))
                        return true;
                }

                // Array-like builtins export NO buffer, so they are keyed off tp_name (never HasAttr):
                // numpy.asarray materializes them (FromArrayLike). Opt-in — it makes decode numpy-dependent.
                if (_options.DecodeArrayLike && IsArrayLikeBuiltin(name))
                    return true;

                return IsNdarraySubclass(objectType);
            }
            catch
            {
                return false;   // CanDecode must never throw into pythonnet's conversion pipeline
            }
        }

        /// <inheritdoc/>
        public bool TryDecode<T>(PyObject pyObj, out T value)
        {
            value = default;
            if (typeof(T) != typeof(NDArray))
                return false;

            NDArray nd = _options.DecodeMode switch
            {
                NumpyCodecMode.Copy => TryDecodeCopy(pyObj, _options.DecodeArrayAdapters),
                NumpyCodecMode.View => TryDecodeView(pyObj, _options.DecodeArrayAdapters),
                _                    => TryDecodeView(pyObj, _options.DecodeArrayAdapters) ??
                                        TryDecodeCopy(pyObj, _options.DecodeArrayAdapters),   // Auto
            };

            // Array-like fallback: a non-buffer list/tuple/scalar reaches here with nd == null because the
            // buffer routes above all decline it. It always materializes as a copy — there is no shared
            // memory to view — so it runs after, and independent of, the view/copy mode above. It is gated
            // on the object being a genuine array-like BUILTIN (IsArrayLikeObject): a numpy array is NEVER
            // rescued here, so a complex64/big-endian source that the view path declines still declines in
            // View mode instead of being silently copied through numpy.asarray.
            if (nd is null && _options.DecodeArrayLike && IsArrayLikeObject(pyObj))
                nd = TryDecodeArrayLike(pyObj);

            if (nd is null)
                return false;   // no view was possible in View mode, or an unsupported dtype/layout in any mode
            value = (T)(object)nd;
            return true;
        }

        /// <summary>
        ///     A zero-copy view over the source, or <c>null</c> when the source has no zero-copy NumSharp
        ///     representation (complex64 / big-endian / non-contiguous non-numpy / stride not an element
        ///     multiple). Any failure is treated as "not viewable" — the Auto path then copies, the View
        ///     path declines the decode.
        /// </summary>
        private static NDArray TryDecodeView(PyObject pyObj, bool useAdapters)
        {
            try
            {
                return useAdapters
                    ? NDArrayPythonInterop.ToNDArrayView(pyObj, allowReadonly: true)
                    : NDArrayPythonInterop.ToNDArrayViewWithoutAdapters(pyObj, allowReadonly: true);
            }
            catch { return null; }
        }

        /// <summary>An independent copy, or <c>null</c> if the source cannot be copied either (no NumSharp dtype).</summary>
        private static NDArray TryDecodeCopy(PyObject pyObj, bool useAdapters)
        {
            try
            {
                return useAdapters
                    ? NDArrayPythonInterop.ToNDArray(pyObj)
                    : NDArrayPythonInterop.ToNDArrayWithoutAdapters(pyObj);
            }
            catch { return null; }
        }

        /// <summary>
        ///     Materialize an array-like Python object (list / tuple / nested / scalar) through
        ///     <c>numpy.asarray</c>, or <c>null</c> when it is not array-like or has no NumSharp dtype
        ///     (a ragged list, an out-of-range bignum, a <c>dict</c>).
        /// </summary>
        private static NDArray TryDecodeArrayLike(PyObject pyObj)
        {
            try { return NDArrayPythonInterop.FromArrayLike(pyObj); }
            catch { return null; }
        }

        /// <summary>
        ///     The buffer-less Python builtins that <c>numpy.asarray</c> materializes into a numeric
        ///     ndarray — accepted for decode only when <see cref="NumpyCodecOptions.DecodeArrayLike"/> is
        ///     set. Keyed off the bare tp_name (<c>list</c>/<c>tuple</c>/<c>int</c>/<c>float</c>/<c>bool</c>/
        ///     <c>complex</c>). Deliberately excludes <c>str</c> (a char sequence → an unsupported string
        ///     dtype), <c>dict</c>, and <c>range</c>.
        /// </summary>
        private static bool IsArrayLikeBuiltin(string name)
            => name == "list" || name == "tuple" || name == "int" || name == "float" || name == "bool" || name == "complex";

        /// <summary>
        ///     True when <paramref name="obj"/>'s Python type is one of the array-like builtins
        ///     (<see cref="IsArrayLikeBuiltin"/>). Gates the array-like decode fallback in
        ///     <see cref="TryDecode{T}"/> so it fires ONLY for genuine non-buffer containers — a numpy
        ///     array (or any buffer exporter) the view/copy paths declined is never rescued by
        ///     <c>numpy.asarray</c>. Call under the GIL.
        /// </summary>
        private static bool IsArrayLikeObject(PyObject obj)
        {
            using PyObject type = obj.GetPythonType();
            using var pyType = new PyType(type);
            return IsArrayLikeBuiltin(pyType.Name);
        }

        /// <summary>Walks <c>__mro__</c> so numpy.matrix / numpy.memmap / user ndarray subclasses decode too.</summary>
        private static bool IsNdarraySubclass(PyType objectType)
        {
            using PyObject mro = objectType.GetAttr("__mro__");
            using var tup = PyTuple.AsTuple(mro);
            long n = tup.Length();
            for (int i = 0; i < n; i++)
            {
                using PyObject t = tup[i];
                using PyObject module = t.GetAttr("__module__");
                using PyObject name = t.GetAttr("__name__");
                if (module.As<string>() == "numpy" && name.As<string>() == "ndarray")
                    return true;
            }

            return false;
        }
    }
}