diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 2e5d60187395..cf5a90dd1b50 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -17,6 +17,7 @@ generate_am_send_wrapper, generate_bin_op_wrapper, generate_bool_wrapper, + generate_constructor_wrapper_function, generate_contains_wrapper, generate_dunder_wrapper, generate_get_wrapper, @@ -252,10 +253,40 @@ def generate_class_reuse( context.declarations[name] = HeaderDeclaration(f"CPyThreadLocal {struct_name} *{name};") -def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: +def can_use_constructor_for_type_call(cl: ClassIR) -> bool: + """Can a call to the type object of cl go straight to the native constructor? + + If so, the type object gets a tp_vectorcall that parses the arguments and calls + the native constructor. Otherwise a call goes through tp_new and tp_init. + """ + init = cl.get_method_and_class("__init__") + if init is None or "__init__" not in init[1].method_decls: + # Without __init__ there are no arguments to parse, and tp_new accepts and + # ignores any that are passed. A decorated __init__ isn't called natively. + return False + if cl.has_method("__new__") or cl.has_non_native_new(): + # If __new__ returns an instance of a subclass, type.__call__ calls __init__ of + # the subclass, but the native constructor doesn't. With a decorated __new__, + # the native constructor calls the type object. + return False + if cl.is_abstract: + # The only thing that stops an abstract class from being instantiated is the + # check of the self argument in the wrapper of __init__. + return False + # If __init__ isn't native, the constructor takes *args and **kwargs and calls + # tp_init. Building a tuple and a dict for it wouldn't be any faster. + return all(arg.kind not in (ARG_STAR, ARG_STAR2) for arg in cl.ctor.sig.args) + + +def generate_class( + cl: ClassIR, module: str, emitter: Emitter, source_paths: dict[str, str] +) -> None: """Generate C code for a class. This is the main entry point to the module. + + Args: + source_paths: Map from module names to source file paths """ name = cl.name name_prefix = cl.name_prefix(emitter.names) @@ -415,6 +446,15 @@ def emit_line() -> None: generate_new_for_class(cl, new_name, vtable_name, setup_name, init_fn, emitter) emit_line() + if init_fn is not None and can_use_constructor_for_type_call(cl): + vectorcall_name = f"{name_prefix}_vectorcall" + init_module = init_fn.decl.module_name + generate_constructor_wrapper_function( + cl, init_fn, vectorcall_name, emitter, source_paths[init_module], init_module + ) + fields["tp_vectorcall"] = vectorcall_name + emit_line() + if cl.allow_interpreted_subclasses: shadow_vtable_name: str | None = generate_vtables( cl, vtable_setup_name + "_shadow", vtable_name + "_shadow", emitter, shadow=True @@ -450,7 +490,7 @@ def emit_line() -> None: cl.struct_name(emitter.names) ) flags.append("_Py_TPFLAGS_HAVE_VECTORCALL") - if not fields.get("tp_vectorcall"): + if not fields.get("tp_call"): # This is just a placeholder to please CPython. It will be # overridden during setup. fields["tp_call"] = "PyVectorcall_Call" diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 6ebc74a96fb8..b336576c33d3 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -755,7 +755,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: for cl in module.classes: if cl.is_ext_class: - generate_class(cl, module_name, emitter) + generate_class(cl, module_name, emitter, self.source_paths) running_flag_classes = {cl.name: cl for cl in module.classes if cl.has_running_flag} @@ -1731,7 +1731,9 @@ def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool: if fn.name == "__call__": # We can use vectorcalls (PEP 590) when supported return True - # TODO: Support fastcall for __init__ and __new__. + # The wrappers of __init__ and __new__ are called from tp_init and tp_new, + # which get a tuple and a dict. A call to the type object can skip them, see + # can_use_constructor_for_type_call(). return fn.name != "__init__" and fn.name != "__new__" return True diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 7aeb664501d7..e21d91926532 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -26,7 +26,7 @@ bitmap_name, ) from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR, RuntimeArg +from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncDecl, FuncIR, RuntimeArg from mypyc.ir.rtypes import ( RInstance, RType, @@ -131,6 +131,23 @@ def make_format_string(func_name: str | None, groups: dict[ArgKind, list[Runtime return format +def stack_parse_function( + real_args: list[RuntimeArg], groups: dict[ArgKind, list[RuntimeArg]] +) -> str: + """Return the CPyArg_ParseStackAndKeywords* variant to use for the given arguments.""" + # Special case some common signatures + if not real_args: + # No args + return "CPyArg_ParseStackAndKeywordsNoArgs" + elif len(real_args) == 1 and len(groups[ARG_POS]) == 1: + # Single positional arg + return "CPyArg_ParseStackAndKeywordsOneArg" + elif len(real_args) == len(groups[ARG_POS]) + len(groups[ARG_OPT]): + # No keyword-only args, *args or **kwargs + return "CPyArg_ParseStackAndKeywordsSimple" + return "CPyArg_ParseStackAndKeywords" + + def generate_wrapper_function( fn: FuncIR, emitter: Emitter, source_path: str, module_name: str ) -> None: @@ -177,20 +194,9 @@ def generate_wrapper_function( nargs = "PyVectorcall_NARGS(nargs)" else: nargs = "nargs" - parse_fn = "CPyArg_ParseStackAndKeywords" - # Special case some common signatures - if not real_args: - # No args - parse_fn = "CPyArg_ParseStackAndKeywordsNoArgs" - elif len(real_args) == 1 and len(groups[ARG_POS]) == 1: - # Single positional arg - parse_fn = "CPyArg_ParseStackAndKeywordsOneArg" - elif len(real_args) == len(groups[ARG_POS]) + len(groups[ARG_OPT]): - # No keyword-only args, *args or **kwargs - parse_fn = "CPyArg_ParseStackAndKeywordsSimple" emitter.emit_lines( "if (!{}(args, {}, kwnames, &parser{})) {{".format( - parse_fn, nargs, "".join(", " + n for n in arg_ptrs) + stack_parse_function(real_args, groups), nargs, "".join(", " + n for n in arg_ptrs) ), "return NULL;", "}", @@ -210,6 +216,64 @@ def generate_wrapper_function( emitter.emit_line("}") +def generate_constructor_wrapper_function( + cl: ClassIR, + init_fn: FuncIR, + func_name: str, + emitter: Emitter, + source_path: str, + module_name: str, +) -> None: + """Generate a vectorcall wrapper for the native constructor of a class. + + This is used as tp_vectorcall of the type object, so that calling the type object + doesn't go through tp_new and tp_init, which get the arguments as a tuple and a + dict and pass them on to the legacy wrapper of __init__. + + The constructor takes the arguments of init_fn, and argument errors are reported + the same way as in the wrapper of init_fn. The source path and the module name are + those of init_fn, which cl may inherit from a class in another module. + """ + ctor = cl.ctor + emitter.emit_line("static PyObject *") + emitter.emit_line( + f"{func_name}(PyObject *type, PyObject *const *args, size_t nargs, PyObject *kwnames)" + ) + emitter.emit_line("{") + + real_args = list(ctor.sig.args) + if ctor.sig.num_bitmap_args: + real_args = real_args[: -ctor.sig.num_bitmap_args] + groups = make_arg_groups(real_args) + assert not groups[ARG_STAR] and not groups[ARG_STAR2], ctor.sig + reordered_args = reorder_arg_groups(groups) + + emitter.emit_line(make_static_kwlist(reordered_args)) + fmt = make_format_string(init_fn.name, groups) + emitter.emit_line(f'static CPyArg_Parser parser = {{"{fmt}", kwlist, 0}};') + for arg in real_args: + emitter.emit_line( + "PyObject *obj_{}{};".format(arg.name, " = NULL" if arg.optional else "") + ) + emitter.emit_lines( + "if (!{}(args, PyVectorcall_NARGS(nargs), kwnames, &parser{})) {{".format( + stack_parse_function(real_args, groups), + "".join(f", &obj_{arg.name}" for arg in reordered_args), + ), + "return NULL;", + "}", + ) + for i in range(ctor.sig.num_bitmap_args): + name = bitmap_name(i) + emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;") + traceback_code = generate_traceback_code(init_fn, emitter, source_path, module_name) + generate_wrapper_core( + ctor, emitter, groups[ARG_OPT] + groups[ARG_NAMED_OPT], traceback_code=traceback_code + ) + + emitter.emit_line("}") + + # Legacy generic wrapper functions # # These take a self object, a Python tuple of positional arguments, @@ -785,7 +849,7 @@ def generate_contains_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: def generate_wrapper_core( - fn: FuncIR, + fn: FuncIR | FuncDecl, emitter: Emitter, optional_args: list[RuntimeArg] | None = None, arg_names: list[str] | None = None, @@ -892,24 +956,27 @@ def __init__(self, cl: ClassIR | None, emitter: Emitter) -> None: self.optional_args: list[RuntimeArg] = [] self.traceback_code = "" - def set_target(self, fn: FuncIR) -> None: + def set_target(self, fn: FuncIR | FuncDecl) -> None: """Set the wrapped function. It's fine to modify the attributes initialized here later to customize the wrapper function. + + The native constructor of a class only has a declaration. """ - self.target_name = fn.name - self.target_cname = fn.cname(self.emitter.names) + decl = fn.decl if isinstance(fn, FuncIR) else fn + self.target_name = decl.name + self.target_cname = decl.cname(self.emitter.names) # Cached native-call expression so cross-group targets go through the # exports table; same as `NATIVE_PREFIX + cname` for in-group calls. - self.target_native_call = self.emitter.native_function_call(fn.decl) - self.num_bitmap_args = fn.sig.num_bitmap_args + self.target_native_call = self.emitter.native_function_call(decl) + self.num_bitmap_args = decl.sig.num_bitmap_args if self.num_bitmap_args: - self.args = fn.args[: -self.num_bitmap_args] + self.args = decl.sig.args[: -self.num_bitmap_args] else: - self.args = fn.args + self.args = decl.sig.args self.arg_names = [arg.name for arg in self.args] - self.ret_type = fn.ret_type + self.ret_type = decl.sig.ret_type def wrapper_name(self) -> str: """Return the name of the wrapper function.""" diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 160245c8996b..7cda15544299 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3710,6 +3710,309 @@ class Derived(Base): def test_inherited() -> None: assert Derived()() == 1 +[case testCallTypeObjectOfNativeClass] +from abc import ABC, abstractmethod +from typing import Any, overload + +from mypy_extensions import i64, mypyc_attr + +class Point: + def __init__(self, x: int, y: int, name: str = "p") -> None: + self.x = x + self.y = y + self.name = name + + @classmethod + def make(cls, x: int) -> "Point": + return cls(x, y=x + 1) + + def copy(self) -> "Point": + return type(self)(self.x, self.y, self.name) + +class Point3(Point): + pass + +class Origin(Point): + def __init__(self, name: str) -> None: + super().__init__(0, 0, name) + +class KwOnly: + def __init__(self, a: int, *, s: str = "x", req: float) -> None: + self.a = a + self.s = s + self.req = req + +class Fixed: + def __init__(self, x: i64 = 5, y: i64 = -1) -> None: + self.x = x + self.y = y + +class NoArgs: + def __init__(self) -> None: + self.x = 1 + +class Fails: + def __init__(self, x: int) -> None: + raise ValueError(str(x)) + +@mypyc_attr(allow_interpreted_subclasses=True) +class Open: + def __init__(self, a: int) -> None: + self.a = a + +class Adder: + def __call__(self, x: int) -> int: + return x + 1 + +class AdderN(Adder): + def __init__(self, n: int) -> None: + self.n = n + +# A call to the type object of one of the classes below doesn't go straight to the +# native constructor. + +class NoInit: + x = 3 + +calls: list[str] = [] + +def logged(f: Any) -> Any: + def wrapper(self: Any) -> None: + calls.append("logged") + f(self) + return wrapper + +class Decorated: + @logged + def __init__(self) -> None: + self.x = 1 + +class WithNew: + def __new__(cls, x: int) -> Any: + if cls is WithNew and x < 0: + # type.__call__ calls __init__ on an instance of a subclass. + return WithNewSub.__new__(WithNewSub, x) + return super().__new__(cls) + + def __init__(self, x: int) -> None: + self.x = x + +class WithNewSub(WithNew): + def __init__(self, x: int) -> None: + self.x = x * 10 + +def logged_new(f: Any) -> Any: + def wrapper(cls: Any, *args: Any) -> Any: + calls.append("logged_new") + return f(cls, *args) + return wrapper + +class DecoratedNew: + @logged_new + def __new__(cls, x: Any = None) -> Any: + return super().__new__(cls) + + # With an overloaded __init__ the native constructor takes no arguments + # (mypyc/mypyc#961) instead of *args and **kwargs. It calls the type object. + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, x: int) -> None: ... + def __init__(self, x: Any = None) -> None: + self.x = x + +class VarArgs: + def __init__(self, *args: int, **kwargs: str) -> None: + self.args = args + self.kwargs = kwargs + +class Shape(ABC): + def __init__(self, sides: int) -> None: + self.sides = sides + + @abstractmethod + def area(self) -> float: ... + +class Square(Shape): + def area(self) -> float: + return 1.0 + +[file other.py] +# Run in both interpreted and compiled mode +from typing import Any + +import native +from testutil import assertRaises + +# In compiled code a call to one of these is a call to the type object. A call to the +# class itself would be a direct call to the native constructor. +Point: Any = native.Point +Point3: Any = native.Point3 +Origin: Any = native.Origin +KwOnly: Any = native.KwOnly +Fixed: Any = native.Fixed +NoArgs: Any = native.NoArgs +Fails: Any = native.Fails +Open: Any = native.Open +AdderN: Any = native.AdderN +NoInit: Any = native.NoInit +Decorated: Any = native.Decorated +WithNew: Any = native.WithNew +DecoratedNew: Any = native.DecoratedNew +VarArgs: Any = native.VarArgs +Shape: Any = native.Shape +Square: Any = native.Square + +def test_arguments() -> None: + p = Point(1, 2) + assert (p.x, p.y, p.name) == (1, 2, "p") + p = Point(1, 2, "q") + assert (p.x, p.y, p.name) == (1, 2, "q") + p = Point(1, y=2, name="q") + assert (p.x, p.y, p.name) == (1, 2, "q") + p = Point(name="q", y=2, x=1) + assert (p.x, p.y, p.name) == (1, 2, "q") + args = (1, 2) + kwargs = {"name": "q"} + p = Point(*args, **kwargs) + assert (p.x, p.y, p.name) == (1, 2, "q") + p = Point(10**30, -(10**30)) + assert (p.x, p.y) == (10**30, -(10**30)) + + k = KwOnly(1, req=2.5) + assert (k.a, k.s, k.req) == (1, "x", 2.5) + k = KwOnly(req=2, s="q", a=1) + assert (k.a, k.s, k.req) == (1, "q", 2.0) + + f = Fixed() + assert (f.x, f.y) == (5, -1) + f = Fixed(7) + assert (f.x, f.y) == (7, -1) + f = Fixed(y=-(2**63)) + assert (f.x, f.y) == (5, -(2**63)) + + assert NoArgs().x == 1 + assert Open(7).a == 7 + assert Open(a=7).a == 7 + +def test_errors() -> None: + with assertRaises(TypeError, "__init__() missing required argument 'y' (pos 2)"): + Point(1) + with assertRaises(TypeError, "__init__() takes at most 3 arguments (4 given)"): + Point(1, 2, "q", 4) + with assertRaises(TypeError, "'z' is an invalid keyword argument for __init__()"): + Point(1, 2, z=3) + with assertRaises(TypeError, "argument for __init__() given by name ('x') and position (1)"): + Point(1, 2, x=3) + with assertRaises(TypeError, "int object expected; got str"): + Point("1", 2) + with assertRaises(TypeError, "str object expected; got int"): + Point(1, 2, name=3) + with assertRaises(TypeError, "__init__() missing required keyword-only argument 'req'"): + KwOnly(1) + with assertRaises(TypeError, "__init__() takes exactly 1 positional argument (2 given)"): + KwOnly(1, "q") + with assertRaises(TypeError, "i64 object expected; got int"): + Fixed(2**63) + with assertRaises(TypeError, "__init__() takes at most 0 arguments (1 given)"): + NoArgs(1) + with assertRaises(TypeError, "__init__() missing required argument 'a' (pos 1)"): + Open() + with assertRaises(ValueError, "5"): + Fails(5) + +def test_subclasses() -> None: + p = Point3(1, 2) + assert type(p) is native.Point3 + assert (p.x, p.y, p.name) == (1, 2, "p") + with assertRaises(TypeError, "__init__() missing required argument 'y' (pos 2)"): + Point3(1) + + o = Origin("o") + assert type(o) is native.Origin + assert (o.x, o.y, o.name) == (0, 0, "o") + with assertRaises(TypeError, "__init__() takes at most 1 argument (2 given)"): + Origin(1, 2) + +def test_calls_in_compiled_methods() -> None: + p = Point.make(4) + assert type(p) is native.Point + assert (p.x, p.y, p.name) == (4, 5, "p") + p = Point3.make(4) + assert type(p) is native.Point3 + assert (p.x, p.y, p.name) == (4, 5, "p") + + p = Point3(1, 2, "q").copy() + assert type(p) is native.Point3 + assert (p.x, p.y, p.name) == (1, 2, "q") + +def test_callable_instance() -> None: + a = AdderN(5) + assert a.n == 5 + assert a(1) == 2 + +def test_classes_without_native_constructor_call() -> None: + assert NoInit().x == 3 + + native.calls.clear() + assert Decorated().x == 1 + assert "logged" in native.calls + + assert WithNew(5).x == 5 + w = WithNew(-5) + assert type(w) is native.WithNewSub + assert w.x == -50 + + native.calls.clear() + assert DecoratedNew(5).x == 5 + assert DecoratedNew().x is None + assert native.calls == ["logged_new", "logged_new"] + + v = VarArgs(1, 2, a="b") + assert v.args == (1, 2) + assert v.kwargs == {"a": "b"} + + s = Square(4) + assert s.sides == 4 + with assertRaises(TypeError): + Shape(4) + +test_arguments() +test_errors() +test_subclasses() +test_calls_in_compiled_methods() +test_callable_instance() +test_classes_without_native_constructor_call() + +[file driver.py] +# Run the tests in both interpreted and compiled mode +import other +import other_interpreted + +from native import Open, Point + +# The slots that the type object used to be called through still work. +p = type.__call__(Point, 1, y=2) +assert (p.x, p.y, p.name) == (1, 2, "p") +p = Point.__new__(Point, 1, 2, "q") +assert (p.x, p.y, p.name) == (1, 2, "q") + +# An interpreted subclass doesn't use the native constructor of its base class. +class Sub(Open): + def __init__(self, a, b): + super().__init__(a) + self.b = b + +class SubNoInit(Open): + pass + +s = Sub(1, b=2) +assert type(s) is Sub +assert (s.a, s.b) == (1, 2) +s2 = SubNoInit(3) +assert type(s2) is SubNoInit +assert s2.a == 3 + [case testClassWithFinalAttribute] from typing import Final diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 77c89d110f5e..47f843ac9d36 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2441,6 +2441,38 @@ class CompiledBase: def value(self) -> int: raise NotImplementedError +[case testMultiModuleCallTypeObjectWithInheritedInit] +from other import Base + +class Child(Base): + pass + +[file other.py] +class Base: + def __init__(self, x: int) -> None: + self.x = x + +[file driver.py] +import other +from native import Child + +c = Child(7) +assert type(c) is Child +assert c.x == 7 +assert Child(x=8).x == 8 + +# An argument with the wrong type is reported in __init__, which is in another module. +try: + Child("7") +except TypeError as e: + assert str(e) == "int object expected; got str" + tb = e.__traceback__.tb_next + assert tb.tb_frame.f_globals is other.__dict__ + assert tb.tb_frame.f_code.co_name == "__init__" + assert tb.tb_lineno == 2 +else: + assert False + [case testFinalMethodCallsWithinAndAcrossModules] from other import FinalMethods, NativeBaseWithFinals, FinalTrait, SubclassOfNativeBase, SubclassOfTrait