This looks straightforward and is far from it. I expect tool support will improve in the future. Meanwhile, this blog post serves as a step by step explanation for what is going on in code that I'm about to push to my team.
Let's take this relatively straightforward python code. It has a function printing an int, and a decorator that makes it argument optional, taking it from a global default if missing:
from unittest import mock
default = 42
def with_default(f):
def wrapped(self, value=None):
if value is None:
value = default
return f(self, value)
return wrapped
class Fiddle:
@with_default
def print(self, value):
print("Answer:", value)
fiddle = Fiddle()
fiddle.print(12)
fiddle.print()
def mocked(self, value=None):
print("Mocked answer:", value)
with mock.patch.object(Fiddle, "print", autospec=True, side_effect=mocked):
fiddle.print(12)
fiddle.print()
It works nicely as expected:
$ python3 test0.py
Answer: 12
Answer: 42
Mocked answer: 12
Mocked answer: None
It lacks functools.wraps and typing, though. Let's add them.
Adding functools.wraps
Adding a simple @functools.wraps, mock unexpectedly stops working:
# python3 test1.py
Answer: 12
Answer: 42
Mocked answer: 12
Traceback (most recent call last):
File "/home/enrico/lavori/freexian/tt/test1.py", line 42, in <module>
fiddle.print()
File "<string>", line 2, in print
File "/usr/lib/python3.11/unittest/mock.py", line 186, in checksig
sig.bind(*args, **kwargs)
File "/usr/lib/python3.11/inspect.py", line 3211, in bind
return self._bind(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/inspect.py", line 3126, in _bind
raise TypeError(msg) from None
TypeError: missing a required argument: 'value'
This is the new code, with explanations and a fix:
# Introduce functools
import functools
from unittest import mock
default = 42
def with_default(f):
@functools.wraps(f)
def wrapped(self, value=None):
if value is None:
value = default
return f(self, value)
# Fix:
# del wrapped.__wrapped__
return wrapped
class Fiddle:
@with_default
def print(self, value):
assert value is not None
print("Answer:", value)
fiddle = Fiddle()
fiddle.print(12)
fiddle.print()
def mocked(self, value=None):
print("Mocked answer:", value)
with mock.patch.object(Fiddle, "print", autospec=True, side_effect=mocked):
fiddle.print(12)
# mock's autospec uses inspect.getsignature, which follows __wrapped__ set
# by functools.wraps, which points to a wrong signature: the idea that
# value is optional is now lost
fiddle.print()
Adding typing
For simplicity, from now on let's change Fiddle.print to match its wrapped signature:
# Give up with making value not optional, to simplify things :(
def print(self, value: int | None = None) -> None:
assert value is not None
print("Answer:", value)
Typing with ParamSpec
# Introduce typing, try with ParamSpec
import functools
from typing import TYPE_CHECKING, ParamSpec, Callable
from unittest import mock
default = 42
P = ParamSpec("P")
def with_default(f: Callable[P, None]) -> Callable[P, None]:
# Using ParamSpec we forward arguments, but we cannot use them!
@functools.wraps(f)
def wrapped(self, value: int | None = None) -> None:
if value is None:
value = default
return f(self, value)
return wrapped
class Fiddle:
@with_default
def print(self, value: int | None = None) -> None:
assert value is not None
print("Answer:", value)
mypy complains inside the wrapper, because while we forward arguments we don't
constrain them, so we can't be sure there is a value in there:
test2.py:17: error: Argument 2 has incompatible type "int"; expected "P.args" [arg-type]
test2.py:19: error: Incompatible return value type (got "_Wrapped[P, None, [Any, int | None], None]", expected "Callable[P, None]") [return-value]
test2.py:19: note: "_Wrapped[P, None, [Any, int | None], None].__call__" has type "Callable[[Arg(Any, 'self'), DefaultArg(int | None, 'value')], None]"
Typing with Callable
We can use explicit Callable argument lists:
# Introduce typing, try with Callable
import functools
from typing import TYPE_CHECKING, Callable, TypeVar
from unittest import mock
default = 42
A = TypeVar("A")
# Callable cannot represent the fact that the argument is optional, so now mypy
# complains if we try to omit it
def with_default(f: Callable[[A, int | None], None]) -> Callable[[A, int | None], None]:
@functools.wraps(f)
def wrapped(self: A, value: int | None = None) -> None:
if value is None:
value = default
return f(self, value)
return wrapped
class Fiddle:
@with_default
def print(self, value: int | None = None) -> None:
assert value is not None
print("Answer:", value)
if TYPE_CHECKING:
reveal_type(Fiddle.print)
fiddle = Fiddle()
fiddle.print(12)
# !! Too few arguments for "print" of "Fiddle" [call-arg]
fiddle.print()
def mocked(self, value=None):
print("Mocked answer:", value)
with mock.patch.object(Fiddle, "print", autospec=True, side_effect=mocked):
fiddle.print(12)
fiddle.print()
Now mypy complains when we try to omit the optional argument, because Callable cannot represent optional arguments:
test3.py:32: note: Revealed type is "def (test3.Fiddle, Union[builtins.int, None])"
test3.py:37: error: Too few arguments for "print" of "Fiddle" [call-arg]
test3.py:46: error: Too few arguments for "print" of "Fiddle" [call-arg]
typing's documentation says:
Callable cannot express complex signatures such as functions that take a variadic number of arguments, overloaded functions, or functions that have keyword-only parameters. However, these signatures can be expressed by defining a Protocol class with a call() method:
Let's do that!
Typing with Protocol, take 1
# Introduce typing, try with Protocol
import functools
from typing import TYPE_CHECKING, Protocol, TypeVar, Generic, cast
from unittest import mock
default = 42
A = TypeVar("A", contravariant=True)
class Printer(Protocol, Generic[A]):
def __call__(_, self: A, value: int | None = None) -> None:
...
def with_default(f: Printer[A]) -> Printer[A]:
@functools.wraps(f)
def wrapped(self: A, value: int | None = None) -> None:
if value is None:
value = default
return f(self, value)
return cast(Printer, wrapped)
class Fiddle:
# function has a __get__ method to generated bound versions of itself
# the Printer protocol does not define it, so mypy is now unable to type
# the bound method correctly
@with_default
def print(self, value: int | None = None) -> None:
assert value is not None
print("Answer:", value)
if TYPE_CHECKING:
reveal_type(Fiddle.print)
fiddle = Fiddle()
# !! Argument 1 to "__call__" of "Printer" has incompatible type "int"; expected "Fiddle"
fiddle.print(12)
fiddle.print()
def mocked(self, value=None):
print("Mocked answer:", value)
with mock.patch.object(Fiddle, "print", autospec=True, side_effect=mocked):
fiddle.print(12)
fiddle.print()
New mypy complaints:
test4.py:41: error: Argument 1 to "__call__" of "Printer" has incompatible type "int"; expected "Fiddle" [arg-type]
test4.py:42: error: Missing positional argument "self" in call to "__call__" of "Printer" [call-arg]
test4.py:50: error: Argument 1 to "__call__" of "Printer" has incompatible type "int"; expected "Fiddle" [arg-type]
test4.py:51: error: Missing positional argument "self" in call to "__call__" of "Printer" [call-arg]
What happens with class methods, is that the function object has a __get__
method that generates a bound versions of itself. Our Printer protocol does not
define it, so mypy is now unable to type the bound method correctly.
Typing with Protocol, take 2
So... we add the function descriptor methos to our Protocol!
A lot of this is taken from this discussion.
# Introduce typing, try with Protocol, harder!
import functools
from typing import TYPE_CHECKING, Protocol, TypeVar, Generic, cast, overload, Union
from unittest import mock
default = 42
A = TypeVar("A", contravariant=True)
# We now produce typing for the whole function descriptor protocol
#
# See https://github.com/python/typing/discussions/1040
class BoundPrinter(Protocol):
"""Protocol typing for bound printer methods."""
def __call__(_, value: int | None = None) -> None:
"""Bound signature."""
class Printer(Protocol, Generic[A]):
"""Protocol typing for printer methods."""
# noqa annotations are overrides for flake8 being confused, giving either D418:
# Function/ Method decorated with @overload shouldn't contain a docstring
# or D105:
# Missing docstring in magic method
#
# F841 is for vulture being confused:
# unused variable 'objtype' (100% confidence)
@overload
def __get__( # noqa: D105
self, obj: A, objtype: type[A] | None = None # noqa: F841
) -> BoundPrinter:
...
@overload
def __get__( # noqa: D105
self, obj: None, objtype: type[A] | None = None # noqa: F841
) -> "Printer[A]":
...
def __get__(
self, obj: A | None, objtype: type[A] | None = None # noqa: F841
) -> Union[BoundPrinter, "Printer[A]"]:
"""Implement function descriptor protocol for class methods."""
def __call__(_, self: A, value: int | None = None) -> None:
"""Unbound signature."""
def with_default(f: Printer[A]) -> Printer[A]:
@functools.wraps(f)
def wrapped(self: A, value: int | None = None) -> None:
if value is None:
value = default
return f(self, value)
return cast(Printer, wrapped)
class Fiddle:
# function has a __get__ method to generated bound versions of itself
# the Printer protocol does not define it, so mypy is now unable to type
# the bound method correctly
@with_default
def print(self, value: int | None = None) -> None:
assert value is not None
print("Answer:", value)
fiddle = Fiddle()
fiddle.print(12)
fiddle.print()
def mocked(self, value=None):
print("Mocked answer:", value)
with mock.patch.object(Fiddle, "print", autospec=True, side_effect=mocked):
fiddle.print(12)
fiddle.print()
It works! It's typed! And mypy is happy!