diff --git a/docs/source/pcapkit/corekit/fields/numbers.rst b/docs/source/pcapkit/corekit/fields/numbers.rst index e1e4cdea56..8d964701c0 100644 --- a/docs/source/pcapkit/corekit/fields/numbers.rst +++ b/docs/source/pcapkit/corekit/fields/numbers.rst @@ -65,7 +65,11 @@ Internal Definitions .. autofunction:: pcapkit.corekit.fields.numbers._rebuild_unregistered_member -.. autofunction:: pcapkit.corekit.fields.numbers._reduce_unregistered_member +.. versionremoved:: 1.6.0 + + ``pcapkit.corekit.fields.numbers._reduce_unregistered_member``, with no + alias; :func:`pcapkit.utilities.pickle.reduce_to` installs the reduction it + built (:issue:`2714`). .. autoclass:: pcapkit.corekit.fields.numbers._CapturedInt :no-members: diff --git a/docs/source/pcapkit/utilities/index.rst b/docs/source/pcapkit/utilities/index.rst index 386443f3b6..d81bfee922 100644 --- a/docs/source/pcapkit/utilities/index.rst +++ b/docs/source/pcapkit/utilities/index.rst @@ -7,9 +7,10 @@ Module Utilities :mod:`pcapkit.utilities` holds the cross-cutting services :mod:`pcapkit` is built with: user-refined exceptions and warnings, logging, the tunable limits of :mod:`~pcapkit.utilities.config`, a compatibility layer, timestamp -conversion, the :mod:`context variables ` its -modules share, and wrappers around third-party libraries. It imports nothing from -:mod:`pcapkit` outside itself; see :ref:`utilities-or-corekit`. +conversion, pickle reductions, the +:mod:`context variables ` its modules share, and +wrappers around third-party libraries. It imports nothing from :mod:`pcapkit` +outside itself; see :ref:`utilities-or-corekit`. .. toctree:: :maxdepth: 2 @@ -20,6 +21,7 @@ modules share, and wrappers around third-party libraries. It imports nothing fro config chardet datetime + pickle contextvars Version Compatibility diff --git a/docs/source/pcapkit/utilities/pickle.rst b/docs/source/pcapkit/utilities/pickle.rst new file mode 100644 index 0000000000..d693c64a12 --- /dev/null +++ b/docs/source/pcapkit/utilities/pickle.rst @@ -0,0 +1,27 @@ +Pickle Reductions +================= + +.. module:: pcapkit.utilities.pickle + +:mod:`pcapkit.utilities.pickle` holds the :mod:`pickle` reductions several +:mod:`pcapkit` classes share, so each is written once (:issue:`2714`). +:class:`ReduceAsProtocol2` lets a :class:`typing.Generic` subclass or a slotted +:class:`dict` pickle under protocol 0 and 1, on GraalPy as on CPython; +:class:`ReduceToName` reduces a singleton to the module-level name holding it; +:class:`ReduceToItems` reduces a multi-valued mapping to a call of its class on its +item pairs; and :func:`reduce_to` makes one instance reduce to a call of a given +function. A pickle a class wrote before it took one of these still loads, and one +it writes with it loads where the class still reduces by its own. + +.. autoclass:: pcapkit.utilities.pickle.ReduceAsProtocol2 + :members: __reduce_ex__ + +.. autoclass:: pcapkit.utilities.pickle.ReduceToName + :members: __reduce__ + +.. autoclass:: pcapkit.utilities.pickle.ReduceToItems + :members: __reduce_ex__ + +.. autofunction:: pcapkit.utilities.pickle.reduce_to + +.. autofunction:: pcapkit.utilities.pickle._reduce_to diff --git a/pcapkit/corekit/enum.py b/pcapkit/corekit/enum.py index e07a0c0067..dc1c47ac76 100644 --- a/pcapkit/corekit/enum.py +++ b/pcapkit/corekit/enum.py @@ -96,16 +96,16 @@ an unknown value does not mint one (:issue:`775`). """ -import functools from typing import TYPE_CHECKING from aenum import extend_enum from pcapkit.corekit.sentinels import NO_DEFAULT from pcapkit.utilities.exceptions import BaseError, CaughtByCaller, EnumKeyError, EnumValueError +from pcapkit.utilities.pickle import reduce_to if TYPE_CHECKING: - from typing import Any, Callable, Type + from typing import Any, Type from typing_extensions import Self @@ -730,11 +730,10 @@ def _make_reducible(cls, obj: 'Any', *args: 'Any') -> 'Any': lookup table grows. The rebuilt member is itself reducible, since the method it comes back through installs this again. - The override goes on the *instance* rather than on the type, which both - :mod:`pickle` and :mod:`copy` honour: each fetches ``__reduce_ex__`` - with :func:`getattr` on the object. :func:`functools.partial` rather - than a closure keeps the member itself out of its own reducer, so being - reducible does not put it in a reference cycle. + The override goes on the *instance* rather than on the type, through + :func:`~pcapkit.utilities.pickle.reduce_to`, which :mod:`pickle` and + :mod:`copy` both honour, and which keeps the member out of its own + reducer. Note: This is opt-in per registry rather than done by @@ -757,8 +756,7 @@ def _make_reducible(cls, obj: 'Any', *args: 'Any') -> 'Any': ``obj``, for use as a ``return`` expression. """ - obj.__reduce_ex__ = functools.partial(_reduce_unregistered_member, cls, args) - return obj + return reduce_to(obj, _rebuild_unregistered_member, cls, args) def _rebuild_unregistered_member(namespace: 'Type[EnumRegistry]', @@ -783,25 +781,3 @@ def _rebuild_unregistered_member(namespace: 'Type[EnumRegistry]', return namespace._unregistered_member(*args) # pylint: disable=protected-access -def _reduce_unregistered_member( # pylint: disable=unused-argument - namespace: 'Type[EnumRegistry]', args: 'tuple[Any, ...]', - protocol: 'int') -> 'tuple[Callable[..., Any], tuple[Any, ...]]': - """The ``__reduce_ex__`` :meth:`EnumRegistry._make_reducible` installs. - - Bound to its first two arguments with :func:`functools.partial`, so that - the reducer holds the ingredients of the member rather than the member - itself. - - Args: - namespace: The registry class the member is an instance of. - args: The arguments to rebuild the member from. - protocol: The :mod:`pickle` protocol version, as passed by - :mod:`pickle` and :mod:`copy`. Unused -- the reduction is the same - for every protocol -- but accepted because both call this with it. - - Returns: - The callable and arguments :mod:`pickle` and :mod:`copy` should rebuild - the member through. - - """ - return (_rebuild_unregistered_member, (namespace, args)) diff --git a/pcapkit/corekit/fields/numbers.py b/pcapkit/corekit/fields/numbers.py index 643c34e7ef..70ae767304 100644 --- a/pcapkit/corekit/fields/numbers.py +++ b/pcapkit/corekit/fields/numbers.py @@ -1,8 +1,15 @@ # -*- coding: utf-8 -*- -"""numerical field class""" +"""numerical field class + +.. versionremoved:: 1.6.0 + + ``pcapkit.corekit.fields.numbers._reduce_unregistered_member``, with no + alias; :func:`pcapkit.utilities.pickle.reduce_to` installs the reduction it + built (:issue:`2714`). + +""" import enum -import functools import math from typing import TYPE_CHECKING, Generic, TypeVar, Union, cast @@ -11,6 +18,7 @@ from pcapkit.corekit.fields.field import Field from pcapkit.corekit.sentinels import NO_VALUE from pcapkit.utilities.exceptions import BaseError, FieldValueError, IntError, ProtocolError +from pcapkit.utilities.pickle import reduce_to __all__ = [ 'NumberField', @@ -958,14 +966,9 @@ def _unregistered_member(namespace: 'Type[StdlibEnum] | Type[AenumEnum]', # NOTE: ``Enum.__reduce_ex__`` reduces a member to ``(cls, (value,))``, # i.e. to the one lookup this member is deliberately absent from, so # pickle and copy would both raise on it without this. Overriding it - # per instance -- which pickle and copy both honour, since both fetch - # it with getattr on the object rather than on its type -- rebuilds an - # equivalent unregistered member instead. functools.partial rather - # than a closure keeps this off obj itself, so the member does not - # become part of a reference cycle merely by being reducible. - obj.__reduce_ex__ = functools.partial( - _reduce_unregistered_member, namespace, value, name, attrs) - return obj + # per instance, which pickle and copy both honour, rebuilds an + # equivalent unregistered member instead. + return reduce_to(obj, _rebuild_unregistered_member, namespace, value, name, attrs) def _rebuild_unregistered_member(namespace: 'Type[StdlibEnum] | Type[AenumEnum]', @@ -996,31 +999,6 @@ def _rebuild_unregistered_member(namespace: 'Type[StdlibEnum] | Type[AenumEnum]' namespace, value, name, **attrs) -def _reduce_unregistered_member( # pylint: disable=unused-argument - namespace: 'Type[StdlibEnum] | Type[AenumEnum]', - value: 'Any', name: 'str', attrs: 'dict[str, Any]', - protocol: 'int') -> 'tuple[Callable[..., Any], tuple[Any, ...]]': - """The ``__reduce_ex__`` :meth:`EnumField._unregistered_member` installs. - - Bound to its first four arguments with :func:`functools.partial`, so that - the reducer holds the ingredients of the member rather than the member - itself. - - Args: - namespace: The registry class the member is an instance of. - value: The member's ``_value_``. - name: The member's ``_name_``. - attrs: The extra attributes the member carries. - protocol: The :mod:`pickle` protocol version, ignored -- the reduction - is the same for all of them, and :mod:`copy` passes 4 here. - - Returns: - A two-tuple of :func:`_rebuild_unregistered_member` and its arguments. - - """ - return (_rebuild_unregistered_member, (namespace, value, name, attrs)) - - class PortEnumField(EnumField): """Enumerated value for transport-layer port fields. diff --git a/pcapkit/corekit/infoclass.py b/pcapkit/corekit/infoclass.py index 7cbb8bac93..c480c0c290 100644 --- a/pcapkit/corekit/infoclass.py +++ b/pcapkit/corekit/infoclass.py @@ -27,6 +27,7 @@ from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict, iter_multi_items from pcapkit.utilities.compat import Mapping, UnionType, final, get_annotations from pcapkit.utilities.exceptions import InfoError, UnsupportedCall, stacklevel +from pcapkit.utilities.pickle import ReduceAsProtocol2, ReduceToItems from pcapkit.utilities.warnings import InfoWarning, warn if TYPE_CHECKING: @@ -318,7 +319,7 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' return super().__new__(cls, name, bases, attrs, **kwargs) # type: ignore[return-value] -class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta): +class Info(ReduceAsProtocol2, Mapping[str, VT], Generic[VT], metaclass=InfoMeta): """Turn dictionaries into :obj:`object` like instances. * :class:`Info` objects are :class:`~collections.abc.Mapping` instances, @@ -975,7 +976,7 @@ def _immutable(self, *args: 'Any', **kwargs: 'Any') -> 'NoReturn': __setstate__ = _immutable -class MultiInfo(_MultiInfo[KT, VT], MultiDict[KT, VT]): +class MultiInfo(ReduceToItems, _MultiInfo[KT, VT], MultiDict[KT, VT]): """An immutable :class:`~pcapkit.corekit.multidict.MultiDict` that is an :class:`Info`. It holds every value of a key, as a @@ -1065,9 +1066,6 @@ def listvalues(self) -> 'Iterator[list[VT]]': """ return (values for (_, values) in self.lists()) - def __reduce_ex__(self, protocol: 'Any') -> 'tuple[type, tuple[list[tuple[KT, VT]]]]': - return type(self), (list(self.items(multi=True)),) - class OrderedMultiInfo(_MultiInfo[KT, VT], OrderedMultiDict[KT, VT]): """An immutable :class:`~pcapkit.corekit.multidict.OrderedMultiDict` that is an :class:`Info`. diff --git a/pcapkit/corekit/multidict.py b/pcapkit/corekit/multidict.py index 8269043933..e69dafbb82 100644 --- a/pcapkit/corekit/multidict.py +++ b/pcapkit/corekit/multidict.py @@ -15,9 +15,10 @@ from typing import TYPE_CHECKING, Generic, TypeVar, overload from pcapkit.utilities.exceptions import MissingKeyError, UnsupportedCall +from pcapkit.utilities.pickle import ReduceToItems, ReduceToName if TYPE_CHECKING: - from typing import Any, Iterable, Iterator, Mapping, NoReturn, Optional, SupportsIndex + from typing import Any, Iterable, Iterator, Mapping, NoReturn, Optional from typing_extensions import Literal, Self @@ -68,13 +69,12 @@ def unlink(self, omd: 'OrderedMultiDict[_KT, _VT]') -> 'None': omd._last_bucket = self.prev # pylint: disable=protected-access -class _Missing: +class _Missing(ReduceToName): + _name = '_missing' + def __repr__(self) -> 'str': return "no value" - def __reduce__(self) -> 'str': - return "_missing" - _missing = _Missing() @@ -470,7 +470,7 @@ def __repr__(self) -> 'str': return f"{type(self).__name__}({list(self.items(multi=True))!r})" -class OrderedMultiDict(MultiDict[_KT, _VT]): +class OrderedMultiDict(ReduceToItems, MultiDict[_KT, _VT]): """Works like a regular :class:`MultiDict` but preserves the order of the fields. To convert the ordered multi dict into a list you can use the :meth:`items` method and pass it ``multi=True``. @@ -537,9 +537,6 @@ def deepcopy(self, memo: 'Optional[dict]' = None) -> 'OrderedMultiDict[_KT, _VT] """ return self.__class__(copy.deepcopy(list(self.items(multi=True)), memo=memo)) - def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': - return type(self), (list(self.items(multi=True)),) - def __getstate__(self) -> 'Any': return list(self.items(multi=True)) diff --git a/pcapkit/corekit/sentinels.py b/pcapkit/corekit/sentinels.py index aa84df2e9d..2c9f7371ab 100644 --- a/pcapkit/corekit/sentinels.py +++ b/pcapkit/corekit/sentinels.py @@ -45,6 +45,7 @@ from typing import TYPE_CHECKING, cast from pcapkit.utilities.compat import final +from pcapkit.utilities.pickle import ReduceToName __all__ = ['Sentinel', 'NULL', 'NO_VALUE', 'NO_DEFAULT'] @@ -197,7 +198,7 @@ def register(cls, subclass: 'type[Any]') -> 'type[Any]': return super().register(subclass) -class Sentinel(metaclass=SentinelMeta): +class Sentinel(ReduceToName, metaclass=SentinelMeta): """Abstract base of every sentinel type, so each behaves like the type of :obj:`None`. A subclass sets ``_name`` to the name of the module-level variable that @@ -210,10 +211,11 @@ class Sentinel(metaclass=SentinelMeta): :obj:`None`, so a caller who builds one by mistake still gets the sentinel. * :meth:`__copy__` and :meth:`__deepcopy__` return ``self``. - * :meth:`__reduce__` returns ``_name``, as ``Ellipsis.__reduce__()`` - returns ``'Ellipsis'``, so :mod:`pickle` stores the instance as a - reference to that module-level name and loads it back as the same - object, at every protocol. Guarding :meth:`__new__` alone would not + * :meth:`~pcapkit.utilities.pickle.ReduceToName.__reduce__`, from + :class:`~pcapkit.utilities.pickle.ReduceToName`, returns ``_name``, as + ``Ellipsis.__reduce__()`` returns ``'Ellipsis'``, so :mod:`pickle` stores + the instance as a reference to that module-level name and loads it back + as the same object, at every protocol. Guarding :meth:`__new__` alone would not cover protocols 0 and 1, which rebuild through :func:`copyreg._reconstructor` and so call :meth:`object.__new__` directly. @@ -318,10 +320,6 @@ def __deepcopy__(self, memo: 'dict[int, Any]') -> 'Self': """ return self - def __reduce__(self) -> 'str': - """Reduce to ``_name``, which :mod:`pickle` stores as a global reference.""" - return self._name - @final class NullType(Sentinel): diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index 3fc2eaf840..7a65ae74f6 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- """shared data models for reassembly""" -import copyreg from typing import TYPE_CHECKING, Generic, TypeVar from pcapkit.corekit.enum import EnumLookup from pcapkit.corekit.infoclass import Info, info_final from pcapkit.utilities.compat import StrEnum, auto +from pcapkit.utilities.pickle import ReduceAsProtocol2 __all__ = ['ReassemblyData', 'Completion', 'Deferred'] if TYPE_CHECKING: - from typing import Any, Callable, Optional, SupportsIndex + from typing import Any, Callable, Optional from pcapkit.const.reg.apptype import TCP as AppType_TCP from pcapkit.const.reg.transtype import TransType @@ -120,7 +120,7 @@ def __bool__(self) -> 'bool': return self is Completion.COMPLETE -class Deferred(Generic[_KT]): +class Deferred(ReduceAsProtocol2, Generic[_KT]): """A postponed analysis of a reassembled payload. A reassembled datagram's ``packet`` is a second, full parse of the payload @@ -232,7 +232,10 @@ class Deferred(Generic[_KT]): (:issue:`2531`). IP's and TCP's analyser is the parse their :mod:`pcapkit.foundation.analysis` module runs, bound to the reassembly's protocol, rather than that protocol's bound ``analyze`` - (:issue:`2581`). + (:issue:`2581`). Protocol 0 and 1 reduce as protocol 2 does, through + :class:`~pcapkit.utilities.pickle.ReduceAsProtocol2`, which carries the + slots in Python's own default state, in place of a reduction and a + ``__getstate__`` of its own (:issue:`2714`). """ @@ -259,122 +262,19 @@ def __call__(self) -> 'ProtocolBase | DTLSHandshake': """ return self.analyze(self.proto, self.payload) - def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': - """Pickling support for protocol 0 and 1. - - :meth:`object.__reduce_ex__` hands protocol 0 and 1 off to - :func:`copyreg._reduce_ex`, which -- *before* it ever consults - :meth:`__getstate__` -- walks ``type(self).__mro__`` looking for the - first base that is either not a heap type or whose own ``__new__`` is - bound to itself, to compute a base-object state via ``base(self)``. On - CPython, :class:`~typing.Generic` is a heap type and its ``__new__`` - is just the inherited ``object.__new__`` (bound to ``object``, not to - ``Generic``), so neither check fires there and the walk continues to - :class:`object` itself, which *is* a non-heap type -- the walk's first - check breaks on it, so ``base`` is ``object`` and the computed state is - :data:`None`. On GraalPy 25.4, ``Generic`` is itself a non-heap - (built-in) type there -- ``Generic.__flags__`` lacks ``_HEAPTYPE`` -- - so the walk's *first* check breaks on ``Generic`` directly, one step - earlier than on CPython, without ever reaching the ``__new__`` check - at all. ``base`` therefore resolves to ``Generic`` itself, and the - walk calls ``Generic(self)``, which raises:: - - TypeError: typing.Generic() takes no arguments - - :meth:`__getstate__`/:meth:`__setstate__` never run in that failure: - they only ever supply the *dict* half of the reduction tuple the walk - above is computing the *base-object* half for, so defining them - (:issue:`2423`, :issue:`2425`) does nothing to protect against a - GraalPy-specific outcome of a walk that happens before either is - consulted. - - This override sidesteps the walk entirely rather than trying to steer - it: for protocol 0 and 1 it hard-codes the one outcome the walk was - ever going to reach for this class -- ``base=object``, - ``state=None`` -- and builds the same ``(callable, args, state)`` - reduction :func:`copyreg._reduce_ex` itself would have, with - :meth:`__getstate__`'s own ``(dict_or_None, slots)`` pair as the - third element, restored through :meth:`__setstate__` exactly as - before. ``Deferred`` has no base below :class:`object` that could - need special handling -- :class:`~typing.Generic` carries no runtime - state of its own -- so hard-coding that outcome changes nothing about - *what* gets pickled, only *how* protocol 0 and 1 get there: the MRO - walk, and ``Generic`` along with it, is no longer part of the route at - all. Protocol 2 and above are untouched, deferring to - :meth:`object.__reduce_ex__` exactly as before this method existed -- - their own ``__reduce_ex__`` never walked the MRO this way to begin - with, building a reduction straight from ``__getstate__`` instead. - - A subclass that overrides ``__reduce__`` is checked for first, and - deferred to, before any of the above: :meth:`object.__reduce_ex__` - itself does this -- for *every* protocol, including 0 and 1, it - calls a non-default ``__reduce__`` in preference to building a - reduction of its own -- so hard-coding the protocol 0/1 tuple here - without the same check would silently drop such a subclass's - override at exactly those two protocols, while protocol 2 and above - (going through ``object.__reduce_ex__`` unchanged) kept honouring - it. - - """ - if type(self).__reduce__ is not object.__reduce__: - return self.__reduce__() - if int(protocol) >= 2: - return object.__reduce_ex__(self, protocol) - return (copyreg._reconstructor, (type(self), object, None), # type: ignore[attr-defined] # pylint: disable=protected-access - self.__getstate__()) - - def __getstate__(self) -> 'tuple[Optional[dict[str, Any]], dict[str, Any]]': - """Pickling support. - - A slotted class with no ``__dict__`` has nothing for the default - :mod:`pickle` reduction to read. :mod:`copyreg` refuses outright with - *"cannot be pickled"* under protocol 0 or 1 whenever - ``type(self).__getstate__`` is still the inherited default -- which is - why defining *any* ``__getstate__`` here, even one that only calls the - default, clears protocols 0 and 1 (:issue:`2423`). Protocol 2 and above - never needed one: their own ``__reduce_ex__`` already walks - ``__slots__`` directly. - - The shape returned -- an instance-``__dict__`` state (or :data:`None`) - paired with a slots-state :obj:`dict` -- is deliberately the same one - Python's own default state construction produces for a slotted class, - rather than a bespoke flat :obj:`dict`. A flat shape would be a *new* - pickle format for this class: a ``Deferred`` already pickled by a - released version under protocol 2 or above -- every one of them built - this same ``(dict_or_None, slots)`` pair -- would fail to unpickle - against a flat-shaped :meth:`__setstate__`, and the reverse. - - The slot names come from :func:`copyreg._slotnames`, the standard - library's own helper for this exact job, rather than a hand-rolled - walk of ``type(self).__mro__`` and each class's ``__slots__``: a - subclass of ``Deferred`` may spell a slot as a bare string - (``__slots__ = 'extra'``, valid Python for one slot named by the - whole string) or as a private, name-mangled one - (``__slots__ = ('__secret',)``, stored on the instance under - ``___secret``), and a hand-rolled walk that assumes every - ``__slots__`` is an iterable of already-resolved attribute names gets - both wrong -- the first by iterating the string's *characters*, the - second by testing the unmangled name ``hasattr`` never finds - (:issue:`2425`). ``_slotnames`` already handles both, the same way - protocol 0 and 1 reduction has relied on it to for decades. - - """ - slotnames = copyreg._slotnames(type(self)) # type: ignore[attr-defined] # pylint: disable=protected-access - slots = {name: getattr(self, name) for name in slotnames if hasattr(self, name)} - dict_state = dict(self.__dict__) if getattr(self, '__dict__', None) else None - return (dict_state, slots) - def __setstate__( self, state: 'tuple[Optional[dict[str, Any]], dict[str, Any]] | dict[str, Any]', ) -> 'None': - """Unpickling support; the counterpart to :meth:`__getstate__`. - - Accepts the ``(dict_or_None, slots)`` pair :meth:`__getstate__` builds - above -- the shape every released, pre-:issue:`2423` pickle already - used under protocol 2 and above -- and, for a pickle made by this - class's own first, unreleased fix for :issue:`2423`, a bare :obj:`dict` - of ``{slot: value}`` (:issue:`2425`). + """Unpickling support. + + Accepts the ``(dict_or_None, slots)`` pair Python's own default state + construction builds for a slotted class, which every pickle of this + class carries under protocol 2 and above, and which + :class:`~pcapkit.utilities.pickle.ReduceAsProtocol2` carries under + protocol 0 and 1 too. It also accepts a bare :obj:`dict` of + ``{slot: value}``, the state a first, unreleased fix for :issue:`2423` + pickled (:issue:`2425`). """ if isinstance(state, dict): diff --git a/pcapkit/foundation/reassembly/reassembly.py b/pcapkit/foundation/reassembly/reassembly.py index 3a89563849..68557bffa1 100644 --- a/pcapkit/foundation/reassembly/reassembly.py +++ b/pcapkit/foundation/reassembly/reassembly.py @@ -19,6 +19,7 @@ from pcapkit.protocols.misc.raw import Raw from pcapkit.utilities.exceptions import FieldValueError, UnsupportedCall from pcapkit.utilities.logging import get_logger +from pcapkit.utilities.pickle import ReduceAsProtocol2 # NB: declared above the ``TYPE_CHECKING`` block, not below it, so that # ``CallbackFn`` can name ``_DT`` outright. As a quoted forward reference it would @@ -106,7 +107,7 @@ def registry(cls) -> 'dict[str, ModuleDescriptor[ReassemblyBase] | Type[Reassemb return Extractor.__reassembly__ -class ReassemblyBase(Generic[_PT, _DT, _IT, _BT], metaclass=ReassemblyMeta): +class ReassemblyBase(ReduceAsProtocol2, Generic[_PT, _DT, _IT, _BT], metaclass=ReassemblyMeta): """Base class for reassembly procedure. Args: diff --git a/pcapkit/foundation/traceflow/data/data.py b/pcapkit/foundation/traceflow/data/data.py index 2549500cbf..29e21463e0 100644 --- a/pcapkit/foundation/traceflow/data/data.py +++ b/pcapkit/foundation/traceflow/data/data.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING from pcapkit.corekit.infoclass import Info, info_final +from pcapkit.utilities.pickle import ReduceAsProtocol2 __all__ = ['TraceFlowData', 'Deferred', 'FrameRecord'] @@ -66,7 +67,7 @@ def __call__(self) -> 'tuple[TCP_Datagram, ...]': return self.reassembly.datagram -class FrameRecord(dict): +class FrameRecord(ReduceAsProtocol2, dict): """A frame as a third-party engine's flow tracing adapter reports it. The mapping is the engine's own dissection of the frame, which every trace @@ -105,15 +106,14 @@ def __init__(self, mapping: 'dict[str, Any]', *, packet: 'bytes', self.frame_info = frame_info self.time_epoch = time_epoch - def __reduce__(self) -> 'tuple[Any, ...]': - # NOTE: spelled out because a ``dict`` subclass with ``__slots__`` cannot - # be pickled with protocol 0 or 1 otherwise - return _frame_record, (dict(self), self.packet, self.frame_info, self.time_epoch) - def _frame_record(mapping: 'dict[str, Any]', packet: 'bytes', frame_info: 'Data_FrameInfo', time_epoch: 'Decimal') -> 'FrameRecord': - """Rebuild a pickled :class:`FrameRecord`. + """Rebuild a :class:`FrameRecord` pickled by a release before 1.6.0. + + Those pickles call this; a :class:`FrameRecord` reduces through + :class:`~pcapkit.utilities.pickle.ReduceAsProtocol2` instead, which carries + the slots under every protocol (:issue:`2714`). Args: mapping: The engine's dissection of the frame. diff --git a/pcapkit/protocols/schema/schema.py b/pcapkit/protocols/schema/schema.py index 138af71cf8..9a5ea5a7da 100644 --- a/pcapkit/protocols/schema/schema.py +++ b/pcapkit/protocols/schema/schema.py @@ -28,6 +28,7 @@ from pcapkit.corekit.sentinels import NO_VALUE from pcapkit.utilities.compat import Mapping, final from pcapkit.utilities.exceptions import NoDefaultValue, ProtocolUnbound, SchemaError, stacklevel +from pcapkit.utilities.pickle import ReduceAsProtocol2 from pcapkit.utilities.warnings import RegistryWarning, SchemaWarning, UnknownFieldWarning, warn if TYPE_CHECKING: @@ -374,7 +375,7 @@ def __new__(cls, name: 'str', bases: 'tuple[type, ...]', attrs: 'dict[str, Any]' return super().__new__(cls, name, bases, attrs, **kwargs) # type: ignore[return-value] -class Schema(Mapping[str, _VT], Generic[_VT], metaclass=SchemaMeta): +class Schema(ReduceAsProtocol2, Mapping[str, _VT], Generic[_VT], metaclass=SchemaMeta): """Schema for protocol headers. Note: diff --git a/pcapkit/utilities/__init__.py b/pcapkit/utilities/__init__.py index 1416d41558..ef671ae5bf 100644 --- a/pcapkit/utilities/__init__.py +++ b/pcapkit/utilities/__init__.py @@ -9,8 +9,8 @@ :mod:`pcapkit` is built with: user-refined exceptions and warnings, logging, the tunable limits of :mod:`~pcapkit.utilities.config`, a compatibility layer, -timestamp conversion, the context variables its modules -share, and wrappers around third-party +timestamp conversion, pickle reductions, the context +variables its modules share, and wrappers around third-party libraries. It imports nothing from :mod:`pcapkit` outside itself (:issue:`1712`). diff --git a/pcapkit/utilities/pickle.py b/pcapkit/utilities/pickle.py new file mode 100644 index 0000000000..07d0467735 --- /dev/null +++ b/pcapkit/utilities/pickle.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +"""Pickle Reductions +======================= + +.. module:: pcapkit.utilities.pickle + +:mod:`pcapkit.utilities.pickle` holds the :mod:`pickle` reductions several +:mod:`pcapkit` classes share, so each is written once (:issue:`2714`): + +* :class:`ReduceAsProtocol2` reduces under every protocol as protocol 2 does, so a + :class:`typing.Generic` subclass or a slotted :class:`dict` pickles under + protocol 0 and 1, on GraalPy as on CPython; +* :class:`ReduceToName` reduces a singleton to the module-level name holding it; +* :class:`ReduceToItems` reduces a multi-valued mapping to a call of its class on + its item pairs; +* :func:`reduce_to` makes one instance reduce to a call of a given function. + +A pickle a class wrote before it took one of these still loads, and one it writes +with it loads where the class still reduces by its own. + +""" + +import functools +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable, SupportsIndex, TypeVar + + _T = TypeVar('_T') + +__all__ = ['ReduceAsProtocol2', 'ReduceToName', 'ReduceToItems', 'reduce_to'] + + +class ReduceAsProtocol2: + """Mixin reducing an instance under every :mod:`pickle` protocol as protocol 2 does. + + Below protocol 2, :meth:`object.__reduce_ex__` hands the instance to + :func:`copyreg._reduce_ex`, which walks ``type(self).__mro__`` for the first + base that is not a heap type and, unless that base is :class:`object`, calls it + on the instance. :class:`~typing.Generic` is a heap type on CPython, so the walk + passes it. GraalPy builds ``Generic`` as a built-in, so the walk stops there and + raises ``TypeError: typing.Generic() takes no arguments`` for any generic class + with no reduction of its own (:issue:`2714`). The same walk refuses a class + with ``__slots__`` and no ``__getstate__`` of its own on every interpreter. + + Protocol 2's reduction reads the class and the instance state without that + walk, slots included, and protocol 0 and 1 write it as an ordinary call of + :func:`copyreg.__newobj__`, restoring the instance through ``__new__`` as + protocol 2 does. The next ``__reduce_ex__`` in the MRO builds it, so a base + with a reduction of its own keeps it, as + :class:`~pcapkit.corekit.infoclass.OrderedMultiInfo` keeps + :class:`~pcapkit.corekit.multidict.OrderedMultiDict`'s, and a subclass + overriding ``__reduce__`` keeps its own. + + It goes before :class:`~typing.Generic` among the bases, as on + :class:`~pcapkit.corekit.infoclass.Info`, + :class:`~pcapkit.protocols.schema.schema.Schema`, + :class:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase` and + :class:`~pcapkit.foundation.reassembly.data.data.Deferred`, and before + :class:`dict` on :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`. + + .. versionadded:: 1.6.0 + + """ + + __slots__ = () + + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'str | tuple[Any, ...]': + """Reduce as protocol 2 does, or as ``protocol`` does if higher. + + Args: + protocol: Pickle protocol in use. + + Returns: + The reduction for ``protocol``, or for protocol 2 below it. + + """ + return super().__reduce_ex__(max(int(protocol), 2)) + + +class ReduceToName: + """Mixin reducing a singleton to the module-level name that holds it. + + :meth:`__reduce__` returns ``_name``, as ``Ellipsis.__reduce__()`` returns + ``'Ellipsis'``, so :mod:`pickle` stores the instance as a reference to that + name in the class's module and loads it back as the same object, at every + protocol. A class using it sets ``_name`` to that name. + + It is shared by every :class:`~pcapkit.corekit.sentinels.Sentinel` and by the + missing-value marker of :mod:`pcapkit.corekit.multidict`. + + .. versionadded:: 1.6.0 + + """ + + __slots__ = () + + def __reduce__(self) -> 'str': + """Reduce to ``_name``, which :mod:`pickle` stores as a global reference.""" + return self._name # type: ignore[attr-defined] # pylint: disable=no-member + + +class ReduceToItems: + """Mixin reducing a multi-valued mapping to a call of its class on its item pairs. + + :meth:`__reduce_ex__` returns ``(type(self), (pairs,))``, the pairs being + ``list(self.items(multi=True))``, under every protocol. Rebuilding through the + class keeps the order of interleaved keys and builds whatever the class keeps + beside the pairs, and needs no state to be restored afterwards, which an + immutable class refuses. + + It is shared by :class:`~pcapkit.corekit.multidict.OrderedMultiDict` and + :class:`~pcapkit.corekit.infoclass.MultiInfo`. + + .. versionadded:: 1.6.0 + + """ + + __slots__ = () + + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'str | tuple[Any, ...]': + """Reduce to a call of the class on the item pairs, under every protocol. + + Args: + protocol: Pickle protocol in use, which does not change the reduction. + + Returns: + ``(type(self), (list(self.items(multi=True)),))``. + + """ + return type(self), (list(self.items(multi=True)),) # type: ignore[attr-defined] # pylint: disable=no-member + + +def reduce_to(obj: '_T', rebuild: 'Callable[..., Any]', *args: 'Any') -> '_T': + """Make ``obj`` reduce to ``rebuild(*args)`` under :mod:`pickle` and :mod:`copy`. + + The reduction goes on the *instance*, as ``obj.__reduce_ex__``, which both + :mod:`pickle` and :mod:`copy` honour, since each fetches it with + :func:`getattr` on the object. That suits an object whose type reduces it + wrongly, as ``Enum.__reduce_ex__`` reduces an unregistered member to the one + value lookup it is absent from. :func:`functools.partial` rather than a + closure keeps ``obj`` out of its own reducer, so being reducible does not put + it in a reference cycle. + + It is shared by :meth:`EnumRegistry._make_reducible + ` and + :meth:`EnumField._unregistered_member + `. + + Args: + obj: The object to make reducible. + rebuild: A module-level callable, which every :mod:`pickle` protocol can + name, rebuilding an equivalent object from ``args``. + *args: The arguments to ``rebuild``. + + Returns: + ``obj``, for use as a ``return`` expression. + + .. versionadded:: 1.6.0 + + """ + obj.__reduce_ex__ = functools.partial(_reduce_to, rebuild, args) # type: ignore[method-assign] + return obj + + +def _reduce_to( # pylint: disable=unused-argument + rebuild: 'Callable[..., Any]', args: 'tuple[Any, ...]', + protocol: 'int') -> 'tuple[Callable[..., Any], tuple[Any, ...]]': + """The ``__reduce_ex__`` :func:`reduce_to` installs, bound to its first two arguments. + + Args: + rebuild: The callable to rebuild through. + args: Its arguments. + protocol: Pickle protocol in use, ignored; :mod:`copy` passes 4. + + Returns: + ``(rebuild, args)``. + + """ + return rebuild, args diff --git a/tests/corekit/test_sentinel_singleton_1842_unit.py b/tests/corekit/test_sentinel_singleton_1842_unit.py index 2caba33848..b3396d91f3 100644 --- a/tests/corekit/test_sentinel_singleton_1842_unit.py +++ b/tests/corekit/test_sentinel_singleton_1842_unit.py @@ -63,11 +63,6 @@ #: Python's own singletons, which a sentinel is to behave like. BUILTINS = (None, Ellipsis, NotImplemented) -#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. -#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, -#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). -LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 - #: Pickles written at ``75a9f237b``, keyed by ``(object name, protocol)``. ``NULL`` #: reduced to the module-level ``_get_null``; the others took the default reduce, #: which is ``copyreg._reconstructor`` below protocol 2 and ``NEWOBJ`` from 2 on. @@ -457,7 +452,7 @@ def test_a_parsed_mpl_seed_keeps_its_identity(self) -> 'None': bad = [] if copy.deepcopy(schema).options[0].seed is not no_value: bad.append('deepcopy') - for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): if pickle.loads(pickle.dumps(schema, protocol=protocol)).options[0].seed is not no_value: bad.append(protocol) self.assertEqual(bad, []) diff --git a/tests/foundation/reassembly/test_datagram_deferred_pickle_2417_unit.py b/tests/foundation/reassembly/test_datagram_deferred_pickle_2417_unit.py index 0495e4b67f..ceb517130f 100644 --- a/tests/foundation/reassembly/test_datagram_deferred_pickle_2417_unit.py +++ b/tests/foundation/reassembly/test_datagram_deferred_pickle_2417_unit.py @@ -44,21 +44,14 @@ #: Test tier (#1958): exercises pickling, whose protocols differ by interpreter. __tier__ = 'ALL' -#: Pickle protocols a whole :class:`Deferred` can go through. +#: Every pickle protocol a whole datagram goes through, ``0`` and ``1`` included. #: -#: Protocols ``0`` and ``1`` are excluded, and not because of anything -#: :issue:`2417` changed: :class:`~pcapkit.foundation.reassembly.data.data.Deferred` -#: declares ``__slots__`` and no ``__getstate__``, which those two protocols -#: cannot serialise at all -- ``TypeError: a class that defines __slots__ -#: without defining __getstate__ cannot be pickled``, raised from -#: :mod:`copyreg`. Measured on CPython 3.14.10 to be independent of the ports: -#: it reproduces for an assigned flow, for an unassigned one, and for a bare -#: ``Deferred(..., 42, b'x')`` holding a plain :obj:`int`, while ``2``-``5`` -#: pass in every one of those cases. That is a separate limitation of the class, -#: not of the members it holds, which is why it is scoped out here rather than -#: worked around; the member-level round trip is pinned on *every* protocol from -#: ``0`` up in :mod:`tests.const.test_const_apptype_unregistered_pickle_2417_unit`. -PROTOCOLS = tuple(range(2, pickle.HIGHEST_PROTOCOL + 1)) +#: Below protocol 2, ``copyreg._reduce_ex`` refuses a slotted class with no +#: ``__getstate__`` (#2423), and its MRO walk is one GraalPy cannot pass (#2433). +#: The slotted :class:`~pcapkit.foundation.reassembly.data.data.Deferred` and the +#: :class:`~pcapkit.corekit.infoclass.Info` datagram around it both reduce as +#: protocol 2 does, which reaches neither (#2714). +PROTOCOLS = tuple(range(pickle.HIGHEST_PROTOCOL + 1)) #: A flow on two unassigned ports, which is what used to fail. UNASSIGNED_BUFID = (ip_address('192.0.2.1'), 50000, ip_address('198.51.100.2'), 65535) diff --git a/tests/foundation/reassembly/test_datagram_pickle_graalpy_2714_unit.py b/tests/foundation/reassembly/test_datagram_pickle_graalpy_2714_unit.py new file mode 100644 index 0000000000..5914806d0d --- /dev/null +++ b/tests/foundation/reassembly/test_datagram_pickle_graalpy_2714_unit.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +"""A reassembled datagram pickles under protocol 0 and 1 on GraalPy (#2714). + +Below protocol 2, :meth:`object.__reduce_ex__` hands an instance to +``copyreg._reduce_ex``, which walks the class's MRO for the first base that is not a +heap type and calls that base on the instance unless it is :class:`object`. Every +:class:`~pcapkit.corekit.infoclass.Info` derives from :class:`typing.Generic`, a heap +type on CPython, so the walk passes it there. GraalPy builds ``Generic`` as a +built-in, so its walk stops at it and raises ``TypeError: typing.Generic() takes no +arguments``: for a reassembled datagram, the ``DatagramID`` it holds and the +``ReassemblyData`` holding it alike. #2433 routes ``Deferred`` round the walk; +``ReduceAsProtocol2`` routes every ``Info`` round it, by reducing as protocol 2 does. + +Pickling does not resolve the deferred ``packet``. A failing subtest's report renders +the datagram with :func:`repr`, which does, so a test pickling one datagram under each +protocol in turn finds it resolved once protocol 0 fails. + +GraalPy is not installed here, so the stand-in of +:mod:`tests.utilities.test_pickle_graalpy_2714_unit` replaces ``copyreg._reduce_ex`` +with GraalPy's walk, treating ``Generic`` as a built-in. + +""" + +from __future__ import annotations + +import pickle +import unittest +from typing import TYPE_CHECKING + +from tests._support import reimport_once_per_class +from tests.foundation.reassembly.test_dtls_handshake_reassembly_2440_unit import (_extract, _frame, + _pcap) +from tests.foundation.reassembly.test_dtls_reassembly_parity_2531_unit import COMPLETE +from tests.foundation.reassembly.test_ip_tcp_analysis_2581_unit import (CONTAINERS, FRAMES, + _analysis, _held, _parsed) +from tests.utilities.test_pickle_graalpy_2714_unit import _graalpy_walk + +if TYPE_CHECKING: + from typing import Any, Callable, Iterator + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + + +class DatagramPickleTests(unittest.TestCase): + """Each engine's datagram pickles under every protocol, through GraalPy's walk.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _engines(self) -> 'Iterator[tuple[str, str, Any, Any, Callable[[Any], Any]]]': + """``(kind, container, datagram, function, value)`` for each engine. + + ``function`` is the engine's analysis function, and ``value`` makes what it + returns comparable by value. + + """ + from pcapkit.foundation.analysis import dtls_handshake + + for container, write in CONTAINERS: + for kind, (frames, _) in FRAMES.items(): + extractor = _extract(self, write(frames), reassembly=True, ip=True, tcp=True) + datagram, = getattr(extractor.reassembly, kind) + yield kind, container, datagram, _analysis(kind)[0], _parsed + extractor = _extract(self, write([_frame(record) for record in COMPLETE]), + reassembly=True, dtls=True) + datagram, = extractor.reassembly.dtls + yield 'dtls', container, datagram, dtls_handshake, lambda packet: packet + + def test_a_datagram_pickles_under_every_protocol(self) -> None: + from pcapkit.foundation.reassembly.data.data import Deferred + + for kind, container, datagram, function, value in self._engines(): + held = _held(datagram) + self.assertIsInstance(held, Deferred) + expected = value(function(datagram)) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(kind=kind, container=container, protocol=protocol): + with _graalpy_walk(): + datagram.__reduce_ex__(protocol) + copy = pickle.loads(pickle.dumps(datagram, protocol=protocol)) + self.assertIs(_held(datagram), held) + self.assertIsInstance(_held(copy), Deferred) + self.assertIs(type(copy.id), type(datagram.id)) + self.assertEqual(copy.id, datagram.id) + self.assertEqual(value(copy.packet), expected) + + def test_the_reassembly_result_pickles_under_every_protocol(self) -> None: + from pcapkit.foundation.reassembly.data.data import Deferred, ReassemblyData + + reassembly = _extract(self, _pcap(FRAMES['ipv4'][0]), reassembly=True, ip=True).reassembly + held = _held(reassembly.ipv4[0]) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + with _graalpy_walk(): + copy = pickle.loads(pickle.dumps(reassembly, protocol=protocol)) + self.assertIs(type(copy), ReassemblyData) + self.assertEqual((len(copy.ipv4), copy.ipv6, copy.tcp, copy.dtls), + (1, reassembly.ipv6, reassembly.tcp, reassembly.dtls)) + self.assertIsInstance(_held(copy.ipv4[0]), Deferred) + self.assertIs(_held(reassembly.ipv4[0]), held) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/reassembly/test_deferred_pickle_2423_unit.py b/tests/foundation/reassembly/test_deferred_pickle_2423_unit.py index 1463d3b7e2..5f05b302ce 100644 --- a/tests/foundation/reassembly/test_deferred_pickle_2423_unit.py +++ b/tests/foundation/reassembly/test_deferred_pickle_2423_unit.py @@ -82,6 +82,18 @@ def _analyze(proto: 'object', payload: 'bytes') -> 'tuple[object, bytes]': return (proto, payload) +def _state(obj: 'object') -> 'object': + """The state :mod:`pickle` carries for ``obj``, under every protocol. + + ``Deferred`` defines no ``__getstate__`` of its own (#2714), so this is + Python's default state for a slotted class: ``object.__getstate__()`` on + 3.11 and later, and the same pair from the protocol-2 reduction below it, + which is why it is read through ``__reduce_ex__(2)[2]``. + + """ + return obj.__reduce_ex__(2)[2] + + class _BareStringSlotSub(_Deferred): """A subclass declaring its own slot as a bare string, not a tuple. @@ -161,7 +173,7 @@ def test_protocol_1_round_trips(self) -> None: self.assertEqual(restored.payload, b'payload') def test_getstate_shape_is_dict_state_and_slots_state(self) -> None: - """``__getstate__`` returns the native ``(dict_or_None, slots)`` pair. + """The state is the native ``(dict_or_None, slots)`` pair. Not a flat :obj:`dict` -- that was the first revision's shape, and :issue:`2425` finding 1 is exactly that it disagreed with the shape @@ -171,7 +183,7 @@ def test_getstate_shape_is_dict_state_and_slots_state(self) -> None: from pcapkit.foundation.reassembly.data.data import Deferred deferred = Deferred(_analyze, 42, b'payload') - state = deferred.__getstate__() + state = _state(deferred) self.assertIsInstance(state, tuple) self.assertEqual(len(state), 2) @@ -261,25 +273,18 @@ def test_setstate_still_accepts_the_first_revisions_flat_shape(self) -> None: def test_a_pickle_made_before_the_2423_fix_still_unpickles(self) -> None: """End-to-end: real pickle bytes, built the way a released version would. - ``Deferred.__getstate__``/``__setstate__`` are removed for just long - enough to dump a real ``Deferred`` instance through Python's default - slotted-class reduction, standing in for a pickle made by any - pre-:issue:`2423` release. The dump itself never touches this - class's own (fixed) ``__getstate__``; only the restore does. + ``Deferred`` defines no ``__getstate__`` (#2714), so a protocol-2 dump + goes through Python's default slotted-class reduction, as any + pre-:issue:`2423` release's did; only the restore goes through + ``__setstate__``. """ from pcapkit.foundation.reassembly.data.data import Deferred - getstate = Deferred.__dict__['__getstate__'] - setstate = Deferred.__dict__['__setstate__'] - del Deferred.__getstate__ - del Deferred.__setstate__ - try: - pre_fix = Deferred(_analyze, 7, b'released') - pickled = pickle.dumps(pre_fix, protocol=2) - finally: - Deferred.__getstate__ = getstate - Deferred.__setstate__ = setstate + self.assertNotIn('__getstate__', vars(Deferred)) + pre_fix = Deferred(_analyze, 7, b'released') + self.assertEqual(pre_fix.__reduce_ex__(2), object.__reduce_ex__(pre_fix, 2)) + pickled = pickle.dumps(pre_fix, protocol=2) restored = pickle.loads(pickled) self.assertEqual(restored.proto, 7) @@ -297,14 +302,15 @@ class DeferredPickleSubclassTests(unittest.TestCase): raising -- the loss is silent on ``__getstate__`` and only surfaces as a missing attribute later, on whatever first reads it back. - Exercised directly through ``__getstate__``/``__setstate__`` rather than a - full :func:`pickle.dumps`/:func:`pickle.loads` round trip: the subclass + Exercised directly through the state :mod:`pickle` carries and + ``__setstate__`` rather than a full :func:`pickle.dumps`/:func:`pickle.loads` + round trip: the subclass has to be defined inside the test for the same reason the module docstring gives for not using a lambda as the analyser -- a class local to a function is not importable by its qualified name, so :mod:`pickle` - cannot locate it to build a real pickle. The two methods are exactly the - mechanism :issue:`2425` finding 2 is about, so calling them directly still - exercises the regression. + cannot locate it to build a real pickle. The state and ``__setstate__`` are + exactly the mechanism :issue:`2425` finding 2 is about, so reading them + directly still exercises the regression. """ @@ -329,7 +335,7 @@ def test_getstate_carries_the_base_classs_slots_as_well_as_the_subclasss(self) - subclass = self._subclass_with_extra_slot() sub = subclass(_analyze, 7, b'x', 'extra-value') - dict_state, slots_state = sub.__getstate__() + dict_state, slots_state = _state(sub) self.assertIsNone(dict_state) self.assertEqual(slots_state, { 'analyze': _analyze, 'proto': 7, 'payload': b'x', 'extra': 'extra-value', @@ -339,7 +345,7 @@ def test_setstate_restores_both_the_subclasss_and_the_base_classs_slots(self) -> subclass = self._subclass_with_extra_slot() sub = subclass(_analyze, 7, b'x', 'extra-value') - state = sub.__getstate__() + state = _state(sub) restored = subclass.__new__(subclass) restored.__setstate__(state) self.assertEqual(restored.proto, 7) @@ -349,7 +355,7 @@ def test_setstate_restores_both_the_subclasss_and_the_base_classs_slots(self) -> class DeferredGetstateEdgeCaseTests(unittest.TestCase): - """The edge cases ``__getstate__``'s MRO walk has to match Python's own. + """The edge cases of the state, which ``Deferred`` takes from Python's own default. Each of these mirrors a branch of the native default state construction (see the module docstring's worked example in @@ -371,7 +377,7 @@ def test_getstate_skips_an_unset_slot(self) -> None: partial.payload = b'p' # ``analyze`` is deliberately left unset. - dict_state, slots_state = partial.__getstate__() + dict_state, slots_state = _state(partial) self.assertIsNone(dict_state) self.assertEqual(slots_state, {'proto': 1, 'payload': b'p'}) @@ -383,7 +389,7 @@ class _WithWeakrefSlot(Deferred): __slots__ = ('__weakref__',) sub = _WithWeakrefSlot(_analyze, 7, b'x') - dict_state, slots_state = sub.__getstate__() + dict_state, slots_state = _state(sub) self.assertIsNone(dict_state) self.assertNotIn('__weakref__', slots_state) self.assertEqual(slots_state, {'analyze': _analyze, 'proto': 7, 'payload': b'x'}) @@ -398,7 +404,7 @@ class _WithBareDict(Deferred): sub = _WithBareDict(_analyze, 7, b'x') sub.extra = 'extra-value' # lands in __dict__, not in a slot - dict_state, slots_state = sub.__getstate__() + dict_state, slots_state = _state(sub) self.assertEqual(dict_state, {'extra': 'extra-value'}) self.assertEqual(slots_state, {'analyze': _analyze, 'proto': 7, 'payload': b'x'}) diff --git a/tests/foundation/reassembly/test_deferred_pickle_graalpy_2433_unit.py b/tests/foundation/reassembly/test_deferred_pickle_graalpy_2433_unit.py index f7854298b6..8825ebc9d0 100644 --- a/tests/foundation/reassembly/test_deferred_pickle_graalpy_2433_unit.py +++ b/tests/foundation/reassembly/test_deferred_pickle_graalpy_2433_unit.py @@ -18,10 +18,11 @@ TypeError: typing.Generic() takes no arguments -:meth:`Deferred.__reduce_ex__` now hard-codes the ``base=object, state=None`` -outcome the walk always reached on CPython for protocol 0 and 1, which means -``copyreg._reduce_ex`` -- and its MRO walk -- is no longer called for those -two protocols at all. +``Deferred`` reduces under protocol 0 and 1 as protocol 2 does, through +:class:`~pcapkit.utilities.pickle.ReduceAsProtocol2` (#2714), so +``copyreg._reduce_ex`` -- and its MRO walk -- is not called for those two +protocols at all. #2435 first did so with a reduction of its own, which +hard-coded the ``base=object, state=None`` outcome the walk reached on CPython. A cross-reviewer confirmed this directly against real GraalPy 25.4.4 (``graalpy3.13-community-25.4.4``, downloaded from the @@ -118,7 +119,7 @@ class DeferredGraalPyPickleTests(unittest.TestCase): GraalPy 25.4's failure happens inside that function, before ``__getstate__`` is ever consulted (see the module docstring), so the fix - this file pins is that ``Deferred.__reduce_ex__`` never calls it for + this file pins is that pickling a ``Deferred`` never calls it for protocol 0 or 1 -- simulated here by making the call raise the same :exc:`TypeError` real GraalPy does. diff --git a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py index cbcfc9a7bf..39704bfc86 100644 --- a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py +++ b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py @@ -23,7 +23,6 @@ import pickle import plistlib import struct -import sys import tempfile import unittest import warnings @@ -37,11 +36,6 @@ RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) -#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. -#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, -#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). -LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 - #: NTP seconds and fraction of the Mobility Message Replay Protection option: #: the UNIX epoch counted from 1900, plus a quarter of a second. NTP = (0x83aa7e80, 0x40000000) @@ -172,7 +166,7 @@ def test_pickling_round_trips(self) -> None: from pcapkit.corekit.infoclass import Info _, mesg_id, timestamp = self.parse() - for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): for value in (mesg_id.ntp_timestamp, timestamp.pmip_timestamp, mesg_id, timestamp): with self.subTest(protocol=protocol, type=type(value).__name__): restored = pickle.loads(pickle.dumps(value, protocol)) diff --git a/tests/utilities/test_pickle_graalpy_2714_unit.py b/tests/utilities/test_pickle_graalpy_2714_unit.py new file mode 100644 index 0000000000..dfa7395d9d --- /dev/null +++ b/tests/utilities/test_pickle_graalpy_2714_unit.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""A generic class pickles under protocol 0 and 1 on GraalPy (#2714). + +Below protocol 2, :meth:`object.__reduce_ex__` hands an instance to +``copyreg._reduce_ex``, which walks the class's MRO for the first base that is not a +heap type and calls that base on the instance unless it is :class:`object`. +:class:`typing.Generic` is a heap type on CPython, so the walk passes it there. +GraalPy builds ``Generic`` as a built-in, so its walk stops at it and raises +``TypeError: typing.Generic() takes no arguments`` for every ``Info``, ``Schema``, +reassembly engine and ``Deferred``. :class:`~pcapkit.utilities.pickle.ReduceAsProtocol2`, +which each of them derives from, reduces as protocol 2 does under every protocol, +so the walk is never reached. The other reductions :mod:`pcapkit.utilities.pickle` +shares never reach it either, and each class using one is pinned here too. + +GraalPy is not installed here, so :func:`_graalpy_reduce_ex` stands in for +``copyreg._reduce_ex`` with GraalPy's walk, treating ``Generic`` as a built-in. +:mod:`tests.foundation.reassembly.test_datagram_pickle_graalpy_2714_unit` uses it +for the reassembled datagrams. + +""" + +from __future__ import annotations + +import copyreg +import decimal +import ipaddress +import pickle +import struct +import unittest +from typing import TYPE_CHECKING, Generic, TypeVar +from unittest import mock + +from tests._support import reimport_once_per_class + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + +_T = TypeVar('_T') + +#: The real hook, which the stand-in defers to wherever GraalPy's walk agrees. +_REDUCE_EX = copyreg._reduce_ex # pylint: disable=protected-access + +#: Every pickle protocol, ``0`` and ``1`` included. +PROTOCOLS = tuple(range(pickle.HIGHEST_PROTOCOL + 1)) + +#: A standalone HOPOPT header carrying one MPL option whose seed is the IPv6 +#: source address, which a lone header has none of, so its schema holds +#: ``NO_VALUE`` (#1842). +MPL_HEADER = bytes.fromhex('3b006d0220070000') +#: An IPv4 header with no options and no payload. +IPV4_HEADER = struct.pack('!BBHHHBBH4s4s', 0x45, 0, 20, 1, 0, 64, 59, 0, + bytes([192, 0, 2, 1]), bytes([198, 51, 100, 2])) + + +class _Plain(Generic[_T]): + """A generic class with no reduction of its own, which GraalPy cannot pickle below 2. + + Module-level, so :mod:`pickle` can find it by name. + + """ + + +def _graalpy_reduce_ex(obj: 'Any', proto: int) -> 'Any': + """``copyreg._reduce_ex``, walking the MRO as GraalPy does. + + The walk stops at the first base that is not a heap type, or whose ``__new__`` is + its own built-in, as ``copyreg._reduce_ex`` does, and also at + :class:`typing.Generic`, which GraalPy builds as a built-in. Stopping there raises + GraalPy's error, from calling ``Generic(obj)``; anywhere else defers to the real + walk, which stops at the same base. + + """ + def stops(base: type) -> bool: + new = base.__new__ + return (base is Generic or not base.__flags__ & 1 << 9 # Py_TPFLAGS_HEAPTYPE + or isinstance(new, type(int.__new__)) and new.__self__ is base) + + # NOTE: the walk always stops, at object if nowhere earlier. + if next(base for base in type(obj).__mro__ if stops(base)) is Generic: + raise TypeError('typing.Generic() takes no arguments') + return _REDUCE_EX(obj, proto) + + +def _graalpy_walk() -> 'Any': + """Patch :func:`_graalpy_reduce_ex` in for ``copyreg._reduce_ex``.""" + return mock.patch.object(copyreg, '_reduce_ex', _graalpy_reduce_ex) + + +def _round_trip(obj: 'Any', protocol: int) -> 'Any': + """``obj`` through :mod:`pickle` at ``protocol``, under GraalPy's walk.""" + with _graalpy_walk(): + return pickle.loads(pickle.dumps(obj, protocol=protocol)) + + +class StandInTests(unittest.TestCase): + """The stand-in refuses what GraalPy refuses, so the passes below are not vacuous.""" + + def test_the_stand_in_refuses_a_generic_class_as_graalpy_does(self) -> None: + refused = r'^typing\.Generic\(\) takes no arguments$' + for protocol in (0, 1): + with self.subTest(protocol=protocol), self.assertRaisesRegex(TypeError, refused): + _round_trip(_Plain(), protocol) + self.assertIsInstance(_round_trip(_Plain(), 2), _Plain) + self.assertIs(copyreg._reduce_ex, _REDUCE_EX) # pylint: disable=protected-access + + +class GenericPickleTests(unittest.TestCase): + """Each class deriving from ``ReduceAsProtocol2`` pickles under every protocol.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_an_info_reduces_as_protocol_2_does(self) -> None: + from pcapkit.corekit.infoclass import Info + from pcapkit.protocols.internet.ipv4 import IPv4 + + info = IPv4(IPV4_HEADER).info + self.assertIsInstance(info, Info) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + self.assertEqual(info.__reduce_ex__(protocol), info.__reduce_ex__(max(protocol, 2))) + copy = _round_trip(info, protocol) + self.assertIs(type(copy), type(info)) + self.assertEqual(copy, info) + + def test_an_ordered_multi_info_keeps_its_own_reduction(self) -> None: + from pcapkit.corekit.infoclass import OrderedMultiInfo + from pcapkit.corekit.multidict import OrderedMultiDict + + options = OrderedMultiInfo(OrderedMultiDict([('a', 1), ('b', 2), ('a', 3)])) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + self.assertEqual(options.__reduce_ex__(protocol), + (OrderedMultiInfo, ([('a', 1), ('b', 2), ('a', 3)],))) + copy = _round_trip(options, protocol) + self.assertIs(type(copy), OrderedMultiInfo) + self.assertEqual(list(copy.items(multi=True)), [('a', 1), ('b', 2), ('a', 3)]) + + def test_a_schema_pickles_and_keeps_its_sentinel(self) -> None: + from pcapkit.corekit.sentinels import NO_VALUE + from pcapkit.protocols.internet.hopopt import HOPOPT + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.schema.schema import Schema + + mpl = HOPOPT(MPL_HEADER, len(MPL_HEADER), extension=True).schema + ipv4 = IPv4(IPV4_HEADER).schema + self.assertIs(mpl.options[0].seed, NO_VALUE) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + copy = _round_trip(mpl, protocol) + self.assertIsInstance(copy, Schema) + self.assertIs(copy.options[0].seed, NO_VALUE) + self.assertEqual(bytes(copy), MPL_HEADER) + copy = _round_trip(ipv4, protocol) + self.assertIs(type(copy), type(ipv4)) + self.assertEqual(bytes(copy), IPV4_HEADER) + + def test_a_reassembly_engine_pickles_mid_datagram(self) -> None: + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Packet + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + bufid = (ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2'), 42, + TransType.UDP) + + def fragment(num: int, fo: int, mf: bool, payload: bytes) -> 'Any': + return Packet(bufid, num, fo, 20, mf, 20 + len(payload), b'ip-header', + bytearray(payload), 1000.0) + + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + engine = IPv4() + engine(fragment(1, 0, True, b'01234567')) + copy = _round_trip(engine, protocol) + self.assertIs(type(copy), IPv4) + copy(fragment(2, 8, False, b'89abcdef')) + datagram, = copy.datagram + self.assertTrue(datagram.completed) + self.assertEqual(datagram.payload, b'0123456789abcdef') + + +class SharedReductionTests(unittest.TestCase): + """Each class on a shared reduction pickles under every protocol, through GraalPy's walk.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_a_deferred_pickles_still_deferred(self) -> None: + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.data import Deferred + from pcapkit.protocols.internet.ipv4 import IPv4 + + payload = struct.pack('!HHHH', 1234, 9, 8, 0) + deferred = Deferred(IPv4.analyze, TransType.UDP, payload) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + copy = _round_trip(deferred, protocol) + self.assertIs(type(copy), Deferred) + self.assertEqual((copy.analyze, copy.proto, copy.payload), + (IPv4.analyze, TransType.UDP, payload)) + self.assertEqual(copy().info, deferred().info) + + def test_a_frame_record_pickles_with_its_slots(self) -> None: + from pcapkit.foundation.traceflow.data.data import FrameRecord + from pcapkit.protocols.data.misc.pcap.frame import FrameInfo + + record = FrameRecord({'packet': b'rebuilt', 'Ethernet': {'type': 2048}}, packet=b'captured', + frame_info=FrameInfo(ts_sec=1, ts_usec=2, incl_len=8, orig_len=9), + time_epoch=decimal.Decimal('1.000002')) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + copy = _round_trip(record, protocol) + self.assertIs(type(copy), FrameRecord) + self.assertEqual(copy, record) + self.assertEqual((copy.packet, copy.frame_info, copy.time_epoch), + (record.packet, record.frame_info, record.time_epoch)) + + def test_a_multi_valued_mapping_reduces_to_its_pairs(self) -> None: + from pcapkit.corekit.infoclass import MultiInfo + from pcapkit.corekit.multidict import OrderedMultiDict + + pairs = [('a', 1), ('b', 2), ('a', 3)] + for kind in (OrderedMultiDict, MultiInfo): + mapping = kind(pairs) + for protocol in PROTOCOLS: + with self.subTest(kind=kind.__name__, protocol=protocol): + self.assertEqual(mapping.__reduce_ex__(protocol), + (kind, (list(mapping.items(multi=True)),))) + copy = _round_trip(mapping, protocol) + self.assertIs(type(copy), kind) + self.assertEqual(list(copy.items(multi=True)), list(mapping.items(multi=True))) + + def test_a_singleton_pickles_as_itself(self) -> None: + from pcapkit.corekit import multidict, sentinels + + for name in ('NULL', 'NO_VALUE', 'NO_DEFAULT', 'ABSENT', 'MISSING'): + for protocol in PROTOCOLS: + with self.subTest(sentinel=name, protocol=protocol): + sentinel = getattr(sentinels, name) + self.assertIs(_round_trip(sentinel, protocol), sentinel) + for protocol in PROTOCOLS: + with self.subTest(sentinel='_missing', protocol=protocol): + missing = multidict._missing # pylint: disable=protected-access + self.assertIs(_round_trip(missing, protocol), missing) + + def test_an_unregistered_member_pickles_unregistered(self) -> None: + from pcapkit.const.reg.apptype import AppType, TransportProtocol + from pcapkit.corekit.fields.numbers import PortEnumField + + tcp = AppType.__registries__[TransportProtocol.tcp] + field = PortEnumField(length=2, namespace=AppType, proto=TransportProtocol.tcp) + before = len(tcp.__members__) + members = (('registry', tcp.get(50000)), ('field', field.unpack(b'\xd0\x9e', {}))) + for source, member in members: + for protocol in PROTOCOLS: + with self.subTest(source=source, protocol=protocol): + copy = _round_trip(member, protocol) + self.assertIs(type(copy), type(member)) + self.assertEqual((copy, copy.port, copy.svc), (member, member.port, member.svc)) + self.assertIn('__reduce_ex__', copy.__dict__) + self.assertEqual(len(tcp.__members__), before) + + +if __name__ == '__main__': + unittest.main()