From 0ffa3ca7183826ca9974fc43488a74a8d0ee9555 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 3 Nov 2026 00:29:01 -0500 Subject: [PATCH] refactor!: hold mapping values in Info or Data, not dict subclasses (#2726) - FrameRecord is an Info rather than a slotted dict subclass, so it is immutable and no dict. Its mapping is the engine's dissection; packet, frame_info and time_epoch sit under hidden dunder keys, read through __getattribute__, so a mapping key of the same name never shadows them. It pickles through _frame_record, as the 1.5 releases do, and to_dict() exports the mapping alone. - TLSKeyLog.entries is a TLSKeyLogEntries data model keyed by label, not a plain dict, and a mapping given for it is stored as one. to_dict() exports it as an OrderedMultiDict, as #1484 requires of every level; TLSKeyLog pickles hold the entries as a dict and restore the model. - The traced Packet's frame annotation names FrameRecord, and its versionchanged notes what from_dict(to_dict()) gives back; the pickle docs drop FrameRecord as their slotted-dict example. Pickles load across main and 1.5.1 and this change both ways at protocols 0, 1, 2 and 5, except main's own FrameRecord pickles, a format no release wrote. Dumps and from_data rebuilds match main. New tests fail on main. --- docs/source/pcapkit/protocols/misc/pcapng.rst | 4 + docs/source/pcapkit/utilities/pickle.rst | 4 +- pcapkit/foundation/traceflow/data/data.py | 104 +++++++--- pcapkit/foundation/traceflow/data/tcp.py | 20 +- pcapkit/protocols/data/__init__.py | 4 +- pcapkit/protocols/data/misc/__init__.py | 5 +- pcapkit/protocols/data/misc/pcapng.py | 79 +++++++- pcapkit/protocols/misc/pcapng.py | 15 +- pcapkit/utilities/pickle.py | 5 +- .../data/test_frame_record_info_2726_unit.py | 185 ++++++++++++++++++ .../test_pcapng_tls_entries_info_2726_unit.py | 184 +++++++++++++++++ .../test_pickle_graalpy_2714_unit.py | 2 +- 12 files changed, 561 insertions(+), 50 deletions(-) create mode 100644 tests/foundation/traceflow/data/test_frame_record_info_2726_unit.py create mode 100644 tests/protocols/misc/test_pcapng_tls_entries_info_2726_unit.py diff --git a/docs/source/pcapkit/protocols/misc/pcapng.rst b/docs/source/pcapkit/protocols/misc/pcapng.rst index fea9668faa..cb1a6d1f39 100644 --- a/docs/source/pcapkit/protocols/misc/pcapng.rst +++ b/docs/source/pcapkit/protocols/misc/pcapng.rst @@ -835,6 +835,10 @@ Data Models :members: :show-inheritance: +.. autoclass:: pcapkit.protocols.data.misc.pcapng.TLSKeyLogEntries + :members: + :show-inheritance: + .. autoclass:: pcapkit.protocols.data.misc.pcapng.TLSKeyLog :members: :show-inheritance: diff --git a/docs/source/pcapkit/utilities/pickle.rst b/docs/source/pcapkit/utilities/pickle.rst index d693c64a12..e9f567b289 100644 --- a/docs/source/pcapkit/utilities/pickle.rst +++ b/docs/source/pcapkit/utilities/pickle.rst @@ -5,8 +5,8 @@ Pickle Reductions :mod:`pcapkit.utilities.pickle` holds the :mod:`pickle` reductions several :mod:`pcapkit` classes share, so each is written once (:issue:`2714`). -:class:`ReduceAsProtocol2` lets a :class:`typing.Generic` subclass or a slotted -:class:`dict` pickle under protocol 0 and 1, on GraalPy as on CPython; +:class:`ReduceAsProtocol2` lets a :class:`typing.Generic` subclass or a class with +``__slots__`` pickle under protocol 0 and 1, on GraalPy as on CPython; :class:`ReduceToName` reduces a singleton to the module-level name holding it; :class:`ReduceToItems` reduces a multi-valued mapping to a call of its class on its item pairs; and :func:`reduce_to` makes one instance reduce to a call of a given diff --git a/pcapkit/foundation/traceflow/data/data.py b/pcapkit/foundation/traceflow/data/data.py index 29e21463e0..94d19c2c62 100644 --- a/pcapkit/foundation/traceflow/data/data.py +++ b/pcapkit/foundation/traceflow/data/data.py @@ -4,13 +4,12 @@ from typing import TYPE_CHECKING from pcapkit.corekit.infoclass import Info, info_final -from pcapkit.utilities.pickle import ReduceAsProtocol2 __all__ = ['TraceFlowData', 'Deferred', 'FrameRecord'] if TYPE_CHECKING: from decimal import Decimal - from typing import Any, Optional + from typing import Any, Mapping, Optional, SupportsIndex from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly @@ -67,7 +66,13 @@ def __call__(self) -> 'tuple[TCP_Datagram, ...]': return self.reassembly.datagram -class FrameRecord(ReduceAsProtocol2, dict): +#: Each attribute a :class:`FrameRecord` carries beside its mapping, to the +#: dunder key holding it, which iteration, ``len()`` and the dumps leave out. +_RECORD = {'packet': '__packet__', 'frame_info': '__frame_info__', 'time_epoch': '__time_epoch__'} + + +@info_final +class FrameRecord(Info): """A frame as a third-party engine's flow tracing adapter reports it. The mapping is the engine's own dissection of the frame, which every trace @@ -76,7 +81,9 @@ class FrameRecord(ReduceAsProtocol2, dict): reads of a frame -- :meth:`PCAPIO._append_value ` takes ``packet``, ``frame_info`` and ``time_epoch`` -- so ``trace_format='pcap'`` writes the - frame's record too (:issue:`1507`). + frame's record too (:issue:`1507`). A key of the mapping and an attribute + of the same name stay apart: ``record['packet']`` is the engine's, and + ``record.packet`` the frame's captured octets. Args: mapping: The engine's dissection of the frame. @@ -84,36 +91,83 @@ class FrameRecord(ReduceAsProtocol2, dict): frame_info: The frame's PCAP record header. time_epoch: The frame's exact UNIX timestamp. - """ + .. versionchanged:: 1.6.0 + + An :class:`~pcapkit.corekit.infoclass.Info` rather than a :obj:`dict` + subclass, so it is immutable and no :obj:`dict`. :meth:`to_dict` + exports the mapping alone, as any :class:`~pcapkit.corekit.infoclass.Info` + exports its keys, and drops the ``packet``, ``frame_info`` and + ``time_epoch`` attributes (:issue:`2726`). - __slots__ = ('packet', 'frame_info', 'time_epoch') + """ - #: The frame's captured octets. - packet: 'bytes' - #: The frame's PCAP record header, with the fraction in the capture's own - #: resolution. - frame_info: 'Data_FrameInfo' - #: The frame's exact UNIX timestamp, from which - #: :meth:`PCAPIO._make_timestamp ` - #: tells which resolution ``frame_info.ts_usec`` is in. A - #: :class:`~decimal.Decimal`, since only that settles it exactly. - time_epoch: 'Decimal' + #: The dunder keys holding the attributes, which ``repr`` and ``str`` + #: leave out as well. + __excluded__ = list(_RECORD.values()) - def __init__(self, mapping: 'dict[str, Any]', *, packet: 'bytes', + # NOTE: annotated for the type checker alone, since the attributes are no + # fields of the mapping, which is the engine's. + if TYPE_CHECKING: + #: The frame's captured octets. + packet: 'bytes' + #: The frame's PCAP record header, with the fraction in the capture's own + #: resolution. + frame_info: 'Data_FrameInfo' + #: The frame's exact UNIX timestamp, from which + #: :meth:`PCAPIO._make_timestamp ` + #: tells which resolution ``frame_info.ts_usec`` is in. A + #: :class:`~decimal.Decimal`, since only that settles it exactly. + time_epoch: 'Decimal' + + def __init__(self, mapping: 'Mapping[str, Any]', *, packet: 'bytes', frame_info: 'Data_FrameInfo', time_epoch: 'Decimal') -> 'None': super().__init__(mapping) - self.packet = packet - self.frame_info = frame_info - self.time_epoch = time_epoch + self.__dict__.update(zip(_RECORD.values(), (packet, frame_info, time_epoch))) + + def __getattribute__(self, name: 'str') -> 'Any': + """Read an attribute, taking :attr:`packet`, :attr:`frame_info` and + :attr:`time_epoch` from the keys holding them. + + So a mapping key of the same name never stands in for one of them. + + Args: + name: Attribute name. + + Raises: + AttributeError: If ``name`` is one of them and the record carries + none, as one :meth:`from_dict` builds from a mapping. + + """ + key = _RECORD.get(name) + if key is None: + return super().__getattribute__(name) + try: + return super().__getattribute__('__dict__')[key] + except KeyError: + raise AttributeError(f'{type(self).__name__!r} object has no attribute {name!r}') from None + + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'tuple[Any, ...]': + """Reduce to a call of :func:`_frame_record`, under every protocol. + + A release before 1.6.0 reduces a record to it as well, so a pickle + loads on either side of 1.6.0. + + Args: + protocol: Pickle protocol in use, which does not change the reduction. + + Returns: + ``(_frame_record, (mapping, packet, frame_info, time_epoch))``. + + """ + return _frame_record, (dict(self), self.packet, self.frame_info, self.time_epoch) -def _frame_record(mapping: 'dict[str, Any]', packet: 'bytes', +def _frame_record(mapping: 'Mapping[str, Any]', packet: 'bytes', frame_info: 'Data_FrameInfo', time_epoch: 'Decimal') -> 'FrameRecord': - """Rebuild a :class:`FrameRecord` pickled by a release before 1.6.0. + """Rebuild a pickled :class:`FrameRecord`. - Those pickles call this; a :class:`FrameRecord` reduces through - :class:`~pcapkit.utilities.pickle.ReduceAsProtocol2` instead, which carries - the slots under every protocol (:issue:`2714`). + Its pickles call this under every protocol, as those of a release before + 1.6.0 do. Args: mapping: The engine's dissection of the frame. diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index e9221bc374..13492fbcab 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -20,6 +20,7 @@ from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly + from pcapkit.foundation.traceflow.data.data import FrameRecord from pcapkit.protocols.data.misc.pcap.frame import Frame as Data_Frame _AT = TypeVar('_AT', 'IPv4Address', 'IPv6Address') @@ -47,6 +48,17 @@ class Packet(Info, Generic[_AT]): * :meth:`pcapkit.foundation.traceflow.traceflow.TraceFlow.dump` * :term:`trace.tcp.packet` + .. versionchanged:: 1.6.0 + + A third-party engine's :attr:`frame` is a + :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`, which + :meth:`to_dict` exports as an + :class:`~pcapkit.corekit.multidict.OrderedMultiDict`; so + ``Packet.from_dict(packet.to_dict())`` holds the frame as an + :class:`~pcapkit.corekit.infoclass.OrderedMultiInfo`, not a + :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord` + (:issue:`2726`). + """ #: Data link type: the PCAP global header's, or that of the PCAP-NG @@ -54,8 +66,10 @@ class Packet(Info, Generic[_AT]): protocol: 'Enum_LinkType' #: Frame number. index: 'int' - #: Extracted frame info. - frame: 'Data_Frame | dict[str, Any]' + #: Extracted frame info: the default engine's frame, a third-party + #: engine's :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`, or + #: the :obj:`dict` PyShark's adapter reports, which carries no frame record. + frame: 'Data_Frame | FrameRecord | dict[str, Any]' #: TCP synchronise (SYN) flag. syn: 'bool' #: TCP finish (FIN) flag. @@ -91,7 +105,7 @@ class Packet(Info, Generic[_AT]): payload: 'bytearray' if TYPE_CHECKING: - def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float', seq: 'int', ack: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | FrameRecord | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float', seq: 'int', ack: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final diff --git a/pcapkit/protocols/data/__init__.py b/pcapkit/protocols/data/__init__.py index 8b5369a8e8..3dfbcc6769 100644 --- a/pcapkit/protocols/data/__init__.py +++ b/pcapkit/protocols/data/__init__.py @@ -49,8 +49,8 @@ 'PCAPNG_ISB_FilterAcceptOption', 'PCAPNG_ISB_OSDropOption', 'PCAPNG_ISB_UsrDelivOption', 'PCAPNG_PACK_FlagsOption', 'PCAPNG_PACK_HashOption', 'PCAPNG_NameResolutionRecord', 'PCAPNG_UnknownRecord', 'PCAPNG_EndRecord', 'PCAPNG_IPv4Record', 'PCAPNG_IPv6Record', - 'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLog', 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', - 'PCAPNG_ZigBeeAPSKey', + 'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLogEntries', 'PCAPNG_TLSKeyLog', + 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', 'PCAPNG_ZigBeeAPSKey', 'PCAPNG_UnknownBlock', 'PCAPNG_SectionHeaderBlock', 'PCAPNG_InterfaceDescriptionBlock', 'PCAPNG_EnhancedPacketBlock', 'PCAPNG_SimplePacketBlock', 'PCAPNG_NameResolutionBlock', 'PCAPNG_InterfaceStatisticsBlock', 'PCAPNG_SystemdJournalExportBlock', 'PCAPNG_DecryptionSecretsBlock', diff --git a/pcapkit/protocols/data/misc/__init__.py b/pcapkit/protocols/data/misc/__init__.py index 7d20db2b7a..3f5489ff05 100644 --- a/pcapkit/protocols/data/misc/__init__.py +++ b/pcapkit/protocols/data/misc/__init__.py @@ -65,6 +65,7 @@ from pcapkit.protocols.data.misc.pcapng import \ SystemdJournalExportBlock as PCAPNG_SystemdJournalExportBlock from pcapkit.protocols.data.misc.pcapng import TLSKeyLog as PCAPNG_TLSKeyLog +from pcapkit.protocols.data.misc.pcapng import TLSKeyLogEntries as PCAPNG_TLSKeyLogEntries from pcapkit.protocols.data.misc.pcapng import UnknownBlock as PCAPNG_UnknownBlock from pcapkit.protocols.data.misc.pcapng import UnknownOption as PCAPNG_UnknownOption from pcapkit.protocols.data.misc.pcapng import UnknownRecord as PCAPNG_UnknownRecord @@ -98,8 +99,8 @@ 'PCAPNG_ISB_FilterAcceptOption', 'PCAPNG_ISB_OSDropOption', 'PCAPNG_ISB_UsrDelivOption', 'PCAPNG_PACK_FlagsOption', 'PCAPNG_PACK_HashOption', 'PCAPNG_NameResolutionRecord', 'PCAPNG_UnknownRecord', 'PCAPNG_EndRecord', 'PCAPNG_IPv4Record', 'PCAPNG_IPv6Record', - 'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLog', 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', - 'PCAPNG_ZigBeeAPSKey', + 'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLogEntries', 'PCAPNG_TLSKeyLog', + 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', 'PCAPNG_ZigBeeAPSKey', 'PCAPNG_UnknownBlock', 'PCAPNG_SectionHeaderBlock', 'PCAPNG_InterfaceDescriptionBlock', 'PCAPNG_EnhancedPacketBlock', 'PCAPNG_SimplePacketBlock', 'PCAPNG_NameResolutionBlock', 'PCAPNG_InterfaceStatisticsBlock', 'PCAPNG_SystemdJournalExportBlock', 'PCAPNG_DecryptionSecretsBlock', diff --git a/pcapkit/protocols/data/misc/pcapng.py b/pcapkit/protocols/data/misc/pcapng.py index 6e72bd220b..8137911dff 100644 --- a/pcapkit/protocols/data/misc/pcapng.py +++ b/pcapkit/protocols/data/misc/pcapng.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """data models for PCAP-NG file format""" +import collections.abc import datetime import decimal from typing import TYPE_CHECKING @@ -29,8 +30,8 @@ 'NameResolutionRecord', 'UnknownRecord', 'EndRecord', 'IPv4Record', 'IPv6Record', - 'DSBSecrets', 'UnknownSecrets', 'TLSKeyLog', 'WireGuardKeyLog', 'ZigBeeNWKKey', - 'ZigBeeAPSKey', + 'DSBSecrets', 'UnknownSecrets', 'TLSKeyLogEntries', 'TLSKeyLog', 'WireGuardKeyLog', + 'ZigBeeNWKKey', 'ZigBeeAPSKey', 'UnknownBlock', 'SectionHeaderBlock', 'InterfaceDescriptionBlock', 'EnhancedPacketBlock', 'SimplePacketBlock', 'NameResolutionBlock', @@ -44,6 +45,7 @@ from datetime import timezone as dt_timezone from decimal import Decimal from ipaddress import IPv4Address, IPv4Interface, IPv6Address, IPv6Interface + from typing import Any, Mapping, Optional from typing_extensions import Literal @@ -829,19 +831,86 @@ class UnknownSecrets(DSBSecrets): def __init__(self, data: 'bytes') -> 'None': ... +@info_final +class TLSKeyLogEntries(Data): + """Data model for the entries of a TLS key log, keyed by label. + + A key is each :class:`~pcapkit.const.pcapng.tls_key_label.TLSKeyLabel` + the log has an entry under, and its value that label's client random to + secret pairs, in log order, as an + :class:`~pcapkit.corekit.infoclass.OrderedMultiInfo`. A label also reads + as an attribute, as ``entries.CLIENT_RANDOM``. + + .. versionadded:: 1.6.0 + + """ + + if TYPE_CHECKING: + def __init__(self, dict_: 'Optional[Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]]' = None) -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + + @info_final class TLSKeyLog(DSBSecrets): - """Data model for TLS key log DSB secrets.""" + """Data model for TLS key log DSB secrets. + + .. versionchanged:: 1.6.0 + + :attr:`entries` is a :class:`TLSKeyLogEntries` rather than a + :obj:`dict`, and a mapping given for it is stored as one; :meth:`to_dict` + exports it as an :class:`~pcapkit.corekit.multidict.OrderedMultiDict`, + as it does any nested data model. A pickle holds it as a :obj:`dict`, so + pickles load across 1.6.0 in both directions (:issue:`2726`). + + """ #: TLS key log entries. - entries: 'dict[Enum_TLSKeyLabel, OrderedMultiInfo[bytes, bytes]]' + entries: 'TLSKeyLogEntries' if TYPE_CHECKING: #: TLS key log text as the block carries it, comments and line order #: included. Set on parsing only, and written back verbatim on rebuild. data: 'str' - def __init__(self, entries: 'dict[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]') -> 'None': ... + def __init__(self, entries: 'TLSKeyLogEntries | Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + + def __post_init__(self) -> 'None': + """Store a mapping given for :attr:`entries` as a :class:`TLSKeyLogEntries`.""" + _hold_entries(self.__dict__) + + def __getstate__(self) -> 'dict[str, Any]': + """Return the state to pickle, holding :attr:`entries` as a :obj:`dict`. + + A release before 1.6.0 loads them as the :obj:`dict` it holds, and + :meth:`__setstate__` loads them as a :class:`TLSKeyLogEntries`. + + """ + state = dict(self.__dict__) + if isinstance(state.get('entries'), TLSKeyLogEntries): + state['entries'] = dict(state['entries']) + return state + + def __setstate__(self, state: 'dict[str, Any]') -> 'None': + """Restore a pickled state, storing :attr:`entries` as a :class:`TLSKeyLogEntries`. + + Args: + state: The state :meth:`__getstate__` returns, or one holding the + entries as a :obj:`dict`, as before 1.6.0. + + """ + self.__dict__.update(state) + _hold_entries(self.__dict__) + + +def _hold_entries(namespace: 'dict[str, Any]') -> 'None': + """Store the ``entries`` in ``namespace`` as a :class:`TLSKeyLogEntries`, unless they are one. + + Args: + namespace: The :attr:`~object.__dict__` of a :class:`TLSKeyLog`. + + """ + entries = namespace.get('entries') + if isinstance(entries, collections.abc.Mapping) and not isinstance(entries, TLSKeyLogEntries): + namespace['entries'] = TLSKeyLogEntries(entries) @info_final diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index a640bb7d2a..c076c208d6 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -129,6 +129,7 @@ from pcapkit.protocols.data.misc.pcapng import \ SystemdJournalExportBlock as Data_SystemdJournalExportBlock from pcapkit.protocols.data.misc.pcapng import TLSKeyLog as Data_TLSKeyLog +from pcapkit.protocols.data.misc.pcapng import TLSKeyLogEntries as Data_TLSKeyLogEntries from pcapkit.protocols.data.misc.pcapng import UnknownBlock as Data_UnknownBlock from pcapkit.protocols.data.misc.pcapng import UnknownOption as Data_UnknownOption from pcapkit.protocols.data.misc.pcapng import UnknownRecord as Data_UnknownRecord @@ -225,8 +226,8 @@ from datetime import timedelta, timezone from decimal import Decimal from ipaddress import IPv4Address, IPv4Interface, IPv6Address, IPv6Interface - from typing import (IO, Any, Callable, ClassVar, Counter, DefaultDict, Optional, Tuple, Type, - TypeVar, Union) + from typing import (IO, Any, Callable, ClassVar, Counter, DefaultDict, Mapping, Optional, Tuple, + Type, TypeVar, Union) from mypy_extensions import DefaultArg, KwArg, NamedArg from typing_extensions import Literal @@ -4355,9 +4356,9 @@ def _read_secrets_tls(self, schema: 'Schema_TLSKeyLog', *, """ secrets = Data_TLSKeyLog( - # NOTE: each label's entries as an OrderedMultiInfo, which ``Info`` - # does not do for a mapping held in a dict (#1484). - entries={label: OrderedMultiInfo(entry) for (label, entry) in schema.entries.items()}, + # NOTE: each label's entries become an OrderedMultiInfo as the model + # stores them (#1484, #2726). + entries=Data_TLSKeyLogEntries(schema.entries), ) # NOTE: the log text is kept as well, since :attr:`entries` holds # neither its comments nor the order of its lines, and a rebuild writes @@ -6856,7 +6857,7 @@ def _make_secrets_unknown(self, type: 'Enum_SecretsType', secrets: 'Optional[Dat ) def _make_secrets_tls(self, type: 'Enum_SecretsType', secrets: 'Optional[Data_TLSKeyLog]' = None, *, - entries: 'Optional[dict[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]]' = None, + entries: 'Optional[Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]]' = None, data: 'Optional[str]' = None, **kwargs: 'Any') -> 'Schema_TLSKeyLog': """Make PCAP-NG TLS Key Log secrets. @@ -6893,7 +6894,7 @@ def _make_secrets_tls(self, type: 'Enum_SecretsType', secrets: 'Optional[Data_TL ) if secrets is not None: # NOTE: each an OrderedMultiInfo, which is an OrderedMultiDict. - entries = cast('dict[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]', secrets.entries) + entries = cast('Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]', secrets.entries) if entries is None: entries = {} diff --git a/pcapkit/utilities/pickle.py b/pcapkit/utilities/pickle.py index 87cf418307..1b10b4a69a 100644 --- a/pcapkit/utilities/pickle.py +++ b/pcapkit/utilities/pickle.py @@ -8,7 +8,7 @@ :mod:`pcapkit` classes share, so each is written once (:issue:`2714`): * :class:`ReduceAsProtocol2` reduces under every protocol as protocol 2 does, so a - :class:`typing.Generic` subclass or a slotted :class:`dict` pickles under + :class:`typing.Generic` subclass or a class with ``__slots__`` pickles under protocol 0 and 1, on GraalPy as on CPython; * :class:`ReduceToName` reduces a singleton to the module-level name holding it; * :class:`ReduceToItems` reduces a multi-valued mapping to a call of its class on @@ -56,8 +56,7 @@ class ReduceAsProtocol2: :class:`~pcapkit.corekit.infoclass.Info`, :class:`~pcapkit.protocols.schema.schema.Schema`, :class:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase` and - :class:`~pcapkit.foundation.reassembly.data.data.Deferred`, and before - :class:`dict` on :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`. + :class:`~pcapkit.foundation.reassembly.data.data.Deferred`. .. versionadded:: 1.6.0 diff --git a/tests/foundation/traceflow/data/test_frame_record_info_2726_unit.py b/tests/foundation/traceflow/data/test_frame_record_info_2726_unit.py new file mode 100644 index 0000000000..a2c2ddfefa --- /dev/null +++ b/tests/foundation/traceflow/data/test_frame_record_info_2726_unit.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +"""A traced frame record is an ``Info``, not a ``dict`` subclass (#2726). + +:class:`~pcapkit.foundation.traceflow.data.data.FrameRecord` is the engine's +dissection of a frame, as a mapping, with ``packet``, ``frame_info`` and +``time_epoch`` beside it as attributes the mapping does not show. It was a slotted +:class:`dict` subclass; the owner ruled on #2726 that no class subclass +:class:`dict` directly where an ``Info`` fits. These tests pin what the change +keeps -- the mapping, the attributes, what the dumpers write, and the dissection +``to_dict`` exports -- and that pickles load across it both ways, since a record +reduces to ``_frame_record`` as the 1.5 releases do. + +""" + +from __future__ import annotations + +import decimal +import importlib.util +import ipaddress +import os +import pickle +import tempfile +import unittest +from typing import TYPE_CHECKING + +from tests._support import reimport_once_per_class + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +#: Every pickle protocol, ``0`` and ``1`` included. +PROTOCOLS = tuple(range(pickle.HIGHEST_PROTOCOL + 1)) + +#: The engine's dissection; its ``packet`` differs from the record's on purpose. +MAPPING = {'packet': b'rebuilt', 'Ethernet': {'type': 2048, 'IPv4': {'ttl': 64}}} + + +def _record() -> 'Any': + """A record whose mapping is :data:`MAPPING`, built after the per-class import.""" + from pcapkit.foundation.traceflow.data.data import FrameRecord + from pcapkit.protocols.data.misc.pcap.frame import FrameInfo + + return FrameRecord(MAPPING, packet=b'captured', + frame_info=FrameInfo(ts_sec=1, ts_usec=2, incl_len=8, orig_len=9), + time_epoch=decimal.Decimal('1.000002')) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class FrameRecordInfoTests(unittest.TestCase): + """The record as an ``Info``: its mapping, its attributes and its exports.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_a_record_is_an_info_and_no_dict(self) -> None: + from pcapkit.corekit.infoclass import Info + + record = _record() + self.assertIsInstance(record, Info) + self.assertNotIsInstance(record, dict) + + def test_the_mapping_is_the_dissection_and_the_attributes_stay_apart(self) -> None: + record = _record() + self.assertEqual(list(record.items()), list(MAPPING.items())) + self.assertEqual(len(record), len(MAPPING)) + self.assertEqual(record, MAPPING) + self.assertEqual(record['packet'], b'rebuilt') + self.assertEqual(record.packet, b'captured') + self.assertEqual((record.frame_info.ts_sec, record.time_epoch), (1, decimal.Decimal('1.000002'))) + self.assertNotIn('frame_info', record) + self.assertNotIn('time_epoch', record) + self.assertNotIn('captured', repr(record)) + + def test_a_record_is_immutable(self) -> None: + from pcapkit.utilities.exceptions import UnsupportedCall + + record = _record() + with self.assertRaises(TypeError): + record['packet'] = b'' # type: ignore[index] + with self.assertRaises(UnsupportedCall): + record.packet = b'' # type: ignore[misc] + self.assertEqual((record['packet'], record.packet), (b'rebuilt', b'captured')) + + def test_to_dict_exports_the_dissection_alone(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.foundation.traceflow.data.tcp import Packet + + record = _record() + traced = Packet(LinkType.ETHERNET, 1, record, True, False, False, + ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2'), + 1, 2, 1.0, 7, 8, b'header', bytearray(b'payload')) + for export in (record.to_dict(), traced.to_dict()['frame']): + self.assertIsInstance(export, OrderedMultiDict) + self.assertEqual(list(export.items(multi=True)), list(MAPPING.items())) + + def test_every_dumper_writes_the_dissection_unchanged(self) -> None: + import dictdumper + + from pcapkit.dumpkit.common import make_dumper + + record = _record() + for kind in (dictdumper.JSON, dictdumper.PLIST, dictdumper.Tree): + with self.subTest(dumper=kind.__name__), tempfile.TemporaryDirectory() as tmp: + texts = [] + for name, value in (('record', record), ('dict', dict(MAPPING))): + path = os.path.join(tmp, name) + dumper = make_dumper(kind)(path) + dumper(value, name='Frame 1') + del dumper + with open(path, encoding='utf-8') as file: + texts.append(file.read()) + self.assertEqual(texts[0], texts[1]) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class FrameRecordPickleTests(unittest.TestCase): + """Pickles load across the change, both ways, under every protocol.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _assert_restored(self, clone: 'Any', record: 'Any') -> None: + from pcapkit.corekit.infoclass import Info + from pcapkit.foundation.traceflow.data.data import FrameRecord + + self.assertIs(type(clone), FrameRecord) + self.assertIsInstance(clone, Info) + self.assertEqual(clone, record) + self.assertEqual((clone.packet, clone.frame_info, clone.time_epoch), + (record.packet, record.frame_info, record.time_epoch)) + + def test_a_record_reduces_to_frame_record_under_every_protocol(self) -> None: + # NOTE: the 1.5 releases reduce a record to ``_frame_record`` too, so a + # pickle written here loads in them. + from pcapkit.foundation.traceflow.data.data import _frame_record + + record = _record() + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + rebuild, args = record.__reduce_ex__(protocol) + self.assertIs(rebuild, _frame_record) + self.assertIs(type(args[0]), dict) + self.assertEqual(args, (MAPPING, b'captured', record.frame_info, decimal.Decimal('1.000002'))) + self._assert_restored(pickle.loads(pickle.dumps(record, protocol=protocol)), record) + + def test_a_pickle_of_a_1_5_release_loads_as_the_info(self) -> None: + # NOTE: the slotted dict subclass of the 1.5 releases, whose __reduce__ + # called _frame_record on its mapping and its slots. + from pcapkit.foundation.traceflow.data.data import _frame_record + + class FrameRecord(dict): + __slots__ = ('packet', 'frame_info', 'time_epoch') + + def __reduce__(self) -> 'Any': + return _frame_record, (dict(self), self.packet, self.frame_info, self.time_epoch) + + record = _record() + old = FrameRecord(MAPPING) + old.packet, old.frame_info, old.time_epoch = record.packet, record.frame_info, record.time_epoch + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + self._assert_restored(pickle.loads(pickle.dumps(old, protocol=protocol)), record) + + def test_copies_keep_the_attributes_and_stay_immutable(self) -> None: + import copy + + from pcapkit.utilities.exceptions import UnsupportedCall + + record = _record() + for kind, clone in (('copy', copy.copy(record)), ('deepcopy', copy.deepcopy(record))): + with self.subTest(kind=kind): + self._assert_restored(clone, record) + with self.assertRaises(UnsupportedCall): + clone.packet = b'' + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/misc/test_pcapng_tls_entries_info_2726_unit.py b/tests/protocols/misc/test_pcapng_tls_entries_info_2726_unit.py new file mode 100644 index 0000000000..7b61b78c0c --- /dev/null +++ b/tests/protocols/misc/test_pcapng_tls_entries_info_2726_unit.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +"""A TLS key log's entries are a data model, not a plain ``dict`` (#2726). + +:attr:`TLSKeyLog.entries ` +maps each label to its client random to secret pairs. It was a plain +:class:`dict` held on the data model; the owner ruled on #2726 that a field in +:mod:`pcapkit.protocols.data` holds an ``Info`` or a data model instead, so it is +a :class:`~pcapkit.protocols.data.misc.pcapng.TLSKeyLogEntries`. These tests pin +the model, what ``to_dict`` exports and ``from_dict`` rebuilds, what the dumpers +write, a byte-exact rebuild, and pickles that hold the entries as the +:class:`dict`, so they load on either side of the change. + +""" + +import copy +import os +import pickle +import tempfile +import unittest +import warnings +from typing import TYPE_CHECKING +from unittest import mock + +from tests.protocols.misc.test_pcapng_dsb_secrets_dict_2698_unit import TLS, _dsb +from tests.protocols.misc.test_pcapng_roundtrip2_unit import PCAPNGTestCase + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + +#: Every pickle protocol, ``0`` and ``1`` included. +PROTOCOLS = tuple(range(pickle.HIGHEST_PROTOCOL + 1)) + +#: A key log with a comment, a label held twice and a second label. +SECRETS = (b'# keys\nCLIENT_RANDOM 00112233 aabbccdd\nCLIENT_RANDOM 4455 eeff\n' + b'EXPORTER_SECRET 0102 0304\n') +OCTETS = _dsb(TLS, SECRETS) + +#: The pairs each label holds, in log order. +PAIRS = { + 'CLIENT_RANDOM': [(bytes.fromhex('00112233'), bytes.fromhex('aabbccdd')), + (bytes.fromhex('4455'), bytes.fromhex('eeff'))], + 'EXPORTER_SECRET': [(bytes.fromhex('0102'), bytes.fromhex('0304'))], +} + + +class _TLSTestCase(PCAPNGTestCase): + """Parse :data:`OCTETS` after the per-class import.""" + + def _block(self) -> 'Any': + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + return self._parse(OCTETS).info + + def _assert_entries(self, entries: 'Any') -> None: + from pcapkit.const.pcapng.tls_key_label import TLSKeyLabel + from pcapkit.corekit.infoclass import OrderedMultiInfo + from pcapkit.protocols.data.misc.pcapng import TLSKeyLogEntries + + self.assertIs(type(entries), TLSKeyLogEntries) + self.assertEqual([type(label) for label in entries], [TLSKeyLabel] * len(PAIRS)) + self.assertEqual({label.name: list(pairs.items(multi=True)) for label, pairs in entries.items()}, + PAIRS) + for pairs in entries.values(): + self.assertIs(type(pairs), OrderedMultiInfo) + + +class TLSKeyLogEntriesTests(_TLSTestCase): + """The entries as a data model: the mapping, ``to_dict``, dumps and a rebuild.""" + + def test_entries_are_a_data_model_keyed_by_label(self) -> None: + from pcapkit.const.pcapng.tls_key_label import TLSKeyLabel + from pcapkit.protocols.data.data import Data + from pcapkit.utilities.exceptions import UnsupportedCall + + entries = self._block().secrets_data.entries + self._assert_entries(entries) + self.assertIsInstance(entries, Data) + self.assertNotIsInstance(entries, dict) + self.assertIn(TLSKeyLabel.CLIENT_RANDOM, entries) + self.assertIs(entries.CLIENT_RANDOM, entries[TLSKeyLabel.CLIENT_RANDOM]) + with self.assertRaises(UnsupportedCall): + entries.CLIENT_RANDOM = None + + def test_a_mapping_given_for_entries_is_stored_as_the_model(self) -> None: + from pcapkit.const.pcapng.tls_key_label import TLSKeyLabel + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.data.misc.pcapng import TLSKeyLog + + pairs = {TLSKeyLabel[name]: OrderedMultiDict(items) for name, items in PAIRS.items()} + self._assert_entries(TLSKeyLog(entries=pairs).entries) + + def test_to_dict_exports_the_entries_and_from_dict_rebuilds_them(self) -> None: + # NOTE: a nested model exports as an OrderedMultiDict at every level (#1484). + from pcapkit.corekit.infoclass import _OrderedMultiDict + from pcapkit.protocols.data.misc.pcapng import TLSKeyLog + + block = self._block() + tls = block.secrets_data + export = tls.to_dict()['entries'] + self.assertIs(type(export), _OrderedMultiDict) + self.assertEqual([type(pairs) for pairs in export.values()], [_OrderedMultiDict] * len(PAIRS)) + self.assertEqual({label.name: list(pairs.items(multi=True)) for label, pairs in export.items()}, + PAIRS) + self.assertIs(type(block.to_dict()['secrets_data']['entries']), _OrderedMultiDict) + rebuilt = TLSKeyLog.from_dict(tls.to_dict()) + self._assert_entries(rebuilt.entries) + self.assertEqual(rebuilt.entries, tls.entries) + + def test_the_block_rebuilds_exactly_from_its_model_and_its_dict(self) -> None: + from pcapkit.protocols.misc.pcapng import PCAPNG + + block = self._block() + for path, data in (('model', block), ('dict', block.to_dict())): + with self.subTest(path=path): + rebuilt = PCAPNG.from_data(data, num=2, sct=1, ctx=self._context()).data + self.assertEqual(rebuilt.hex(), OCTETS.hex()) + + def test_every_dumper_writes_the_entries_as_the_dict(self) -> None: + import dictdumper + + from pcapkit.dumpkit.common import make_dumper + + tls = self._block().secrets_data + held = copy.copy(tls) + held.__dict__['entries'] = dict(tls.entries) + for kind in (dictdumper.JSON, dictdumper.PLIST, dictdumper.Tree): + with self.subTest(dumper=kind.__name__), tempfile.TemporaryDirectory() as tmp: + texts = [] + for name, value in (('model', tls), ('dict', held)): + path = os.path.join(tmp, name) + dumper = make_dumper(kind)(path) + dumper(value, name='Secrets') + del dumper + with open(path, encoding='utf-8') as file: + texts.append(file.read()) + self.assertEqual(texts[0], texts[1]) + + +class TLSKeyLogPickleTests(_TLSTestCase): + """Pickles hold the entries as a :class:`dict`, and load them as the model.""" + + def test_a_pickle_holds_the_entries_as_a_dict(self) -> None: + block = self._block() + tls = block.secrets_data + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + state = tls.__reduce_ex__(protocol)[2] + self.assertIs(type(state['entries']), dict) + for value in (tls, block): + payload = pickle.dumps(value, protocol=protocol) + self.assertNotIn(b'TLSKeyLogEntries', payload) + clone = pickle.loads(payload) + entries = clone.entries if value is tls else clone.secrets_data.entries + self._assert_entries(entries) + self.assertEqual(entries, tls.entries) + + def test_a_pickle_holding_a_dict_loads_as_the_model(self) -> None: + # NOTE: the state of a model holding its entries as a dict, as one with + # no __getstate__ of its own pickles it. + tls = self._block().secrets_data + state = dict(tls.__dict__) + state['entries'] = dict(tls.entries) + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + with mock.patch.object(type(tls), '__getstate__', lambda self: state, create=True): + payload = pickle.dumps(tls, protocol=protocol) + clone = pickle.loads(payload) + self._assert_entries(clone.entries) + self.assertEqual(clone.data, SECRETS.decode()) + + def test_the_model_alone_pickles_as_itself(self) -> None: + entries = self._block().secrets_data.entries + for protocol in PROTOCOLS: + with self.subTest(protocol=protocol): + clone = pickle.loads(pickle.dumps(entries, protocol=protocol)) + self._assert_entries(clone) + self.assertEqual(clone, entries) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utilities/test_pickle_graalpy_2714_unit.py b/tests/utilities/test_pickle_graalpy_2714_unit.py index a4286f0994..20518d2e5f 100644 --- a/tests/utilities/test_pickle_graalpy_2714_unit.py +++ b/tests/utilities/test_pickle_graalpy_2714_unit.py @@ -208,7 +208,7 @@ def test_a_deferred_pickles_still_deferred(self) -> None: (IPv4.analyze, TransType.UDP, payload)) self.assertEqual(copy().info, deferred().info) - def test_a_frame_record_pickles_with_its_slots(self) -> None: + def test_a_frame_record_pickles_with_its_attributes(self) -> None: from pcapkit.foundation.traceflow.data.data import FrameRecord from pcapkit.protocols.data.misc.pcap.frame import FrameInfo