diff --git a/docs/source/contributing/conventions/index.rst b/docs/source/contributing/conventions/index.rst index b7fed7ade2..333b693f5f 100644 --- a/docs/source/contributing/conventions/index.rst +++ b/docs/source/contributing/conventions/index.rst @@ -20,8 +20,9 @@ House Conventions * :ref:`extension-header-subclassing` -- a protocol class hierarchy; * :ref:`protocol-family-grouping` -- how a protocol family's base and members spell :meth:`~pcapkit.protocols.protocol.Protocol.id`, how a protocol's - package, modules and classes are named, and how quietly a next-layer fallback keeps a - refused layer as :class:`~pcapkit.protocols.misc.raw.protocol.Raw`; + package, modules and classes are named and imported, and how quietly a + next-layer fallback keeps a refused layer as + :class:`~pcapkit.protocols.misc.raw.protocol.Raw`; * :ref:`protocol-member-order` -- the order of members in a protocol class body; * :ref:`protocol-layer-placement` -- which subpackage a dissector belongs in; * :ref:`process` -- the repository rather than any of its code; diff --git a/docs/source/contributing/conventions/protocol-family-grouping.rst b/docs/source/contributing/conventions/protocol-family-grouping.rst index 80fb679879..e2f9b497bd 100644 --- a/docs/source/contributing/conventions/protocol-family-grouping.rst +++ b/docs/source/contributing/conventions/protocol-family-grouping.rst @@ -456,6 +456,18 @@ these files with ``filter_files`` set, so one named on the command line is skipp too, and ``make isort`` skips every ``__init__.py`` besides. Ruled in review of :pr:`2753` (:issue:`2534`). +**The packages above a protocol's package import it on first use**, ruled by the +owner in session and recorded on :issue:`2730`. :mod:`pcapkit.protocols`, its +layer packages, :mod:`~pcapkit.protocols.misc.pcap` and +:mod:`~pcapkit.protocols.misc.checksum` name each protocol class under +``if TYPE_CHECKING:`` and bind the ``__getattr__`` and ``__dir__`` that +:func:`~pcapkit.utilities.importlib.lazy_exports` returns, as a :mod:`pcapkit.const` +package does (:ref:`const-vendor-pairing`), and :mod:`pcapkit` re-exports +:mod:`pcapkit.protocols` the same way. A protocol's own package keeps the imports +above, since importing its ``protocol`` module imports its ``schema`` and ``data`` +anyway. :data:`pcapkit.protocols.__proto__` names every protocol class, so +:mod:`pcapkit.protocols.registry` builds it when it is first reached. + **A family's shared pieces live in the family base's package**, and its members import them (:issue:`2534`): :class:`~pcapkit.protocols.link.c_tag.protocol.C_Tag` and :class:`~pcapkit.protocols.link.s_tag.protocol.S_Tag` read the schema and data diff --git a/docs/source/pcapkit/protocols/index.rst b/docs/source/pcapkit/protocols/index.rst index ddf072d5fb..112d21f2b5 100644 --- a/docs/source/pcapkit/protocols/index.rst +++ b/docs/source/pcapkit/protocols/index.rst @@ -196,9 +196,15 @@ from given information. The class hierarchy of :mod:`pcapkit.protocols`: Protocol Registry ----------------- +.. module:: pcapkit.protocols.registry + .. autodata:: pcapkit.protocols.__proto__ :no-value: + :mod:`pcapkit.protocols.registry` builds it the first time it is reached, and + imports every protocol class :mod:`pcapkit.protocols` exports to do so + (:issue:`2730`). + .. seealso:: Please refer to :func:`pcapkit.foundation.registry.protocols.protocol.register_protocol` diff --git a/pcapkit/__init__.py b/pcapkit/__init__.py index 7c55683138..3cee7782cc 100644 --- a/pcapkit/__init__.py +++ b/pcapkit/__init__.py @@ -75,10 +75,12 @@ ############################################################################### import warnings +from typing import TYPE_CHECKING import tbtrim from pcapkit.utilities.exceptions import BaseError +from pcapkit.utilities.importlib import lazy_exports from pcapkit.utilities.logging import DEVMODE from pcapkit.utilities.warnings import DevModeWarning @@ -93,7 +95,9 @@ exception=BaseError, strict=False) from pcapkit.interface import * -from pcapkit.protocols import * + +if TYPE_CHECKING: + from pcapkit.protocols import * __all__ = [ 'extract', 'reassemble', 'trace', # Interface Functions @@ -134,3 +138,6 @@ #: version number __version__ = '1.5.1' + +# Protocols are imported the first time they are reached (#2730). +__getattr__, __dir__ = lazy_exports(__name__, globals(), packages=('protocols',)) diff --git a/pcapkit/foundation/reassembly/reassembly.py b/pcapkit/foundation/reassembly/reassembly.py index 5d44e240e7..0c1f268958 100644 --- a/pcapkit/foundation/reassembly/reassembly.py +++ b/pcapkit/foundation/reassembly/reassembly.py @@ -15,7 +15,6 @@ import math from typing import TYPE_CHECKING, Generic, Type, TypeVar, cast -from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.misc.raw.protocol import Raw from pcapkit.utilities.exceptions import FieldValueError, UnsupportedCall from pcapkit.utilities.logging import get_logger @@ -79,6 +78,9 @@ def protocol(cls) -> 'Type[ProtocolBase]': """Protocol of current reassembly object.""" if hasattr(cls, '__protocol_type__'): return cls.__protocol_type__ + # NOTE: read here, not on import, which would build it and import every protocol (#2730). + from pcapkit.protocols import \ + __proto__ as protocol_registry # pylint: disable=import-outside-toplevel return protocol_registry.get(cls.name.upper(), Raw) @property diff --git a/pcapkit/foundation/registry/protocols/protocol.py b/pcapkit/foundation/registry/protocols/protocol.py index dcb1ffc525..94fd57f4fe 100644 --- a/pcapkit/foundation/registry/protocols/protocol.py +++ b/pcapkit/foundation/registry/protocols/protocol.py @@ -25,7 +25,6 @@ import aenum from pcapkit.corekit.sentinels import MISSING -from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.protocol import ProtocolBase from pcapkit.utilities.exceptions import RegistryError from pcapkit.utilities.logging import get_logger @@ -120,6 +119,10 @@ class rather than supplied by a caller, and this function is the funnel (:issue:`2529`). """ + # NOTE: read here, not on import, which would build it and import every protocol (#2730). + from pcapkit.protocols import \ + __proto__ as protocol_registry # pylint: disable=import-outside-toplevel + if not isinstance(protocol, type): raise RegistryError(f'protocol must be a class, not {protocol!r}') if not issubclass(protocol, ProtocolBase): @@ -230,6 +233,9 @@ def _release_name(target: 'tuple[type, Any]') -> 'None': target: The ``(destination, code)`` being registered over. """ + from pcapkit.protocols import \ + __proto__ as protocol_registry # pylint: disable=import-outside-toplevel + lease = _CODE_LEASES.pop(target, None) if lease is None: return @@ -274,6 +280,9 @@ def _register_name(protocol: 'Type[ProtocolBase]', *targets: 'tuple[type, Any]') *targets: The ``(destination, code)`` pairs it was registered at. """ + from pcapkit.protocols import \ + __proto__ as protocol_registry # pylint: disable=import-outside-toplevel + name = protocol.__name__.upper() with _LEASE_LOCK: stack = _NAME_LEASES.get(name) diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index ce839c5354..b94d71f38d 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -21,7 +21,6 @@ from pcapkit.corekit.module import ModuleDescriptor from pcapkit.dumpkit.common import make_dumper -from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.misc.raw.protocol import Raw from pcapkit.utilities.exceptions import FileExists, RegistryError, UnsupportedCall, stacklevel from pcapkit.utilities.logging import get_logger @@ -80,6 +79,9 @@ def protocol(cls) -> 'Type[ProtocolBase]': """Protocol of current object.""" if hasattr(cls, '__protocol_type__'): return cls.__protocol_type__ + # NOTE: read here, not on import, which would build it and import every protocol (#2730). + from pcapkit.protocols import \ + __proto__ as protocol_registry # pylint: disable=import-outside-toplevel return protocol_registry.get(cls.name.upper(), Raw) @property diff --git a/pcapkit/protocols/__init__.py b/pcapkit/protocols/__init__.py index 74a5bb82b7..fe4bba1015 100644 --- a/pcapkit/protocols/__init__.py +++ b/pcapkit/protocols/__init__.py @@ -8,21 +8,18 @@ :mod:`pcapkit.protocols` is a collection of all protocol families, with detailed implementation and methods. +Each protocol class, and :data:`__proto__`, is imported the first time it is +reached rather than with this package, through +:func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ from typing import TYPE_CHECKING +# NOTE: ahead of any protocol module: defining ``Protocol`` imports Raw and +# NoPayload, which import it back, so either imported first is met half-built. # Base Class for Protocols from pcapkit.protocols.protocol import Protocol -# Utility Classes for Protocols -from pcapkit.protocols.misc import * - -# Protocols & Macros -from pcapkit.protocols.link import * -from pcapkit.protocols.internet import * -from pcapkit.protocols.transport import * -from pcapkit.protocols.application import * - # Protocol Data from pcapkit.protocols import data from pcapkit.protocols.data import Data @@ -31,11 +28,25 @@ from pcapkit.protocols import schema from pcapkit.protocols.schema import Schema +from pcapkit.utilities.importlib import lazy_exports + if TYPE_CHECKING: from typing import Type from pcapkit.protocols.protocol import ProtocolBase + # Utility Classes for Protocols + from pcapkit.protocols.misc import * + + # Protocols & Macros + from pcapkit.protocols.link import * + from pcapkit.protocols.internet import * + from pcapkit.protocols.transport import * + from pcapkit.protocols.application import * + + #: dict[str, ~typing.Type[~pcapkit.protocols.protocol.ProtocolBase]]: Protocol registry. + __proto__ = {} # type: dict[str, Type[ProtocolBase]] + __all__ = [ # PCAP Headers 'Header', 'Frame', @@ -74,14 +85,7 @@ 'OSPF', 'RARP', 'DRARP', 'SSL', 'SSLv2', 'SSLv3', 'TLS', 'DTLS', -] - -#: dict[str, ~typing.Type[~pcapkit.protocols.protocol.ProtocolBase]]: Protocol registry. -__proto__ = {} # type: dict[str, Type[ProtocolBase]] -for name in __all__: - __proto__[name.upper()] = globals()[name] -__all__.extend(( # Protocol Numbers 'LINKTYPE', 'ETHERTYPE', 'TRANSTYPE', 'APPTYPE', @@ -90,4 +94,10 @@ # Protocol Schema 'Schema', 'schema', -)) +] + +# NOTE: ``Protocol`` is bound above, and named here so that ``dir()`` lists it. +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Protocol': ('protocol', 'Protocol'), + '__proto__': ('registry', '__proto__'), +}, packages=('misc', 'link', 'internet', 'transport', 'application')) diff --git a/pcapkit/protocols/application/__init__.py b/pcapkit/protocols/application/__init__.py index e8ad98c21b..cc6864e88a 100644 --- a/pcapkit/protocols/application/__init__.py +++ b/pcapkit/protocols/application/__init__.py @@ -8,6 +8,11 @@ :mod:`pcapkit.protocols.application` is a collection of all protocols in application layer. +Each protocol class, and the base class +:class:`~pcapkit.protocols.application.application.Application`, is imported the +first time it is reached rather than with this package, through +:func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ # TODO: Implements BGP, DHCP, DHCPv6, DNS, IMAP, LDAP, MQTT, # NNTP, NTP, ONC:RPC, POP, RIP, RTP, SIP, SMTP, SNMP, @@ -17,29 +22,33 @@ # dependency; see pcapkit.protocols.application.ngap.protocol. Importing the module # is free -- ``pycrate`` is imported inside the parse path, not here. -# Base Class for Application Layer -from pcapkit.protocols.application.application import Application - -# Utility Classes for Protocols -from pcapkit.protocols.application.ftp.protocol import FTP -from pcapkit.protocols.application.ftp_data.protocol import FTP_DATA -from pcapkit.protocols.application.httpv1.protocol import HTTPv1 -from pcapkit.protocols.application.httpv2.protocol import HTTPv2 -from pcapkit.protocols.application.ngap.protocol import NGAP -from pcapkit.protocols.application.ospf.protocol import OSPF -from pcapkit.protocols.application.rarp.protocol import RARP -from pcapkit.protocols.application.drarp.protocol import DRARP -from pcapkit.protocols.application.sslv2.protocol import SSLv2 -from pcapkit.protocols.application.tls.protocol import TLS -from pcapkit.protocols.application.sslv3.protocol import SSLv3 -from pcapkit.protocols.application.dtls.protocol import DTLS - -# Base Classes -from pcapkit.protocols.application.http.protocol import HTTP -from pcapkit.protocols.application.ssl.protocol import SSL +from typing import TYPE_CHECKING # Application Layer Protocol Numbers from pcapkit.const.reg.apptype import AppType as APPTYPE +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + # Base Class for Application Layer + from pcapkit.protocols.application.application import Application + + # Utility Classes for Protocols + from pcapkit.protocols.application.ftp import FTP + from pcapkit.protocols.application.ftp_data import FTP_DATA + from pcapkit.protocols.application.httpv1 import HTTPv1 + from pcapkit.protocols.application.httpv2 import HTTPv2 + from pcapkit.protocols.application.ngap import NGAP + from pcapkit.protocols.application.ospf import OSPF + from pcapkit.protocols.application.rarp import RARP + from pcapkit.protocols.application.drarp import DRARP + from pcapkit.protocols.application.sslv2 import SSLv2 + from pcapkit.protocols.application.tls import TLS + from pcapkit.protocols.application.sslv3 import SSLv3 + from pcapkit.protocols.application.dtls import DTLS + + # Base Classes + from pcapkit.protocols.application.http import HTTP + from pcapkit.protocols.application.ssl import SSL __all__ = [ 'APPTYPE', @@ -50,3 +59,21 @@ 'RARP', 'DRARP', 'SSL', 'SSLv2', 'SSLv3', 'TLS', 'DTLS', ] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Application': ('application', 'Application'), + 'FTP': ('ftp', 'FTP'), + 'FTP_DATA': ('ftp_data', 'FTP_DATA'), + 'HTTP': ('http', 'HTTP'), + 'HTTPv1': ('httpv1', 'HTTPv1'), + 'HTTPv2': ('httpv2', 'HTTPv2'), + 'NGAP': ('ngap', 'NGAP'), + 'OSPF': ('ospf', 'OSPF'), + 'RARP': ('rarp', 'RARP'), + 'DRARP': ('drarp', 'DRARP'), + 'SSL': ('ssl', 'SSL'), + 'SSLv2': ('sslv2', 'SSLv2'), + 'SSLv3': ('sslv3', 'SSLv3'), + 'TLS': ('tls', 'TLS'), + 'DTLS': ('dtls', 'DTLS'), +}) diff --git a/pcapkit/protocols/internet/__init__.py b/pcapkit/protocols/internet/__init__.py index eacc2c807c..a101eeeaaa 100644 --- a/pcapkit/protocols/internet/__init__.py +++ b/pcapkit/protocols/internet/__init__.py @@ -8,41 +8,50 @@ :mod:`pcapkit.protocols.internet` is the collection of all protocols in the internet layer, with detailed implementation and methods. +Each protocol class, and the base class +:class:`~pcapkit.protocols.internet.internet.Internet`, is imported the first time +it is reached rather than with this package, through +:func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ # TODO: Implement ECN, Shim6. -# Base Class for Internet Layer -from pcapkit.protocols.internet.internet import Internet - -# Utility Classes for Protocols -from pcapkit.protocols.internet.ah.protocol import AH -from pcapkit.protocols.internet.esp.protocol import ESP -from pcapkit.protocols.internet.icmpv4.protocol import ICMPv4 -from pcapkit.protocols.internet.icmpv6.protocol import ICMPv6 -from pcapkit.protocols.internet.igmpv1.protocol import IGMPv1 -from pcapkit.protocols.internet.igmpv2.protocol import IGMPv2 -from pcapkit.protocols.internet.igmpv3.protocol import IGMPv3 -from pcapkit.protocols.internet.ipv4.protocol import IPv4 -from pcapkit.protocols.internet.ipv6.protocol import IPv6 -from pcapkit.protocols.internet.ipx.protocol import IPX - -# IPv6 Extension Headers -from pcapkit.protocols.internet.hip.protocol import HIP -from pcapkit.protocols.internet.hopopt.protocol import HOPOPT -from pcapkit.protocols.internet.ipv6_frag.protocol import IPv6_Frag -from pcapkit.protocols.internet.ipv6_ext.protocol import IPv6_Ext -from pcapkit.protocols.internet.ipv6_opts.protocol import IPv6_Opts -from pcapkit.protocols.internet.ipv6_route.protocol import IPv6_Route -from pcapkit.protocols.internet.mh.protocol import MH +from typing import TYPE_CHECKING # Ethertype IEEE 802 Numbers from pcapkit.const.reg.ethertype import EtherType as ETHERTYPE +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + # Base Class for Internet Layer + from pcapkit.protocols.internet.internet import Internet + + # Utility Classes for Protocols + from pcapkit.protocols.internet.ah import AH + from pcapkit.protocols.internet.esp import ESP + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + from pcapkit.protocols.internet.igmpv1 import IGMPv1 + from pcapkit.protocols.internet.igmpv2 import IGMPv2 + from pcapkit.protocols.internet.igmpv3 import IGMPv3 + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.internet.ipx import IPX -# Base Classes -from pcapkit.protocols.internet.icmp.protocol import ICMP -from pcapkit.protocols.internet.igmp.protocol import IGMP -from pcapkit.protocols.internet.ip.protocol import IP -from pcapkit.protocols.internet.ipsec.protocol import IPsec + # IPv6 Extension Headers + from pcapkit.protocols.internet.hip import HIP + from pcapkit.protocols.internet.hopopt import HOPOPT + from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag + from pcapkit.protocols.internet.ipv6_ext import IPv6_Ext + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + from pcapkit.protocols.internet.mh import MH + + # Base Classes + from pcapkit.protocols.internet.icmp import ICMP + from pcapkit.protocols.internet.igmp import IGMP + from pcapkit.protocols.internet.ip import IP + from pcapkit.protocols.internet.ipsec import IPsec __all__ = [ 'ETHERTYPE', # Protocol Numbers @@ -52,3 +61,28 @@ 'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_Ext', 'IPv6_Opts', 'IPv6_Route', 'MH', # IPv6 Extension Header ] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Internet': ('internet', 'Internet'), + 'AH': ('ah', 'AH'), + 'ESP': ('esp', 'ESP'), + 'ICMP': ('icmp', 'ICMP'), + 'ICMPv4': ('icmpv4', 'ICMPv4'), + 'ICMPv6': ('icmpv6', 'ICMPv6'), + 'IGMP': ('igmp', 'IGMP'), + 'IGMPv1': ('igmpv1', 'IGMPv1'), + 'IGMPv2': ('igmpv2', 'IGMPv2'), + 'IGMPv3': ('igmpv3', 'IGMPv3'), + 'IP': ('ip', 'IP'), + 'IPsec': ('ipsec', 'IPsec'), + 'IPv4': ('ipv4', 'IPv4'), + 'IPv6': ('ipv6', 'IPv6'), + 'IPX': ('ipx', 'IPX'), + 'HIP': ('hip', 'HIP'), + 'HOPOPT': ('hopopt', 'HOPOPT'), + 'IPv6_Frag': ('ipv6_frag', 'IPv6_Frag'), + 'IPv6_Ext': ('ipv6_ext', 'IPv6_Ext'), + 'IPv6_Opts': ('ipv6_opts', 'IPv6_Opts'), + 'IPv6_Route': ('ipv6_route', 'IPv6_Route'), + 'MH': ('mh', 'MH'), +}) diff --git a/pcapkit/protocols/link/__init__.py b/pcapkit/protocols/link/__init__.py index 3c37634d12..1f45d8118c 100644 --- a/pcapkit/protocols/link/__init__.py +++ b/pcapkit/protocols/link/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pylint: disable=unused-import,unused-wildcard-import,fixme +# pylint: disable=unused-import,fixme """Link Layer Protocols ========================== @@ -8,39 +8,47 @@ :mod:`pcapkit.protocols.link` is a collection of all protocols in the link layer, with detailed implementation and methods. +Each protocol class, and the base class :class:`~pcapkit.protocols.link.link.Link`, +is imported the first time it is reached rather than with this package, through +:func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ # TODO: Implement DSL, EAPOL, FDDI, ISDN, NDP, PPP. -# Base Class for Link Layer -from pcapkit.protocols.link.link import Link - -# Link Layer Protocols -from pcapkit.protocols.link.arp.protocol import ARP -from pcapkit.protocols.link.inarp.protocol import InARP -from pcapkit.protocols.link.ethernet.protocol import Ethernet -from pcapkit.protocols.link.frad.protocol import FRAD -from pcapkit.protocols.link.gre.protocol import GRE -from pcapkit.protocols.link.ieee802_11.protocol import IEEE802_11 -from pcapkit.protocols.link.loopback.protocol import Loopback -from pcapkit.protocols.link.netlink.protocol import Netlink -from pcapkit.protocols.link.radiotap.protocol import Radiotap - -# Linux "Cooked" Capture Encapsulation Family -from pcapkit.protocols.link.linux_sll.protocol import LinuxSLL -from pcapkit.protocols.link.linux_sll1.protocol import LinuxSLL1 -from pcapkit.protocols.link.linux_sll2.protocol import LinuxSLL2 - -# VLAN Tag Family -from pcapkit.protocols.link.vlan.protocol import VLAN -from pcapkit.protocols.link.c_tag.protocol import C_Tag -from pcapkit.protocols.link.s_tag.protocol import S_Tag - -# L2TP Family -from pcapkit.protocols.link.l2tp.protocol import L2TP -from pcapkit.protocols.link.l2tpv2.protocol import L2TPv2 +from typing import TYPE_CHECKING # Link-Layer Header Type Values from pcapkit.const.reg.linktype import LinkType as LINKTYPE +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + # Base Class for Link Layer + from pcapkit.protocols.link.link import Link + + # Link Layer Protocols + from pcapkit.protocols.link.arp import ARP + from pcapkit.protocols.link.inarp import InARP + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.protocols.link.frad import FRAD + from pcapkit.protocols.link.gre import GRE + from pcapkit.protocols.link.ieee802_11 import IEEE802_11 + from pcapkit.protocols.link.loopback import Loopback + from pcapkit.protocols.link.netlink import Netlink + from pcapkit.protocols.link.radiotap import Radiotap + + # Linux "Cooked" Capture Encapsulation Family + from pcapkit.protocols.link.linux_sll import LinuxSLL + from pcapkit.protocols.link.linux_sll1 import LinuxSLL1 + from pcapkit.protocols.link.linux_sll2 import LinuxSLL2 + + # VLAN Tag Family + from pcapkit.protocols.link.vlan import VLAN + from pcapkit.protocols.link.c_tag import C_Tag + from pcapkit.protocols.link.s_tag import S_Tag + + # L2TP Family + from pcapkit.protocols.link.l2tp import L2TP + from pcapkit.protocols.link.l2tpv2 import L2TPv2 __all__ = [ # Protocol Numbers @@ -50,3 +58,24 @@ 'ARP', 'C_Tag', 'Ethernet', 'FRAD', 'GRE', 'IEEE802_11', 'InARP', 'L2TP', 'L2TPv2', 'LinuxSLL', 'LinuxSLL1', 'LinuxSLL2', 'Loopback', 'Netlink', 'Radiotap', 'S_Tag', 'VLAN', ] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Link': ('link', 'Link'), + 'ARP': ('arp', 'ARP'), + 'C_Tag': ('c_tag', 'C_Tag'), + 'Ethernet': ('ethernet', 'Ethernet'), + 'FRAD': ('frad', 'FRAD'), + 'GRE': ('gre', 'GRE'), + 'IEEE802_11': ('ieee802_11', 'IEEE802_11'), + 'InARP': ('inarp', 'InARP'), + 'L2TP': ('l2tp', 'L2TP'), + 'L2TPv2': ('l2tpv2', 'L2TPv2'), + 'LinuxSLL': ('linux_sll', 'LinuxSLL'), + 'LinuxSLL1': ('linux_sll1', 'LinuxSLL1'), + 'LinuxSLL2': ('linux_sll2', 'LinuxSLL2'), + 'Loopback': ('loopback', 'Loopback'), + 'Netlink': ('netlink', 'Netlink'), + 'Radiotap': ('radiotap', 'Radiotap'), + 'S_Tag': ('s_tag', 'S_Tag'), + 'VLAN': ('vlan', 'VLAN'), +}) diff --git a/pcapkit/protocols/misc/__init__.py b/pcapkit/protocols/misc/__init__.py index 53dd70f996..365b068241 100644 --- a/pcapkit/protocols/misc/__init__.py +++ b/pcapkit/protocols/misc/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pylint: disable=unused-wildcard-import +# pylint: disable=unused-import """Auxiliary Protocols ========================= @@ -11,20 +11,27 @@ indication of empty payload, the PCAP and PCAP-NG file format classes, and the checksum pseudo-headers in :mod:`pcapkit.protocols.misc.checksum`. +Each class is imported the first time it is reached rather than with this +package, through :func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ +from typing import TYPE_CHECKING + +from pcapkit.utilities.importlib import lazy_exports -# PCAP Headers -from pcapkit.protocols.misc.pcap import * +if TYPE_CHECKING: + # PCAP Headers + from pcapkit.protocols.misc.pcap import Frame, Header -# PCAPNG Format -from pcapkit.protocols.misc.pcapng.protocol import * + # PCAPNG Format + from pcapkit.protocols.misc.pcapng import PCAPNG -# Miscellaneous Classes for Protocols -from pcapkit.protocols.misc.raw.protocol import Raw -from pcapkit.protocols.misc.no_payload.protocol import NoPayload + # Miscellaneous Classes for Protocols + from pcapkit.protocols.misc.raw import Raw + from pcapkit.protocols.misc.no_payload import NoPayload -# Checksum Pseudo-Headers -from pcapkit.protocols.misc.checksum import IP_Pseudo + # Checksum Pseudo-Headers + from pcapkit.protocols.misc.checksum import IP_Pseudo __all__ = [ # PCAP Headers @@ -42,3 +49,12 @@ # Checksum Pseudo-Headers 'IP_Pseudo', ] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Header': ('pcap', 'Header'), + 'Frame': ('pcap', 'Frame'), + 'PCAPNG': ('pcapng', 'PCAPNG'), + 'NoPayload': ('no_payload', 'NoPayload'), + 'Raw': ('raw', 'Raw'), + 'IP_Pseudo': ('checksum', 'IP_Pseudo'), +}) diff --git a/pcapkit/protocols/misc/checksum/__init__.py b/pcapkit/protocols/misc/checksum/__init__.py index a768681c09..5dffc0701e 100644 --- a/pcapkit/protocols/misc/checksum/__init__.py +++ b/pcapkit/protocols/misc/checksum/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# pylint: disable=unused-import """Checksum Pseudo-Headers ============================= @@ -9,7 +10,19 @@ :class:`~pcapkit.protocols.misc.checksum.ip_pseudo.protocol.IP_Pseudo`. The checksum itself is in :mod:`pcapkit.corekit.checksum`. +The pseudo-header is imported the first time it is reached rather than with this +package, through :func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ -from pcapkit.protocols.misc.checksum.ip_pseudo.protocol import IP_Pseudo +from typing import TYPE_CHECKING + +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + from pcapkit.protocols.misc.checksum.ip_pseudo import IP_Pseudo __all__ = ['IP_Pseudo'] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'IP_Pseudo': ('ip_pseudo', 'IP_Pseudo'), +}) diff --git a/pcapkit/protocols/misc/pcap/__init__.py b/pcapkit/protocols/misc/pcap/__init__.py index 67717312ac..ddc86ed39c 100644 --- a/pcapkit/protocols/misc/pcap/__init__.py +++ b/pcapkit/protocols/misc/pcap/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pylint: disable=unused-import,unused-wildcard-import,fixme +# pylint: disable=unused-import """PCAP File Headers ======================= @@ -10,8 +10,21 @@ (:class:`~pcapkit.protocols.misc.pcap.header.protocol.Header`) and frame header (:class:`~pcapkit.protocols.misc.pcap.frame.protocol.Frame`). +Each is imported the first time it is reached rather than with this package, +through :func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ -from pcapkit.protocols.misc.pcap.frame.protocol import Frame -from pcapkit.protocols.misc.pcap.header.protocol import Header +from typing import TYPE_CHECKING + +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header __all__ = ['Frame', 'Header'] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Frame': ('frame', 'Frame'), + 'Header': ('header', 'Header'), +}) diff --git a/pcapkit/protocols/registry.py b/pcapkit/protocols/registry.py new file mode 100644 index 0000000000..c89bba712e --- /dev/null +++ b/pcapkit/protocols/registry.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Protocol Registry +======================= + +.. module:: pcapkit.protocols.registry + +:mod:`pcapkit.protocols.registry` builds :data:`pcapkit.protocols.__proto__`, the +registry of protocol classes by name, which :mod:`pcapkit.protocols` exports from +here. It names every protocol class that package exports, so importing it imports +them all; the package puts that off until the registry is first reached +(:issue:`2730`). + +.. versionadded:: 1.6.0 + +""" +from typing import TYPE_CHECKING + +from pcapkit import protocols +from pcapkit.protocols.protocol import ProtocolBase + +if TYPE_CHECKING: + from typing import Type + +__all__ = [] # type: list[str] + +#: dict[str, ~typing.Type[~pcapkit.protocols.protocol.ProtocolBase]]: Protocol +#: registry, bound before it is filled so that a read while it is being built +#: finds this object. +__proto__ = {} # type: dict[str, Type[ProtocolBase]] +for _name in protocols.__all__: + _value = getattr(protocols, _name) + if isinstance(_value, type) and issubclass(_value, ProtocolBase): + __proto__[_name.upper()] = _value diff --git a/pcapkit/protocols/transport/__init__.py b/pcapkit/protocols/transport/__init__.py index 8047a0c866..55d1292dd4 100644 --- a/pcapkit/protocols/transport/__init__.py +++ b/pcapkit/protocols/transport/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pylint: disable=unused-import,unused-wildcard-import,fixme +# pylint: disable=unused-import,fixme """Transport Layer Protocols =============================== @@ -8,22 +8,39 @@ :mod:`pcapkit.protocols.transport` is a collection of all protocols in the transport layer, with detailed implementation and methods. +Each protocol class, and the base class +:class:`~pcapkit.protocols.transport.transport.Transport`, is imported the first +time it is reached rather than with this package, through +:func:`~pcapkit.utilities.importlib.lazy_exports` (:issue:`2730`). + """ # TODO: Implement RSVP. -# Base Class for Transport Layer -from pcapkit.protocols.transport.transport import Transport - -# Transport Layer Protocols -from pcapkit.protocols.transport.dccp.protocol import DCCP -from pcapkit.protocols.transport.sctp.protocol import SCTP -from pcapkit.protocols.transport.tcp.protocol import TCP -from pcapkit.protocols.transport.udp.protocol import UDP +from typing import TYPE_CHECKING # Transport Layer Protocol Numbers from pcapkit.const.reg.transtype import TransType as TRANSTYPE +from pcapkit.utilities.importlib import lazy_exports + +if TYPE_CHECKING: + # Base Class for Transport Layer + from pcapkit.protocols.transport.transport import Transport + + # Transport Layer Protocols + from pcapkit.protocols.transport.dccp import DCCP + from pcapkit.protocols.transport.sctp import SCTP + from pcapkit.protocols.transport.tcp import TCP + from pcapkit.protocols.transport.udp import UDP __all__ = [ 'TRANSTYPE', # Protocol Numbers 'DCCP', 'TCP', 'UDP', 'SCTP', # Transport Layer Protocols ] + +__getattr__, __dir__ = lazy_exports(__name__, globals(), { + 'Transport': ('transport', 'Transport'), + 'DCCP': ('dccp', 'DCCP'), + 'SCTP': ('sctp', 'SCTP'), + 'TCP': ('tcp', 'TCP'), + 'UDP': ('udp', 'UDP'), +}) diff --git a/tests/conftest.py b/tests/conftest.py index c4f5fe7393..db73157af9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,15 +77,17 @@ _pinned_snapshot = None # type: Optional[dict[str, types.ModuleType]] #: What :func:`pytest_sessionstart` imports before the pin: the package, plus every -#: module of :mod:`pcapkit.const`, whose packages import their enumerations on first -#: use rather than with themselves -- the app-type registries for GitHub issue -#: #1538, and every enumeration for #2729. Read off the tree rather than listed, -#: so a module the crawlers add is warmed the day it lands. See that hook for why -#: they cannot be left out. +#: module of :mod:`pcapkit.const` and :mod:`pcapkit.protocols`, whose packages import +#: what they export on first use rather than with themselves -- the app-type +#: registries for GitHub issue #1538, every enumeration for #2729, and every protocol +#: for #2730. Read off the tree rather than listed, so a module added is warmed the +#: day it lands; the placeholder sources under ``NotImplemented/`` are no module +#: anything imports. See that hook for why they cannot be left out. WARM_MODULES = ('pcapkit', *sorted( '.'.join(path.relative_to(ROOT).with_suffix('').parts[:-1] if path.name == '__init__.py' else path.relative_to(ROOT).with_suffix('').parts) - for path in (ROOT / 'pcapkit' / 'const').rglob('*.py'))) + for package in ('const', 'protocols') for path in (ROOT / 'pcapkit' / package).rglob('*.py') + if 'NotImplemented' not in path.parts)) def _pin_module_snapshot() -> None: @@ -179,7 +181,9 @@ def pytest_sessionstart(session: 'pytest.Session') -> 'None': :mod:`tests.project.test_conventions_doc_claims` counted both generations, and its registry count came out wrong on the 3.10-3.13 unit legs of PR #1560. Every :mod:`pcapkit.const` package binds what it imports on first use the same - way (#2729), so every module of it is warmed. + way (#2729), so every module of it is warmed, and so does every package above a + protocol's (#2730): a protocol class bound there would otherwise outlive its + module's entry, and fail to :mod:`pickle` against the next generation of it. Failure here is not an error. A checkout without the runtime dependencies installed cannot import the package at all -- which is a supported way to run diff --git a/tests/const/test_const_apptype_lazy_1538_unit.py b/tests/const/test_const_apptype_lazy_1538_unit.py index 54efcb09be..7a69dc3ca9 100644 --- a/tests/const/test_const_apptype_lazy_1538_unit.py +++ b/tests/const/test_const_apptype_lazy_1538_unit.py @@ -16,9 +16,8 @@ The subprocess checks run in a fresh interpreter, since the question is what a cold import leaves in :data:`sys.modules` and this process has long since imported everything. :data:`PRELUDE` imports :mod:`pcapkit`, whose protocol -modules import the const modules they name at module level -- -:mod:`pcapkit.protocols.application.ngap.protocol` imports SCTP's registry, for one, which -#2730 is to move -- so a check of the const packages themselves runs on :data:`BARE`, which +modules import the const modules they name at module level, so a check of the +const packages themselves runs on :data:`BARE`, which binds :mod:`pcapkit` as a bare package and so runs none of :file:`pcapkit/__init__.py`. Each check fails on an eager tree at its first ``loaded()`` or ``enumerations()``; what follows it pins that nothing a caller @@ -111,7 +110,7 @@ def emit(**kwargs): ''' + inspect.getsource(import_system_names) -#: A child that imports :mod:`pcapkit`, protocols and all. +#: A child that imports :mod:`pcapkit`, and the protocol modules it imports with itself. PRELUDE = HELPERS + f''' import pcapkit assert pcapkit.__file__.startswith({str(ROOT)!r}), pcapkit.__file__ @@ -154,10 +153,10 @@ class LazyAppTypeImportTests(unittest.TestCase): """What a cold import loads, and what loads each registry afterwards.""" def test_import_pcapkit_leaves_the_registries_unimported(self) -> 'None': - # NOTE: SCTP's is left out: pcapkit.protocols.application.ngap.protocol imports it - # at module level, which LazyConstImportTests accounts for (#2730). + # NOTE: SCTP's too, since pcapkit.protocols.application.ngap.protocol, which + # imports it at module level, loads on first use (#2730). out = run_child(''' - emit(loaded=loaded('tcp', 'udp', 'dccp'), + emit(loaded=loaded(), eager='pcapkit.const.reg.apptype.apptype' in sys.modules) ''') self.assertEqual(out['loaded'], []) @@ -391,7 +390,8 @@ def test_import_pcapkit_loads_only_what_other_modules_import(self) -> 'None': const packages imports at module level, or one :data:`EAGER` names. Those are the protocol, schema and data modules naming the const module - they read; moving them is GitHub issue #2730. + they read, of the protocols :mod:`pcapkit` imports with itself; the + protocol packages load the rest on first use (GitHub issue #2730). """ out = run_child(''' @@ -583,9 +583,7 @@ def setUp(self) -> 'None': def test_hooks(self) -> 'None': import pcapkit.const.reg.apptype as pkg - # NOTE: TCP, UDP and DCCP, since pcapkit.protocols.application.ngap.protocol - # imports SCTP's submodule at module level (#2730). - for name in ('TCP', 'UDP', 'DCCP'): + for name in ('TCP', 'UDP', 'SCTP', 'DCCP'): self.assertNotIn(name, vars(pkg)) self.assertNotIn(f'pcapkit.const.reg.apptype.{name.lower()}', sys.modules) getattr_ = vars(pkg)['__getattr__'] diff --git a/tests/foundation/registry/test_protocols.py b/tests/foundation/registry/test_protocols.py index ce985115d3..e827b4f2d9 100644 --- a/tests/foundation/registry/test_protocols.py +++ b/tests/foundation/registry/test_protocols.py @@ -103,12 +103,12 @@ def test_register_protocol_warns_when_a_colliding_name_overwrites(self) -> None: """ from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.application.http import protocol as http from pcapkit.protocols.application.application import Application from pcapkit.utilities.warnings import RegistryWarning - self._guard_registry(reg_protocol.protocol_registry, 'HTTP') + self._guard_registry(protocol_registry, 'HTTP') # Defining a Protocol subclass registers it, so this displaces the base. with warnings.catch_warnings(): @@ -120,7 +120,7 @@ def test_register_protocol_warns_when_a_colliding_name_overwrites(self) -> None: self.assertEqual(len(set(colliding)), 2) self.assertEqual({cls.__name__.upper() for cls in colliding}, {'HTTP'}) - reg_protocol.protocol_registry['HTTP'] = http.HTTP + protocol_registry['HTTP'] = http.HTTP # Both directions, so the test is not pinned to one registration order. for incumbent, replacement in ((http.HTTP, custom), (custom, http.HTTP)): @@ -132,7 +132,7 @@ def test_register_protocol_warns_when_a_colliding_name_overwrites(self) -> None: # The overwrite still happens. The warning reports the # collision, it does not resolve it -- resolving it is #514's # business. - self.assertIs(reg_protocol.protocol_registry['HTTP'], replacement) + self.assertIs(protocol_registry['HTTP'], replacement) messages = self._registry_warnings(caught, RegistryWarning) self.assertEqual(len(messages), 1) @@ -160,12 +160,12 @@ def test_a_protocol_subclass_shadowing_a_builtin_name_warns(self) -> None: """ from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.protocol import Protocol from pcapkit.utilities.warnings import RegistryWarning - incumbent = reg_protocol.protocol_registry['HTTP'] - self._guard_registry(reg_protocol.protocol_registry, 'HTTP') + incumbent = protocol_registry['HTTP'] + self._guard_registry(protocol_registry, 'HTTP') with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') @@ -173,8 +173,8 @@ def test_a_protocol_subclass_shadowing_a_builtin_name_warns(self) -> None: class HTTP(Protocol): """A user protocol that happens to be named like the built-in.""" - self.assertIs(reg_protocol.protocol_registry['HTTP'], HTTP) - self.assertIsNot(reg_protocol.protocol_registry['HTTP'], incumbent) + self.assertIs(protocol_registry['HTTP'], HTTP) + self.assertIsNot(protocol_registry['HTTP'], incumbent) messages = self._registry_warnings(caught, RegistryWarning) self.assertEqual(len(messages), 1) @@ -198,13 +198,13 @@ def test_register_protocol_stays_quiet_when_nothing_is_displaced(self) -> None: """ from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.transport.tcp.protocol import TCP from pcapkit.protocols.transport.udp.protocol import UDP from pcapkit.utilities.warnings import RegistryWarning UnitProtocol = self._unit_protocol() - self._guard_registry(reg_protocol.protocol_registry, 'UNITPROTOCOL') + self._guard_registry(protocol_registry, 'UNITPROTOCOL') for label in ('fresh-key', 'identical-repeat'): with self.subTest(case=label): @@ -212,7 +212,7 @@ def test_register_protocol_stays_quiet_when_nothing_is_displaced(self) -> None: warnings.simplefilter('always') registry.register_protocol(UnitProtocol) - self.assertIs(reg_protocol.protocol_registry['UNITPROTOCOL'], UnitProtocol) + self.assertIs(protocol_registry['UNITPROTOCOL'], UnitProtocol) self.assertEqual(self._registry_warnings(caught, RegistryWarning), []) # ... and the real call shape, through two wrapper registrars. Each ends @@ -382,7 +382,7 @@ def test_register_protocol_disambiguates_classes_sharing_a_repr(self) -> None: """ from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.utilities.warnings import RegistryWarning first = self._unit_protocol() @@ -392,8 +392,8 @@ def test_register_protocol_disambiguates_classes_sharing_a_repr(self) -> None: self.assertIsNot(first, second) self.assertEqual(repr(first), repr(second)) - self._guard_registry(reg_protocol.protocol_registry, 'UNITPROTOCOL') - reg_protocol.protocol_registry['UNITPROTOCOL'] = first + self._guard_registry(protocol_registry, 'UNITPROTOCOL') + protocol_registry['UNITPROTOCOL'] = first with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') @@ -401,7 +401,7 @@ def test_register_protocol_disambiguates_classes_sharing_a_repr(self) -> None: # The guard's firing condition: the overwrite happens and is # reported exactly once. - self.assertIs(reg_protocol.protocol_registry['UNITPROTOCOL'], second) + self.assertIs(protocol_registry['UNITPROTOCOL'], second) messages = self._registry_warnings(caught, RegistryWarning) self.assertEqual(len(messages), 1) @@ -419,13 +419,13 @@ def test_register_protocol_disambiguates_classes_sharing_a_repr(self) -> None: def test_register_protocol_validates_and_updates_registry(self) -> None: from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.utilities.exceptions import RegistryError UnitProtocol = self._unit_protocol() - self._guard_registry(reg_protocol.protocol_registry, 'UNITPROTOCOL') + self._guard_registry(protocol_registry, 'UNITPROTOCOL') registry.register_protocol(UnitProtocol) - self.assertIs(reg_protocol.protocol_registry['UNITPROTOCOL'], UnitProtocol) + self.assertIs(protocol_registry['UNITPROTOCOL'], UnitProtocol) with self.assertRaises(RegistryError): registry.register_protocol(object) # type: ignore[arg-type] @@ -474,10 +474,11 @@ def test_top_level_link_internet_and_transport_protocol_wrappers(self) -> None: from pcapkit.foundation.registry.protocols import misc as reg_misc from pcapkit.foundation.registry.protocols import protocol as reg_protocol from pcapkit.foundation.registry.protocols import transport as reg_transport + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.utilities.exceptions import ProtocolError, RegistryError UnitProtocol = self._unit_protocol() - self._guard_registry(reg_protocol.protocol_registry, 'UNITPROTOCOL') + self._guard_registry(protocol_registry, 'UNITPROTOCOL') raw_module = ('pcapkit.protocols.misc.raw.protocol', 'Raw') class_routes = [ diff --git a/tests/project/test_conventions_doc_claims.py b/tests/project/test_conventions_doc_claims.py index 22a954f0bd..c7e17bb47d 100644 --- a/tests/project/test_conventions_doc_claims.py +++ b/tests/project/test_conventions_doc_claims.py @@ -527,7 +527,12 @@ class ExtensionHeaderClassificationTests(unittest.TestCase): """The page's bases-per-header table against the declarations themselves.""" def setUp(self) -> 'None': - import pcapkit.protocols.internet # noqa: F401 # populates __subclasses__ + import pcapkit.protocols.internet as internet + + # NOTE: the package imports each protocol on first use (#2730), so reach + # every one, which populates __subclasses__. + for name in internet.__all__: + getattr(internet, name) from pcapkit.protocols.internet.internet import Internet from pcapkit.protocols.internet.ipv6_ext.protocol import IPv6_Ext diff --git a/tests/project/test_public_api.py b/tests/project/test_public_api.py index 0e511b7f1c..284d3394ff 100644 --- a/tests/project/test_public_api.py +++ b/tests/project/test_public_api.py @@ -186,7 +186,7 @@ class exists at ``pcapkit/const/mh/handover_initiate_status.py`` and is used by #: build their manager classes. ``TYPE_CHECKING`` is :mod:`typing`'s. Re-exporting #: any of them would claim ownership of a name this package does not own. #: -#: **Import-time machinery, and one leaked loop variable.** +#: **Import-time machinery.** #: :mod:`pcapkit` imports ``BaseError``, ``DEVMODE`` and ``DevModeWarning`` to wire #: up the ``tbtrim`` excepthook, not to publish them -- the top-level surface is #: interface functions, macros and protocols, and those names are documented under @@ -196,11 +196,7 @@ class exists at ``pcapkit/const/mh/handover_initiate_status.py`` and is used by #: fix for one. :mod:`pcapkit.vendor` likewise imports ``ModuleNotFoundError``, #: ``stacklevel``, ``VendorWarning`` and ``warn`` only to warn when ``requests`` #: or ``bs4`` is missing; they are :mod:`pcapkit.utilities`' names, and listing -#: them would hand them to that same crawler loop. ``pcapkit.protocols.name`` is -#: the loop variable of the ``for name in __all__`` that builds ``__proto__``, -#: left behind in the namespace; renaming it ``_name`` would express that better -#: than this entry does, but that is a change to a module body rather than to an -#: export list. +#: them would hand them to that same crawler loop. DELIBERATE_NON_EXPORTS = { 'pcapkit': ('BaseError', 'DEVMODE', 'DevModeWarning', 'ROOT'), 'pcapkit.corekit.fields': ('Field',), @@ -209,7 +205,7 @@ class exists at ``pcapkit/const/mh/handover_initiate_status.py`` and is used by 'TYPE_CHECKING', 'info_final'), 'pcapkit.foundation.traceflow': ('Info', 'TYPE_CHECKING', 'TraceFlow', 'info_final'), - 'pcapkit.protocols': ('Protocol', 'TYPE_CHECKING', 'name'), + 'pcapkit.protocols': ('Protocol', 'TYPE_CHECKING'), 'pcapkit.protocols.application': ('Application',), 'pcapkit.protocols.internet': ('Internet',), 'pcapkit.protocols.link': ('Link',), diff --git a/tests/protocols/application/test_http_member_names_2529_unit.py b/tests/protocols/application/test_http_member_names_2529_unit.py index 22b6ffdc03..99689285d5 100644 --- a/tests/protocols/application/test_http_member_names_2529_unit.py +++ b/tests/protocols/application/test_http_member_names_2529_unit.py @@ -125,11 +125,11 @@ def test_a_member_built_directly_reports_its_own_name(self) -> None: def test_registering_a_member_keeps_the_base_under_http(self) -> None: """Each member's own key holds it, and ``'HTTP'`` keeps the base.""" from pcapkit.foundation.registry import protocols as registry - from pcapkit.foundation.registry.protocols import protocol as reg_protocol + from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.application.http.protocol import HTTP from pcapkit.utilities.warnings import RegistryWarning - names = reg_protocol.protocol_registry + names = protocol_registry before = dict(names) def restore() -> None: diff --git a/tests/protocols/internet/test_ipv6_ext_unit.py b/tests/protocols/internet/test_ipv6_ext_unit.py index bdac90e649..b3b48f71c9 100644 --- a/tests/protocols/internet/test_ipv6_ext_unit.py +++ b/tests/protocols/internet/test_ipv6_ext_unit.py @@ -412,9 +412,12 @@ def test_id_composes_every_direct_concrete_subclass(self) -> None: """Derived from the live hierarchy, so a new member cannot drift.""" import inspect - import pcapkit.protocols # noqa: F401 # pylint: disable=unused-import + import pcapkit.protocols.internet as internet from pcapkit.protocols.internet.ipv6_ext.protocol import IPv6_Ext + # NOTE: the package imports each protocol on first use (#2730). + for name in internet.__all__: + getattr(internet, name) members = [klass for klass in IPv6_Ext.__subclasses__() if not inspect.isabstract(klass)] self.assertCountEqual(IPv6_Ext.id(), [klass.__name__ for klass in members]) for klass in members: @@ -703,9 +706,13 @@ def _subclasses(base: type) -> 'list[type]': def _family(self) -> 'list[type]': """The discovered family, with every member module imported first.""" - import pcapkit.protocols.internet # noqa: F401 # populates __subclasses__ + import pcapkit.protocols.internet as internet from pcapkit.protocols.internet.ipv6_ext.protocol import IPv6_Ext + # NOTE: the package imports each protocol on first use (#2730), so + # reach every one, which populates __subclasses__. + for name in internet.__all__: + getattr(internet, name) return self._subclasses(IPv6_Ext) @staticmethod diff --git a/tests/protocols/test_protocol_code_registration_unit.py b/tests/protocols/test_protocol_code_registration_unit.py index 26e1100e43..4ccd24bf22 100644 --- a/tests/protocols/test_protocol_code_registration_unit.py +++ b/tests/protocols/test_protocol_code_registration_unit.py @@ -663,9 +663,16 @@ def test_every_destination_kind_row_is_enforced(self) -> None: @staticmethod def _declaring_classes() -> 'set[type]': """Every ``ProtocolBase`` descendant declaring a ``__proto_key__`` of its own.""" - import pcapkit # noqa: F401 # pylint: disable=unused-import + import importlib + import pkgutil + + import pcapkit.protocols as package from pcapkit.protocols.protocol import ProtocolBase + # NOTE: the protocol packages import each protocol on first use (#2730). + for info in pkgutil.walk_packages(package.__path__, 'pcapkit.protocols.'): + if 'NotImplemented' not in info.name: + importlib.import_module(info.name) found = set() pending = [ProtocolBase] while pending: diff --git a/tests/protocols/test_protocol_packages_lazy_2730_unit.py b/tests/protocols/test_protocol_packages_lazy_2730_unit.py new file mode 100644 index 0000000000..ed40b7788c --- /dev/null +++ b/tests/protocols/test_protocol_packages_lazy_2730_unit.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +"""The packages above a protocol's package import it on first use (GitHub issue #2730). + +:mod:`pcapkit.protocols`, its layer packages, :mod:`pcapkit.protocols.misc.pcap` and +:mod:`pcapkit.protocols.misc.checksum` name each protocol class only under +:data:`~typing.TYPE_CHECKING` and import it through +:func:`pcapkit.utilities.importlib.lazy_exports`. :mod:`pcapkit` re-exports +:mod:`pcapkit.protocols` the same way, and :data:`pcapkit.protocols.__proto__` is +built by :mod:`pcapkit.protocols.registry` the first time it is reached. A protocol's +own package keeps its literal imports, which the #2534 layout tests pin. + +* :class:`LazyPackageSweepTests` holds each such package, read off the tree, to what + its eager import did: every export resolves to what its import names, ``import *`` + binds them all, :func:`dir` lists them and the layer's base class but none of the + helpers, and every protocol class pickles as itself. +* :class:`ColdImportTests` runs in a fresh interpreter, since this one has long since + imported everything. Each check fails on an eager tree at its first look at + :data:`sys.modules`, bar the Shim6 one, which guards a registration made as a + module is imported: :mod:`pcapkit.protocols.internet.ipv6_ext.protocol` writes + ``Internet.__proto__[TransType.Shim6]`` as it is imported, so that entry exists only + because ``import pcapkit`` imports that module, through HIP and MH, which + :mod:`pcapkit.foundation.registry.protocols.internet` imports. + +""" + +import ast +import importlib +import importlib.util +import json +import os +import pathlib +import pickle +import subprocess +import sys +import textwrap +import types +import unittest +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): exercises the import system and pickling, which differ between interpreters. +__tier__ = 'ALL' + +ROOT = pathlib.Path(__file__).resolve().parents[2] + +#: Every package under :mod:`pcapkit.protocols` that is not a protocol's own, read off the tree. +PACKAGES = sorted('.'.join(path.relative_to(ROOT).parent.parts) + for path in (ROOT / 'pcapkit' / 'protocols').rglob('__init__.py') + if path.parent == ROOT / 'pcapkit' / 'protocols' or not (path.parent / 'protocol.py').exists()) + +#: The exports a package imports with itself, and why: the framework modules every +#: protocol module imports anyway, and the protocol-number enumerations, which are +#: :mod:`pcapkit.const` modules rather than protocols. +EAGER = { + 'pcapkit.protocols': {'Data', 'data', 'Schema', 'schema'}, + 'pcapkit.protocols.application': {'APPTYPE'}, + 'pcapkit.protocols.internet': {'ETHERTYPE'}, + 'pcapkit.protocols.link': {'LINKTYPE'}, + 'pcapkit.protocols.transport': {'TRANSTYPE'}, +} + +#: The base class each package binds outside its ``__all__``, which ``dir()`` listed eagerly. +BASES = { + 'pcapkit.protocols': {'Protocol'}, + 'pcapkit.protocols.application': {'Application'}, + 'pcapkit.protocols.internet': {'Internet'}, + 'pcapkit.protocols.link': {'Link'}, + 'pcapkit.protocols.transport': {'Transport'}, +} + + +def init_path(package: 'str') -> 'pathlib.Path': + """The ``__init__.py`` of ``package``.""" + return ROOT.joinpath(*package.split('.'), '__init__.py') + + +def declared(package: 'str') -> 'dict[str, tuple[str, str, bool]]': + """What the ``__init__.py`` of ``package`` imports at module level, ``TYPE_CHECKING`` included. + + Each bound name maps to the module it is imported from, its name there, and + whether the import is under ``TYPE_CHECKING``; a star import maps each name + its module's ``__all__`` lists to that module and the same name. + + """ + names = {} # type: dict[str, tuple[str, str, bool]] + + def visit(body: 'list[ast.stmt]', typing_only: 'bool') -> 'None': + for node in body: + if isinstance(node, ast.If): + guarded = isinstance(node.test, ast.Name) and node.test.id == 'TYPE_CHECKING' + visit(node.body, typing_only or guarded) + visit(node.orelse, typing_only) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + for alias in node.names: + if alias.name == '*': + source = importlib.import_module(node.module) + names.update((name, (node.module, name, typing_only)) for name in source.__all__) + else: + names[alias.asname or alias.name] = (node.module, alias.name, typing_only) + + visit(ast.parse(init_path(package).read_text(encoding='utf-8')).body, False) + return names + + +def import_system_names(module: 'types.ModuleType') -> 'set[str]': + """The names importing ``module`` would leave in it, were its body empty.""" + fresh = importlib.util.module_from_spec(module.__spec__) # type: ignore[arg-type] + exec('', vars(fresh)) # pylint: disable=exec-used + return set(dir(fresh)) + + +class LazyPackageSweepTests(unittest.TestCase): + """Each package above a protocol's, as an eager import of it would leave it.""" + + def test_the_sweep_found_the_packages(self) -> 'None': + """An empty walk would let every loop below pass without checking a package.""" + for package in ('pcapkit.protocols', 'pcapkit.protocols.link', 'pcapkit.protocols.misc.pcap', + 'pcapkit.protocols.misc.checksum'): + self.assertIn(package, PACKAGES) + self.assertNotIn('pcapkit.protocols.link.vlan', PACKAGES) + + def test_every_name_in_all_resolves_to_what_its_import_names(self) -> 'None': + for package in PACKAGES: + module = importlib.import_module(package) + imports = declared(package) + names = [*module.__all__, *BASES.get(package, ())] + with self.subTest(package=package): + self.assertTrue(module.__all__) + self.assertEqual([name for name in names if name not in imports], [], + 'name each export in an import under TYPE_CHECKING') + eager = sorted(name for name in module.__all__ if not imports[name][2]) + self.assertEqual(eager, sorted(EAGER.get(package, ())), + 'import each protocol under TYPE_CHECKING alone, so that it loads on first use') + for name in names: + source, attribute, _ = imports[name] + self.assertIs(getattr(module, name), getattr(importlib.import_module(source), attribute), + f'{package}.{name}') + + def test_star_import_binds_every_name(self) -> 'None': + for package in [*PACKAGES, 'pcapkit']: + module = importlib.import_module(package) + namespace = {} # type: dict[str, Any] + exec(f'from {package} import *', namespace) # pylint: disable=exec-used + namespace.pop('__builtins__') + with self.subTest(package=package): + self.assertEqual(sorted(namespace), sorted(module.__all__)) + for name, value in namespace.items(): + self.assertIs(value, getattr(module, name)) + + def test_pcapkit_re_exports_the_protocols(self) -> 'None': + import pcapkit + import pcapkit.protocols + + for name in pcapkit.protocols.__all__: + with self.subTest(name=name): + self.assertIs(getattr(pcapkit, name), getattr(pcapkit.protocols, name)) + + def test_dir_lists_what_an_eager_import_bound(self) -> 'None': + # NOTE: the eager import bound every submodule its imports name, and any + # other one already imported; the first are listed whether imported or + # not, as they are not on a cold import. ``registry`` is where + # ``__proto__`` comes from, which ``dir()`` lists as the eager import did. + for package in PACKAGES: + module = importlib.import_module(package) + named = {source[len(package) + 1:].split('.')[0] for (source, _, _) in declared(package).values() + if source.startswith(f'{package}.')} + submodules = named | {name for (name, value) in vars(module).items() + if isinstance(value, types.ModuleType) and value.__name__ == f'{package}.{name}'} + extra = {'__proto__', 'registry'} if package == 'pcapkit.protocols' else set() + with self.subTest(package=package): + self.assertTrue(named) + self.assertEqual(dir(module), sorted({*import_system_names(module), '__all__', *module.__all__, + *BASES.get(package, ()), *submodules, *extra})) + for helper in ('TYPE_CHECKING', 'lazy_exports', '__getattr__', '__dir__'): + self.assertIn(helper, vars(module)) + self.assertNotIn(helper, dir(module)) + + def test_every_protocol_class_pickles_as_itself(self) -> 'None': + import pcapkit.protocols + from pcapkit.protocols.protocol import ProtocolBase + + classes = [getattr(pcapkit.protocols, name) for name in pcapkit.protocols.__all__] + classes = [value for value in classes if isinstance(value, type) and issubclass(value, ProtocolBase)] + self.assertIn(pcapkit.protocols.IPv4, classes) + for value in classes: + with self.subTest(protocol=value.__name__): + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertIs(pickle.loads(pickle.dumps(value, protocol)), value) + + +#: What every child runs first: the helpers, then a cold ``import pcapkit`` from this tree. +PRELUDE = f''' +import io, json, pickle, struct, sys, warnings + +def loaded(*names): + return [name for name in names if name in sys.modules] + +def emit(**kwargs): + print(json.dumps(kwargs)) + +def ipv4(proto, payload): + return struct.pack('>BBHHHBBH4s4s', 0x45, 0, 20 + len(payload), 0, 0, 64, proto, 0, + bytes([127, 0, 0, 1]), bytes([127, 0, 0, 1])) + payload + +#: An IGMPv2 general query, eight octets: type 0x11, max response time 10s. +IGMP = bytes([0x11, 0x64, 0, 0]) + bytes(4) + +import pcapkit +assert pcapkit.__file__.startswith({str(ROOT)!r}), pcapkit.__file__ +''' + + +def run_child(code: 'str') -> 'dict[str, Any]': + """Run :data:`PRELUDE` then ``code`` in a fresh interpreter; return its last ``emit`` line.""" + environ = dict(os.environ) + environ.pop('PYTHONSAFEPATH', None) + environ['PYTHONPATH'] = os.pathsep.join( + [str(ROOT), environ['PYTHONPATH']] if environ.get('PYTHONPATH') else [str(ROOT)]) + result = subprocess.run([sys.executable, '-c', PRELUDE + textwrap.dedent(code)], # nosec: B603 + cwd=str(ROOT), env=environ, capture_output=True, text=True, + timeout=300, check=False) + if result.returncode != 0: + raise AssertionError(f'child exited {result.returncode}:\n{result.stderr}') + return json.loads(result.stdout.splitlines()[-1]) + + +class ColdImportTests(unittest.TestCase): + """What a cold ``import pcapkit`` leaves unimported, and what reaches it later.""" + + def test_import_pcapkit_binds_no_protocol_class(self) -> 'None': + """No package holds a protocol class, nor the registry, until it is reached.""" + out = run_child(f''' + import importlib + import pcapkit.protocols + bound = {{}} + for package in {PACKAGES!r}: + module = importlib.import_module(package) + bound[package] = sorted(name for name in module.__all__ if name in vars(module)) + emit(bound=bound, top=sorted(name for name in pcapkit.protocols.__all__ if name in vars(pcapkit)), + registry=['__proto__' in vars(pcapkit.protocols), 'pcapkit.protocols.registry' in sys.modules], + loaded=loaded('pcapkit.protocols.link.ethernet', 'pcapkit.protocols.link.s_tag', + 'pcapkit.protocols.internet.igmp', 'pcapkit.protocols.application.ngap')) + ''') + self.assertEqual(out['bound'], {package: sorted(EAGER.get(package, ())) for package in PACKAGES}) + self.assertEqual(out['top'], []) + self.assertEqual(out['registry'], [False, False]) + self.assertEqual(out['loaded'], []) + + def test_reaching_a_name_imports_its_package(self) -> 'None': + out = run_child(''' + import pcapkit.protocols.link as link + steps = [loaded('pcapkit.protocols.link.ethernet')] + ethernet = link.Ethernet + steps.append(loaded('pcapkit.protocols.link.ethernet')) + from pcapkit.protocols.link.ethernet.protocol import Ethernet + emit(steps=steps, same=[ethernet is Ethernet, vars(link)['Ethernet'] is Ethernet, + pcapkit.protocols.Ethernet is Ethernet, pcapkit.Ethernet is Ethernet, + link.ethernet is sys.modules['pcapkit.protocols.link.ethernet']]) + ''') + self.assertEqual(out['steps'], [[], ['pcapkit.protocols.link.ethernet']]) + self.assertEqual(out['same'], [True] * 5) + + def test_dispatch_imports_a_next_layer_not_yet_imported(self) -> 'None': + """IPv4 dispatches protocol number 2 to IGMP, which picks IGMPv2, neither imported yet.""" + out = run_child(''' + from pcapkit.protocols.internet.ipv4.protocol import IPv4 + before = loaded('pcapkit.protocols.internet.igmp', 'pcapkit.protocols.internet.igmpv2') + packet = IPv4(io.BytesIO(ipv4(2, IGMP)), 20 + len(IGMP)) + emit(before=before, chain=str(packet.protochain), layer=type(packet.payload).__module__, + registry='pcapkit.protocols.registry' in sys.modules) + ''') + self.assertEqual(out['before'], []) + self.assertEqual(out['chain'], 'IPv4:IGMPv2') + self.assertEqual(out['layer'], 'pcapkit.protocols.internet.igmpv2.protocol') + self.assertFalse(out['registry'], 'dispatching a next layer read the name registry') + + def test_a_registration_made_on_import_is_found_by_a_dissector(self) -> 'None': + """Shim6's entry exists after a bare ``import pcapkit``, so IPv6 parses it as ``IPv6_Ext``.""" + out = run_child(''' + from pcapkit.const.reg.transtype import TransType + from pcapkit.protocols.internet.internet import Internet + entry = Internet.__proto__.get(TransType.Shim6) + entry = None if entry is None else f'{entry.__module__}.{entry.__qualname__}' + from pcapkit.protocols.internet.ipv6.protocol import IPv6 + udp = struct.pack('>HHHH', 12345, 53, 8 + 8, 0) + b'shim6-ok' + body = bytes([int(TransType.UDP), 1]) + bytes(14) + udp + raw = struct.pack('>IHBB', 6 << 28, len(body), int(TransType.Shim6), 64) + raw += (bytes(15) + bytes([1])) * 2 + body + packet = IPv6(io.BytesIO(raw), len(raw)) + emit(entry=entry, chain=str(packet.protochain), + headers=[type(header).__name__ for _, header in packet.extension_headers.items(multi=True)]) + ''') + self.assertEqual(out['entry'], 'pcapkit.protocols.internet.ipv6_ext.protocol.IPv6_Ext') + self.assertEqual(out['headers'], ['IPv6_Ext']) + self.assertEqual(out['chain'], 'IPv6:IPv6-Ext:UDP:Raw') + + def test_a_selector_reaches_a_class_not_yet_imported(self) -> 'None': + """``'S_Tag' in`` a C-Tag frame expands through the registry, which imports S_Tag.""" + out = run_child(''' + Ethernet = pcapkit.protocols.link.Ethernet + tag = bytes([0x81, 0x00, 0x00, 0x01, 0x08, 0x00]) + udp = struct.pack('>HHHH', 1, 2, 8, 0) + raw = bytes([0xff] * 6) + bytes(range(6)) + tag + ipv4(17, udp) + frame = Ethernet(io.BytesIO(raw), len(raw)) + before = loaded('pcapkit.protocols.link.s_tag', 'pcapkit.protocols.registry') + emit(before=before, chain=str(frame.protochain), selected=['S_Tag' in frame, 'C_Tag' in frame], + after=loaded('pcapkit.protocols.link.s_tag', 'pcapkit.protocols.registry')) + ''') + self.assertEqual(out['before'], []) + self.assertEqual(out['chain'], 'Ethernet:802.1Q:IPv4:UDP') + self.assertEqual(out['selected'], [True, True]) + self.assertEqual(out['after'], ['pcapkit.protocols.link.s_tag', 'pcapkit.protocols.registry']) + + def test_the_registry_is_whole_when_first_read(self) -> 'None': + out = run_child(''' + import pcapkit.protocols as protocols + from pcapkit.protocols.protocol import ProtocolBase + before = ['__proto__' in vars(protocols), loaded('pcapkit.protocols.registry')] + from pcapkit.protocols import __proto__ as registry + names = [name for name in protocols.__all__ if isinstance(getattr(protocols, name), type) + and issubclass(getattr(protocols, name), ProtocolBase)] + emit(before=before, keys=sorted(registry), names=sorted(name.upper() for name in names), + values=all(registry[name.upper()] is getattr(protocols, name) for name in names), + same=[registry is protocols.__proto__, registry is vars(protocols)['__proto__'], + registry is sys.modules['pcapkit.protocols.registry'].__proto__]) + ''') + self.assertEqual(out['before'], [False, []]) + self.assertIn('IPV4', out['keys']) + self.assertEqual(out['keys'], out['names']) + self.assertTrue(out['values']) + self.assertEqual(out['same'], [True] * 3) + + def test_a_user_protocol_meets_the_shipped_name_before_any_read(self) -> 'None': + """Defining a :class:`~pcapkit.protocols.protocol.Protocol` subclass first still warns on a shipped name.""" + out = run_child(''' + from pcapkit.protocols.protocol import Protocol + from pcapkit.utilities.warnings import RegistryWarning + before = loaded('pcapkit.protocols.link.ethernet', 'pcapkit.protocols.registry') + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + + class Ethernet(Protocol): + """A user protocol named like the shipped one.""" + + emit(before=before, messages=[str(w.message) for w in caught if issubclass(w.category, RegistryWarning)], + registered=pcapkit.protocols.__proto__['ETHERNET'] is Ethernet) + ''') + self.assertEqual(out['before'], []) + self.assertEqual(len(out['messages']), 1, out['messages']) + self.assertIn('pcapkit.protocols.link.ethernet.protocol.Ethernet', out['messages'][0]) + self.assertTrue(out['registered']) + + def test_a_pickle_loads_where_its_class_is_not_yet_imported(self) -> 'None': + dumped = run_child(''' + from pcapkit.protocols.internet.ipv4.protocol import IPv4 + layer = IPv4(io.BytesIO(ipv4(2, IGMP)), 20 + len(IGMP)).payload + emit(payload=pickle.dumps(layer).hex(), info=repr(layer.info)) + ''') + out = run_child(f''' + before = loaded('pcapkit.protocols.internet.igmpv2') + layer = pickle.loads(bytes.fromhex({dumped['payload']!r})) + emit(before=before, layer=f'{{type(layer).__module__}}.{{type(layer).__qualname__}}', info=repr(layer.info)) + ''') + self.assertEqual(out['before'], []) + self.assertEqual(out['layer'], 'pcapkit.protocols.internet.igmpv2.protocol.IGMPv2') + self.assertEqual(out['info'], dumped['info']) + + +if __name__ == '__main__': + unittest.main()