fix(utilities): serialise first resolution of lazy exports and module descriptors so concurrent imports cannot deadlock (#2771)

docs/source/pcapkit/utilities/importlib.rst

@@ -7,9 +7,14 @@ Lazy Imports
 :pep:`562` ``__getattr__`` and ``__dir__`` that import each name it exports the first
 time the name is reached, rather than with the package itself. Every package under
 :mod:`pcapkit.const` imports its enumerations that way (:issue:`2729`), so a caller
-pays only for the enumerations it uses. This module imports nothing from
-:mod:`pcapkit`; see :ref:`utilities-or-corekit`.
+pays only for the enumerations it uses. Those first imports, and that of a
+:class:`~pcapkit.corekit.module.ModuleDescriptor`, hold :data:`import_lock`, one
+thread at a time (:issue:`2771`). This module imports nothing from :mod:`pcapkit`;
+see :ref:`utilities-or-corekit`.
 
 .. versionadded:: 1.6.0
 
 .. autofunction:: pcapkit.utilities.importlib.lazy_exports
+
+.. autodata:: pcapkit.utilities.importlib.import_lock
+   :no-value:

pcapkit/corekit/module.py

@@ -16,6 +16,7 @@
 
 from pcapkit.corekit.sentinels import NULL
 from pcapkit.utilities.exceptions import ProtocolError
+from pcapkit.utilities.importlib import import_lock
 
 __all__ = ['ModuleDescriptor']
 
@@ -83,6 +84,13 @@ def klass(self) -> 'Type[_T]':
             pre-reload class forever, and an instance of it fails
             :func:`isinstance` against the live one.
 
+            A first import, of a module not in :data:`sys.modules` yet, holds
+            :data:`~pcapkit.utilities.importlib.import_lock`, as the first imports
+            of :func:`~pcapkit.utilities.importlib.lazy_exports` do: threads
+            dispatching at once to protocols not imported yet otherwise deadlock
+            the import system (:issue:`2771`). A module already imported, or
+            part way through another thread's import, is read without it.
+
         """
         if self.name is NULL:
             # ``module`` is a ``str`` and the caller never named a class --
@@ -110,7 +118,11 @@ def klass(self) -> 'Type[_T]':
                 # from the ``getattr`` below instead, with the same message.
                 pass
         try:
-            return getattr(importlib.import_module(self.module), name)
+            if self.module in sys.modules:
+                return getattr(importlib.import_module(self.module), name)
+            # NOTE: the lock lazy_exports holds, so racing first imports cannot deadlock (#2771).
+            with import_lock:
+                return getattr(importlib.import_module(self.module), name)
         except AttributeError as error:
             # GitHub issue #832: a ``register_*`` helper that builds a
             # descriptor from a bad class name fails here with a bare stdlib

pcapkit/foundation/engines/pcap.py

@@ -13,6 +13,7 @@
 from typing import TYPE_CHECKING
 
 from pcapkit.corekit.io import SeekableReader
+from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.foundation.engines.engine import EngineBase
 from pcapkit.protocols.misc.pcap.frame.protocol import Frame
 from pcapkit.protocols.misc.pcap.frame.schema import RECORD_HEADER_LEN
@@ -23,13 +24,22 @@
 __all__ = ['PCAP']
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from pcapkit.const.reg.linktype import LinkType as Enum_LinkType
     from pcapkit.corekit.version import VersionInfo
 
 #: logging.Logger: Module-level logger, a child of the package-wide
 #: :data:`pcapkit.utilities.logging.logger`.
 logger = get_logger(__name__)
 
+#: The toolkit functions :meth:`PCAP.read_frame` hands each frame to, resolved
+#: there rather than imported, so their first import holds
+#: :data:`~pcapkit.utilities.importlib.import_lock` (:issue:`2771`).
+_TOOLKIT = tuple(ModuleDescriptor('pcapkit.toolkit.pcap', name) for name in (
+    'dtls_reassembly', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow',
+))  # type: tuple[ModuleDescriptor[Any], ...]
+
 
 class PCAP(EngineBase[Frame]):
     """PCAP file extraction support.
@@ -155,8 +165,8 @@ def read_frame(self) -> 'Frame':
             Parsed frame instance.
 
         """
-        from pcapkit.toolkit.pcap import (dtls_reassembly, ipv4_reassembly, ipv6_reassembly,
-                                          tcp_reassembly, tcp_traceflow)
+        dtls_reassembly, ipv4_reassembly, ipv6_reassembly, tcp_reassembly, tcp_traceflow = (
+            descriptor.klass for descriptor in _TOOLKIT)
         ext = self._extractor
 
         # read frame header

pcapkit/foundation/engines/pcapng.py

@@ -16,6 +16,7 @@
 from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType
 from pcapkit.corekit.infoclass import Info, info_final
 from pcapkit.corekit.io import SeekableReader
+from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.corekit.schema.padding import pad_length
 from pcapkit.dumpkit.pcap import PCAPIO
 from pcapkit.foundation.engines.engine import EngineBase
@@ -33,7 +34,7 @@
 logger = get_logger(__name__)
 
 if TYPE_CHECKING:
-    from typing import Literal, Optional
+    from typing import Any, Literal, Optional
 
     from pcapkit.foundation.extraction import Extractor
     from pcapkit.protocols.misc.pcapng.data import PCAPNG as Data_PCAPNG
@@ -50,6 +51,13 @@
         SystemdJournalExportBlock as Data_SystemdJournalExportBlock
     from pcapkit.protocols.misc.pcapng.data import UnknownBlock as Data_UnknownBlock
 
+#: The toolkit functions :meth:`PCAPNG.read_frame` hands each frame to, resolved
+#: there rather than imported, so their first import holds
+#: :data:`~pcapkit.utilities.importlib.import_lock` (:issue:`2771`).
+_TOOLKIT = tuple(ModuleDescriptor('pcapkit.toolkit.pcapng', name) for name in (
+    'dtls_reassembly', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow',
+))  # type: tuple[ModuleDescriptor[Any], ...]
+
 
 @info_final
 class Context(Info):
@@ -211,8 +219,8 @@ def read_frame(self) -> 'P_PCAPNG':
             Parsed PCAP-NG block.
 
         """
-        from pcapkit.toolkit.pcapng import (dtls_reassembly, ipv4_reassembly, ipv6_reassembly,
-                                            tcp_reassembly, tcp_traceflow)
+        dtls_reassembly, ipv4_reassembly, ipv6_reassembly, tcp_reassembly, tcp_traceflow = (
+            descriptor.klass for descriptor in _TOOLKIT)
         ext = self._extractor
 
         while True:

pcapkit/protocols/application/dtls/protocol.py

@@ -109,6 +109,7 @@
 from pcapkit.const.tls.extension_type import ExtensionType as Enum_ExtensionType
 from pcapkit.corekit.context import ProtocolContext
 from pcapkit.corekit.decorators import beholder
+from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.protocols.application.dtls.data import DTLS as Data_DTLS
 from pcapkit.protocols.application.dtls.data import DTLSCIDCiphertext as Data_DTLSCIDCiphertext
 from pcapkit.protocols.application.dtls.data import DTLSCiphertext as Data_DTLSCiphertext
@@ -126,12 +127,19 @@
 
     from typing_extensions import Literal
 
+    from pcapkit.const.reg.apptype.udp import UDP as AppType_UDP
     from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType
     from pcapkit.protocols.protocol import ProtocolBase
     from pcapkit.protocols.schema import Schema
 
 __all__ = ['DTLS', 'DTLSContext']
 
+#: The UDP port registry :meth:`DTLS.__index__` reads its port from, resolved
+#: there rather than imported, as ``import pcapkit`` leaves it unloaded
+#: (:issue:`1538`), so its first import holds
+#: :data:`~pcapkit.utilities.importlib.import_lock` (:issue:`2771`).
+_UDP: 'ModuleDescriptor[AppType_UDP]' = ModuleDescriptor('pcapkit.const.reg.apptype.udp', 'UDP')
+
 #: Length of a legacy record header -- content type, legacy version, epoch,
 #: sequence number and length (:rfc:`9147#section-4`).
 _LEGACY_HEADER_LENGTH = 13
@@ -768,10 +776,7 @@ def __index__(cls) -> 'int':  # type: ignore[override]
         .. _IANA: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml
 
         """
-        # imported here, as ``import pcapkit`` leaves the UDP registry unloaded (#1538)
-        from pcapkit.const.reg.apptype.udp import UDP  # pylint: disable=import-outside-toplevel
-
-        return int(UDP.coaps)  # type: ignore[call-overload]
+        return int(_UDP.klass.coaps)  # type: ignore[call-overload]
 
     ##########################################################################
     # Utilities.

pcapkit/protocols/application/ftp/protocol.py

@@ -35,6 +35,7 @@
 from pcapkit.const.ftp.command import Command as Enum_Command
 from pcapkit.const.ftp.return_code import ReturnCode as Enum_ReturnCode
 from pcapkit.corekit.enum import EnumLookup
+from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.protocols.application.application import Application
 from pcapkit.protocols.application.ftp.data import FTP as Data_FTP
 from pcapkit.protocols.application.ftp.data import Request as Data_Request
@@ -49,8 +50,16 @@
 
     from typing_extensions import Literal
 
+    from pcapkit.const.reg.apptype.tcp import TCP as AppType_TCP
+
 __all__ = ['FTP']
 
+#: The TCP port registry :meth:`FTP.__index__` reads its port from, resolved
+#: there rather than imported, as ``import pcapkit`` leaves it unloaded
+#: (:issue:`1538`), so its first import holds
+#: :data:`~pcapkit.utilities.importlib.import_lock` (:issue:`2771`).
+_TCP: 'ModuleDescriptor[AppType_TCP]' = ModuleDescriptor('pcapkit.const.reg.apptype.tcp', 'TCP')
+
 # regex for FTP, per :rfc:`959#section-4` (replies) and :rfc:`959#section-5`
 # (commands): a command is ``<command> [<SP> <argument>] <CRLF>``, a reply line is
 # ``<code> [<SP> <text>] <CRLF>``, and the first line of a multi-line reply is
@@ -290,10 +299,7 @@ def __index__(cls) -> 'int':  # type: ignore[override]
            :exc:`~pcapkit.utilities.exceptions.IntError` (:issue:`2627`).
 
         """
-        # imported here, as ``import pcapkit`` leaves the TCP registry unloaded (#1538)
-        from pcapkit.const.reg.apptype.tcp import TCP  # pylint: disable=import-outside-toplevel
-
-        return int(TCP.ftp)  # type: ignore[call-overload]
+        return int(_TCP.klass.ftp)  # type: ignore[call-overload]
 
     ##########################################################################
     # Utilities.

pcapkit/protocols/application/ftp_data/protocol.py

@@ -22,14 +22,23 @@
 """
 from typing import TYPE_CHECKING
 
+from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.protocols.application.application import Application
 from pcapkit.protocols.misc.raw.protocol import Raw
 
 if TYPE_CHECKING:
     from typing_extensions import Literal
 
+    from pcapkit.const.reg.apptype.tcp import TCP as AppType_TCP
+
 __all__ = ['FTP_DATA']
 
+#: The TCP port registry :meth:`FTP_DATA.__index__` reads its port from, resolved
+#: there rather than imported, as ``import pcapkit`` leaves it unloaded
+#: (:issue:`1538`), so its first import holds
+#: :data:`~pcapkit.utilities.importlib.import_lock` (:issue:`2771`).
+_TCP: 'ModuleDescriptor[AppType_TCP]' = ModuleDescriptor('pcapkit.const.reg.apptype.tcp', 'TCP')
+
 
 class FTP_DATA(Application, Raw):  # pylint: disable=abstract-method
     """This class implements FTP data channel transmission.
@@ -85,7 +94,4 @@ def __index__(cls) -> 'int':  # type: ignore[override]
            :exc:`~pcapkit.utilities.exceptions.IntError` (:issue:`2627`).
 
         """
-        # imported here, as ``import pcapkit`` leaves the TCP registry unloaded (#1538)
-        from pcapkit.const.reg.apptype.tcp import TCP  # pylint: disable=import-outside-toplevel
-
-        return int(TCP.ftp_data)  # type: ignore[call-overload]
+        return int(_TCP.klass.ftp_data)  # type: ignore[call-overload]