Better deduping of UV pythons

.github/workflows/auto_test.yml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.15-dev", "3.14", "3.13", "3.12"]
+        python-version: ["3.15", "3.14", "3.13", "3.12"]
 
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

pyproject.toml

@@ -73,7 +73,7 @@ testpaths = [
 
 [tool.ruff.lint]
 # I001 / RUF023 - Import, __all__ and __slots__ sorting rules.
-ignore = ["I001", "RUF022", "RUF023"]
+ignore = ["I001", "RUF022", "RUF023", "PLR0402"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 # The details script is written to run under ancient Python as well as new Python

src/ducktools/pythonfinder/darwin/__init__.py

@@ -29,7 +29,8 @@
     from collections.abc import Iterator
 
 from .. import linux
-from ..shared import get_uv_pythons, DetailFinder,PythonInstall
+from ..shared import DetailFinder, PythonInstall
+from ..uv_search import get_uv_pythons
 
 
 # This is the difference from the linux methods
@@ -63,8 +64,8 @@ def get_python_installs(
     finder = DetailFinder() if finder is None else finder
 
     chain_commands = [
-        linux.get_pyenv_pythons(finder=finder),
         get_uv_pythons(finder=finder),
+        linux.get_pyenv_pythons(finder=finder),
         get_path_pythons(finder=finder),
     ]
     with finder:

src/ducktools/pythonfinder/linux/__init__.py

@@ -35,10 +35,9 @@
     DetailFinder,
     PythonInstall,
     get_folder_pythons,
-    get_uv_pythons,
-    get_uv_python_path
 )
 from .pyenv_search import get_pyenv_pythons, get_pyenv_root
+from ..uv_search import get_uv_python_path, get_uv_pythons
 
 
 KNOWN_MANAGED_PATHS = {
@@ -106,8 +105,8 @@ def get_python_installs(
     finder = DetailFinder() if finder is None else finder
 
     chain_commands: list[Iterator[PythonInstall]] = [
-        get_pyenv_pythons(finder=finder),
         get_uv_pythons(finder=finder),
+        get_pyenv_pythons(finder=finder),
         get_path_pythons(finder=finder),
     ]
     with finder:

src/ducktools/pythonfinder/shared.py

@@ -49,6 +49,8 @@
     ]
 )
 
+type _IdentifyingTuple = tuple[str | None, str, str, str, bool]
+
 FULL_PY_VER_RE = r"(?P<major>\d+)\.(?P<minor>\d+)\.?(?P<micro>\d*)-?(?P<releaselevel>a|b|c|rc)?(?P<serial>\d*)?"
 
 
@@ -364,7 +366,7 @@ def version_str(self) -> str:
         return version_tuple_to_str(self.version)
 
     @property
-    def implementation_version(self) -> tuple[int, int, int, str, int] | None:
+    def implementation_version(self) -> tuple[int, int, int, str, int]:
         """
         The implementation version may differ from the 'Version' for implementations
         other than CPython.
@@ -375,12 +377,19 @@ def implementation_version(self) -> tuple[int, int, int, str, int] | None:
         """
         if self._implementation_version is None:
             if implementation_ver := self.metadata.get(f"{self.implementation}_version"):
-                if len(implementation_ver) == 3:
-                    self._implementation_version = (*implementation_ver, "final", 0)  # type: ignore
+                if len(implementation_ver) == 5:
+                    ver_tuple = implementation_ver
+                elif len(implementation_ver) == 3:
+                    ver_tuple = (*implementation_ver, "final", 0)
                 else:
-                    self._implementation_version = implementation_ver
+                    raise ValueError(
+                        f"Implementation version number {implementation_ver!r} "
+                        "does not match known Python implementation version formats"
+                    )
             else:
-                self._implementation_version = self.version
+                ver_tuple = self.version
+
+            self._implementation_version = ver_tuple
 
         return self._implementation_version
 
@@ -391,6 +400,21 @@ def implementation_version_str(self) -> str:
         """
         return version_tuple_to_str(self.implementation_version)
 
+    @property
+    def freethreaded(self):
+        return self.metadata.get("freethreaded", False)
+
+    @property
+    def _identifying_tuple(self) -> _IdentifyingTuple:
+        # An identifying tuple to be used for deduplication
+        return (
+            self.managed_by,
+            self.version_str,
+            self.implementation,
+            self.implementation_version_str,
+            self.freethreaded,
+        )
+
     @classmethod
     def from_str(
         cls,
@@ -507,83 +531,3 @@ def get_folder_pythons(
                 install = finder.get_install_details(p, managed_by=managed_by)
                 if install:
                     yield install
-
-
-# UV Specific finder
-def get_uv_python_path() -> str | None:
-    # Attempt to get cache
-    try:
-        with open(INSTALLER_CACHE_PATH) as f:
-            installer_cache = _laz.json.load(f)
-    except (FileNotFoundError, _laz.json.JSONDecodeError):
-        installer_cache = {}
-
-    uv_python_dir = installer_cache.get("uv")
-    if uv_python_dir and os.path.exists(uv_python_dir):
-        return uv_python_dir
-
-    # Cache failed
-    try:
-        uv_python_find = _laz.subprocess.run(
-            ["uv", "python", "dir"],
-            check=True,
-            text=True,
-            capture_output=True
-        )
-    except (_laz.subprocess.CalledProcessError, FileNotFoundError):
-        uv_python_dir = None
-    else:
-        # remove newline
-        uv_python_dir = uv_python_find.stdout.strip()
-
-    # Fill cache and update the cache file
-    installer_cache["uv"] = uv_python_dir
-    os.makedirs(os.path.dirname(INSTALLER_CACHE_PATH), exist_ok=True)
-    with open(INSTALLER_CACHE_PATH, 'w') as f:
-        _laz.json.dump(installer_cache, f)
-
-    return uv_python_dir
-
-
-def _implementation_from_uv_dir(
-    direntry: os.DirEntry,
-    finder: DetailFinder | None = None,
-) -> PythonInstall | None:
-    if sys.platform == "win32":
-        python_paths = [
-            os.path.join(direntry, "python.exe"),
-            os.path.join(direntry, "bin/python.exe"),  # graalpy
-        ]
-    else:
-        python_paths = [
-            os.path.join(direntry, "bin/python")
-        ]
-
-    install: PythonInstall | None = None
-    finder = DetailFinder() if finder is None else finder
-
-    for pth in python_paths:
-        if os.path.exists(pth):
-            install = finder.get_install_details(pth, managed_by="Astral")
-            break
-
-    return install
-
-
-def get_uv_pythons(finder=None) -> Iterator[PythonInstall]:
-    # This takes some shortcuts over the regular pythonfinder
-    # As the UV folders give the python version and the implementation
-    if (
-        (uv_python_path := get_uv_python_path())
-        and os.path.exists(uv_python_path)
-    ):
-        finder = DetailFinder() if finder is None else finder
-
-        with finder, os.scandir(uv_python_path) as fld:
-            for f in fld:
-                if (
-                    f.is_dir()
-                    and not f.is_symlink()
-                    and (install := _implementation_from_uv_dir(f, finder=finder))
-                ):
-                    yield install

src/ducktools/pythonfinder/uv_search.py

@@ -0,0 +1,123 @@
+import os
+import os.path
+import sys
+
+try:
+    from _collections_abc import Iterator
+except ImportError:
+    from collections.abc import Iterator
+
+from ducktools.lazyimporter import LazyImporter, ModuleImport
+
+from .shared import (
+    _IdentifyingTuple,
+    INSTALLER_CACHE_PATH,
+    DetailFinder,
+    PythonInstall,
+)
+
+_laz = LazyImporter(
+    [
+        ModuleImport("json"),
+        ModuleImport("subprocess"),
+    ]
+)
+
+# UV Specific finder
+def get_uv_python_path() -> str | None:
+    # Attempt to get cache
+    try:
+        with open(INSTALLER_CACHE_PATH) as f:
+            installer_cache = _laz.json.load(f)
+    except (FileNotFoundError, _laz.json.JSONDecodeError):
+        installer_cache = {}
+
+    uv_python_dir = installer_cache.get("uv")
+    if uv_python_dir and os.path.exists(uv_python_dir):
+        return uv_python_dir
+
+    # Cache failed
+    try:
+        uv_python_find = _laz.subprocess.run(
+            ["uv", "python", "dir"],
+            check=True,
+            text=True,
+            capture_output=True
+        )
+    except (_laz.subprocess.CalledProcessError, FileNotFoundError):
+        uv_python_dir = None
+    else:
+        # remove newline
+        uv_python_dir = uv_python_find.stdout.strip()
+
+    # Fill cache and update the cache file
+    installer_cache["uv"] = uv_python_dir
+    os.makedirs(os.path.dirname(INSTALLER_CACHE_PATH), exist_ok=True)
+    with open(INSTALLER_CACHE_PATH, 'w') as f:
+        _laz.json.dump(installer_cache, f)
+
+    return uv_python_dir
+
+
+def _implementation_from_uv_dir(
+    direntry: os.DirEntry,
+    finder: DetailFinder | None = None,
+) -> PythonInstall | None:
+    if sys.platform == "win32":
+        python_paths = [
+            os.path.join(direntry, "python.exe"),
+            os.path.join(direntry, "bin/python.exe"),  # graalpy
+        ]
+    else:
+        python_paths = [
+            os.path.join(direntry, "bin/python")
+        ]
+
+    install: PythonInstall | None = None
+    finder = DetailFinder() if finder is None else finder
+
+    for pth in python_paths:
+        if os.path.exists(pth):
+            install = finder.get_install_details(pth, managed_by="Astral")
+            break
+
+    return install
+
+
+def get_uv_pythons(finder=None) -> Iterator[PythonInstall]:
+    """
+    Search UVs python folders for unique python installs.
+
+    If multiple folders are found with the same Python version,
+    implementation name and implementation version and freethreading support
+    then only the executable with the shortest path will be given.
+
+    So if you have CPython 3.x.y installed in 3.x.y and 3.x directories
+    only the install with the 3.x path will be given. This is the
+    case even if the 3.x path is a symlink to the 3.x.y folder.
+    """
+    # Filtering is done by the triple of
+    # (python version, implementation name, implementation version)
+    if (
+        (uv_python_path := get_uv_python_path())
+        and os.path.exists(uv_python_path)
+    ):
+        finder = DetailFinder() if finder is None else finder
+
+        with finder, os.scandir(uv_python_path) as fld:
+            unique_releases: dict[_IdentifyingTuple, list[PythonInstall]] = {}
+
+            # Create a folder of stdlib paths to folders
+            for f in fld:
+                if (
+                    f.is_dir()
+                    and (install := _implementation_from_uv_dir(f, finder=finder))
+                ):
+                    identifier = install._identifying_tuple
+                    release_group = unique_releases.setdefault(identifier, [])
+                    release_group.append(install)
+
+            for install_group in unique_releases.values():
+                # Sort the group by path length so the shortest item is first
+                install_group.sort(key=lambda py: len(py.executable))
+                yield install_group[0]

src/ducktools/pythonfinder/win32/__init__.py

@@ -22,16 +22,15 @@
 # SOFTWARE.
 from __future__ import annotations
 
-import itertools
-
 try:
     from _collections_abc import Iterator
 except ImportError:
     from collections.abc import Iterator
 
-from ..shared import PythonInstall, get_uv_pythons, DetailFinder
+from ..shared import _IdentifyingTuple, PythonInstall, DetailFinder
 from .pyenv_search import get_pyenv_pythons
 from .registry_search import get_registered_pythons
+from ..uv_search import get_uv_pythons
 
 
 def get_python_installs(
@@ -44,12 +43,28 @@ def get_python_installs(
     finder = DetailFinder() if finder is None else finder
 
     with finder:
-        for py in itertools.chain(
-            get_registered_pythons(),
-            get_pyenv_pythons(finder=finder),
-            get_uv_pythons(finder=finder),
-        ):
-            # Compare by stdlib paths for uniqueness
+        uv_matches: dict[_IdentifyingTuple, PythonInstall] = {
+            py._identifying_tuple: py
+            for py in get_uv_pythons(finder=finder)
+        }
+
+        for py in get_registered_pythons():
+            if match := uv_matches.get(py._identifying_tuple):
+                # Registry entries have better metadata, update the metadata in the retrieved entries
+                match.metadata.update(py.metadata)
+            else:
+                stdlib_path = py.paths.get("stdlib")
+                if stdlib_path:
+                    if stdlib_path not in listed_stdlibs:
+                        yield py
+                        listed_stdlibs.add(stdlib_path)
+                elif py.executable not in listed_bins:
+                    yield py
+                    listed_bins.add(py.executable)
+
+        yield from uv_matches.values()
+
+        for py in get_pyenv_pythons(finder=finder):
             stdlib_path = py.paths.get("stdlib")
             if stdlib_path:
                 if stdlib_path not in listed_stdlibs: