diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index 602074282c..b24822cfa7 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -30,6 +30,7 @@ from pyiceberg.catalog.rest.response import TokenResponse, _handle_non_200_response from pyiceberg.exceptions import OAuthError +from pyiceberg.utils.transient import _TransientStateMixin AUTH_MANAGER = "auth.manager" @@ -127,7 +128,7 @@ def auth_header(self) -> str: return f"Bearer {self._token}" -class OAuth2TokenProvider: +class OAuth2TokenProvider(_TransientStateMixin): """Thread-safe OAuth2 token provider with token refresh support.""" client_id: str @@ -141,6 +142,9 @@ class OAuth2TokenProvider: _expires_at: int _lock: threading.Lock + # The token expiry is based on time.monotonic(), which is only meaningful within one process. + _transient_attrs = ("_token", "_expires_at", "_lock") + def __init__( self, client_id: str, @@ -156,7 +160,9 @@ def __init__( self.scope = scope self.refresh_margin = refresh_margin self.expires_in = expires_in + self._init_transient() + def _init_transient(self) -> None: self._token = None self._expires_at = 0 self._lock = threading.Lock() diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 49f10b1310..33bc709de4 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -23,7 +23,6 @@ import os import threading from collections.abc import Callable -from copy import copy from functools import lru_cache from typing import ( TYPE_CHECKING, @@ -96,6 +95,7 @@ from pyiceberg.typedef import Properties from pyiceberg.types import strtobool from pyiceberg.utils.properties import get_first_property_value, get_header_properties, property_as_bool +from pyiceberg.utils.transient import _TransientStateMixin logger = logging.getLogger(__name__) @@ -443,14 +443,19 @@ def to_input_file(self) -> FsspecInputFile: return FsspecInputFile(location=self.location, fs=self._fs) -class FsspecFileIO(FileIO): +class FsspecFileIO(_TransientStateMixin, FileIO): """A FileIO implementation that uses fsspec.""" + _transient_attrs = ("_thread_locals",) + def __init__(self, properties: Properties): self._scheme_to_fs: dict[str, Callable[..., AbstractFileSystem]] = dict(SCHEME_TO_FS) - self._thread_locals = threading.local() + self._init_transient() super().__init__(properties=properties) + def _init_transient(self) -> None: + self._thread_locals = threading.local() + @override def new_input(self, location: str) -> FsspecInputFile: """Get an FsspecInputFile instance to read bytes from the file at the given location. @@ -521,14 +526,3 @@ def _get_fs(self, scheme: str, hostname: str | None = None) -> AbstractFileSyste return _adls(self.properties, hostname) return self._scheme_to_fs[scheme](self.properties) - - def __getstate__(self) -> dict[str, Any]: - """Create a dictionary of the FsSpecFileIO fields used when pickling.""" - fileio_copy = copy(self.__dict__) - del fileio_copy["_thread_locals"] - return fileio_copy - - def __setstate__(self, state: dict[str, Any]) -> None: - """Deserialize the state into a FsSpecFileIO instance.""" - self.__dict__ = state - self._thread_locals = threading.local() diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 96ff36e859..7214d46f14 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -39,7 +39,6 @@ import warnings from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator -from copy import copy from dataclasses import dataclass from decimal import Decimal from enum import Enum @@ -193,6 +192,7 @@ from pyiceberg.utils.decimal import unscaled_to_decimal from pyiceberg.utils.properties import get_first_property_value, property_as_bool, property_as_int from pyiceberg.utils.singleton import Singleton +from pyiceberg.utils.transient import _TransientStateMixin from pyiceberg.utils.truncate import truncate_upper_bound_binary_string, truncate_upper_bound_text_string if TYPE_CHECKING: @@ -399,13 +399,17 @@ def to_input_file(self) -> PyArrowFile: return self -class PyArrowFileIO(FileIO): +class PyArrowFileIO(_TransientStateMixin, FileIO): fs_by_scheme: Callable[[str, str | None], FileSystem] + _transient_attrs = ("fs_by_scheme",) def __init__(self, properties: Properties = EMPTY_DICT): - self.fs_by_scheme: Callable[[str, str | None], FileSystem] = lru_cache(self._initialize_fs) + self._init_transient() super().__init__(properties=properties) + def _init_transient(self) -> None: + self.fs_by_scheme = lru_cache(self._initialize_fs) + @staticmethod def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[str, str, str]: r"""Return (scheme, netloc, path) for the given location. @@ -727,17 +731,6 @@ def delete(self, location: str | InputFile | OutputFile) -> None: raise PermissionError(f"Cannot delete file, access denied: {location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error - def __getstate__(self) -> dict[str, Any]: - """Create a dictionary of the PyArrowFileIO fields used when pickling.""" - fileio_copy = copy(self.__dict__) - fileio_copy["fs_by_scheme"] = None - return fileio_copy - - def __setstate__(self, state: dict[str, Any]) -> None: - """Deserialize the state into a PyArrowFileIO instance.""" - self.__dict__ = state - self.fs_by_scheme = lru_cache(self._initialize_fs) - def schema_to_pyarrow( schema: Schema | IcebergType, diff --git a/pyiceberg/utils/transient.py b/pyiceberg/utils/transient.py new file mode 100644 index 0000000000..e5e362ec77 --- /dev/null +++ b/pyiceberg/utils/transient.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Support for pickling objects that hold process-local state such as locks and caches.""" + +from typing import Any, ClassVar + + +class _TransientStateMixin: + """Excludes process-local attributes from pickling and rebuilds them on unpickle. + + Subclasses list those attributes in `_transient_attrs`, create them in `_init_transient`, + and call `_init_transient` from `__init__`. Place the mixin before other bases so its + `__getstate__` and `__setstate__` take precedence. + """ + + _transient_attrs: ClassVar[tuple[str, ...]] = () + + def _init_transient(self) -> None: + """Create the attributes listed in `_transient_attrs`.""" + + def __getstate__(self) -> dict[str, Any]: + """Return the instance state without the transient attributes.""" + state = self.__dict__.copy() + for attr in self._transient_attrs: + state.pop(attr, None) + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + """Restore the instance state and rebuild the transient attributes.""" + self.__dict__.update(state) + self._init_transient() diff --git a/tests/catalog/test_rest_auth.py b/tests/catalog/test_rest_auth.py index ae5d40f5aa..3202fa7d8f 100644 --- a/tests/catalog/test_rest_auth.py +++ b/tests/catalog/test_rest_auth.py @@ -16,13 +16,21 @@ # under the License. import base64 +import pickle from unittest.mock import MagicMock, patch import pytest import requests from requests_mock import Mocker -from pyiceberg.catalog.rest.auth import AuthManagerAdapter, BasicAuthManager, EntraAuthManager, GoogleAuthManager, NoopAuthManager +from pyiceberg.catalog.rest.auth import ( + AuthManagerAdapter, + BasicAuthManager, + EntraAuthManager, + GoogleAuthManager, + NoopAuthManager, + OAuth2TokenProvider, +) TEST_URI = "https://iceberg-test-catalog/" GOOGLE_CREDS_URI = "https://oauth2.googleapis.com/token" @@ -249,3 +257,19 @@ def test_entra_auth_manager_token_failure(mock_default_cred: MagicMock, rest_moc # Verify no requests were made with a blank/missing auth header history = rest_mock.request_history assert len(history) == 0 + + +def test_oauth2_token_provider_pickle_round_trip(requests_mock: Mocker) -> None: + token_url = f"{TEST_URI}v1/oauth/tokens" + requests_mock.post( + token_url, + [{"json": {"access_token": "first", "expires_in": 3600}}, {"json": {"access_token": "second", "expires_in": 3600}}], + ) + provider = OAuth2TokenProvider(client_id="client", client_secret="secret", token_url=token_url) + assert provider.get_token() == "first" + + unpickled = pickle.loads(pickle.dumps(provider)) + + # The cached token is not carried over, since its expiry is relative to the original process' clock + assert unpickled.get_token() == "second" + assert unpickled.client_id == "client" diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 51aa86c67f..cc01d1313d 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -26,11 +26,13 @@ from pyiceberg.io import ( ARROW_FILE_IO, PY_IO_IMPL, + FileIO, _import_file_io, _infer_file_io_from_scheme, _is_local_path, load_file_io, ) +from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFileIO @@ -75,10 +77,13 @@ def test_custom_local_output_file() -> None: assert len(output_file) == 3 -def test_pickled_pyarrow_round_trip() -> None: +@pytest.mark.parametrize("file_io_class", [PyArrowFileIO, FsspecFileIO]) +def test_pickled_file_io_round_trip(file_io_class: type[FileIO]) -> None: with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") - file_io = PyArrowFileIO() + file_io = file_io_class(properties={}) + # Exercise the cached filesystem so the transient state is populated before pickling + file_io.new_input(location=os.path.abspath(file_location)).exists() serialized_file_io = pickle.dumps(file_io) deserialized_file_io = pickle.loads(serialized_file_io) absolute_file_location = os.path.abspath(file_location)