codekingpro/portable-devtools

1from __future__ import annotations2 3import contextlib4import inspect5import logging6import os7import pathlib8import sys9import time10import warnings11from abc import ABCMeta, abstractmethod12from dataclasses import dataclass13from threading import local14from typing import TYPE_CHECKING, Any, TypeVar15from weakref import WeakValueDictionary16 17from ._error import Timeout18 19#: Sentinel indicating that no explicit file permission mode was passed.20#: When used, lock files are created with 0o666 (letting umask and default ACLs control the final permissions)21#: and fchmod is skipped so that POSIX default ACL inheritance is preserved.22_UNSET_FILE_MODE: int = -123 24if TYPE_CHECKING:25    from collections.abc import Callable26    from types import TracebackType27 28    from ._read_write import ReadWriteLock29    from ._soft_rw import SoftReadWriteLock30 31    if sys.version_info >= (3, 11):  # pragma: no cover (py311+)32        from typing import Self33    else:  # pragma: no cover (<py311)34        from typing_extensions import Self35 36 37_LOGGER = logging.getLogger("filelock")38 39# On Windows os.path.realpath calls CreateFileW with share_mode=0, which blocks concurrent DeleteFileW and causes40# livelocks under threaded contention with SoftFileLock. os.path.abspath is purely string-based and avoids this.41_canonical = os.path.abspath if sys.platform == "win32" else os.path.realpath42 43 44class _ThreadLocalRegistry(local):45    def __init__(self) -> None:46        super().__init__()47        self.held: dict[str, int] = {}48 49 50_registry = _ThreadLocalRegistry()51 52 53# This is a helper class which is returned by :meth:`BaseFileLock.acquire` and wraps the lock to make sure __enter__54# is not called twice when entering the with statement. If we would simply return *self*, the lock would be acquired55# again in the *__enter__* method of the BaseFileLock, but not released again automatically. issue #37 (memory leak)56class AcquireReturnProxy:57    """A context-aware object that will release the lock file when exiting."""58 59    def __init__(self, lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock) -> None:60        self.lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock = lock61 62    def __enter__(self) -> BaseFileLock | ReadWriteLock | SoftReadWriteLock:63        return self.lock64 65    def __exit__(66        self,67        exc_type: type[BaseException] | None,68        exc_value: BaseException | None,69        traceback: TracebackType | None,70    ) -> None:71        self.lock.release()72 73 74@dataclass75class FileLockContext:76    """A dataclass which holds the context for a ``BaseFileLock`` object."""77 78    # The context is held in a separate class to allow optional use of thread local storage via the79    # ThreadLocalFileContext class.80 81    #: The path to the lock file.82    lock_file: str83 84    #: The default timeout value.85    timeout: float86 87    #: The mode for the lock files88    mode: int89 90    #: Whether the lock should be blocking or not91    blocking: bool92 93    #: The default polling interval value.94    poll_interval: float95 96    #: The lock lifetime in seconds; ``None`` means the lock never expires.97    lifetime: float | None = None98 99    #: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held100    lock_file_fd: int | None = None101 102    #: The lock counter is used for implementing the nested locking mechanism.103    lock_counter: int = 0  # When the lock is acquired is increased and the lock is only released, when this value is 0104 105 106class ThreadLocalFileContext(FileLockContext, local):107    """A thread local version of the ``FileLockContext`` class."""108 109 110_T = TypeVar("_T", bound="BaseFileLock")111 112 113class FileLockMeta(ABCMeta):114    _instances: WeakValueDictionary[str, BaseFileLock]115 116    def __call__(  # noqa: PLR0913117        cls: type[_T],118        lock_file: str | os.PathLike[str],119        timeout: float = -1,120        mode: int = _UNSET_FILE_MODE,121        thread_local: bool = True,  # noqa: FBT001, FBT002122        *,123        blocking: bool = True,124        is_singleton: bool = False,125        poll_interval: float = 0.05,126        lifetime: float | None = None,127        **kwargs: Any,  # capture remaining kwargs for subclasses  # noqa: ANN401128    ) -> _T:129        if is_singleton:130            instance = cls._instances.get(str(lock_file))131            if instance:132                params_to_check = {133                    "thread_local": (thread_local, instance.is_thread_local()),134                    "timeout": (timeout, instance.timeout),135                    "mode": (mode, instance._context.mode),  # noqa: SLF001136                    "blocking": (blocking, instance.blocking),137                    "poll_interval": (poll_interval, instance.poll_interval),138                    "lifetime": (lifetime, instance.lifetime),139                }140 141                non_matching_params = {142                    name: (passed_param, set_param)143                    for name, (passed_param, set_param) in params_to_check.items()144                    if passed_param != set_param145                }146                if not non_matching_params:147                    return instance  # ty: ignore[invalid-return-type]  # https://github.com/astral-sh/ty/issues/3231148 149                # parameters do not match; raise error150                msg = "Singleton lock instances cannot be initialized with differing arguments"151                msg += "\nNon-matching arguments: "152                for param_name, (passed_param, set_param) in non_matching_params.items():153                    msg += f"\n\t{param_name} (existing lock has {set_param} but {passed_param} was passed)"154                raise ValueError(msg)155 156        # Workaround to make `__init__`'s params optional in subclasses157        # E.g. virtualenv changes the signature of the `__init__` method in the `BaseFileLock` class descendant158        # (https://github.com/tox-dev/filelock/pull/340)159 160        all_params = {161            "timeout": timeout,162            "mode": mode,163            "thread_local": thread_local,164            "blocking": blocking,165            "is_singleton": is_singleton,166            "poll_interval": poll_interval,167            "lifetime": lifetime,168            **kwargs,169        }170 171        present_params = inspect.signature(cls.__init__).parameters172        init_params = {key: value for key, value in all_params.items() if key in present_params}173 174        instance = super().__call__(lock_file, **init_params)175 176        if is_singleton:177            cls._instances[str(lock_file)] = instance178 179        return instance180 181 182class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):183    """184    Abstract base class for a file lock object.185 186    Provides a reentrant, cross-process exclusive lock backed by OS-level primitives. Subclasses implement the actual187    locking mechanism (:class:`UnixFileLock <filelock.UnixFileLock>`, :class:`WindowsFileLock188    <filelock.WindowsFileLock>`, :class:`SoftFileLock <filelock.SoftFileLock>`).189 190    """191 192    _instances: WeakValueDictionary[str, BaseFileLock]193 194    def __init_subclass__(cls, **kwargs: dict[str, Any]) -> None:195        """Setup unique state for lock subclasses."""196        super().__init_subclass__(**kwargs)197        cls._instances = WeakValueDictionary()198 199    def __init__(  # noqa: PLR0913200        self,201        lock_file: str | os.PathLike[str],202        timeout: float = -1,203        mode: int = _UNSET_FILE_MODE,204        thread_local: bool = True,  # noqa: FBT001, FBT002205        *,206        blocking: bool = True,207        is_singleton: bool = False,208        poll_interval: float = 0.05,209        lifetime: float | None = None,210    ) -> None:211        """212        Create a new lock object.213 214        :param lock_file: path to the file215        :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the216            acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a217            negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.218        :param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and219            default ACLs, preserving POSIX default ACL inheritance in shared directories.220        :param thread_local: Whether this object's internal context should be thread local or not. If this is set to221            ``False`` then the lock will be reentrant across threads.222        :param blocking: whether the lock should be blocking or not223        :param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock224            file. This is useful if you want to use the lock object for reentrant locking without needing to pass the225            same object around.226        :param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value227            in the acquire method, if no poll_interval value (``None``) is given.228        :param lifetime: maximum time in seconds a lock can be held before it is considered expired. When set, a waiting229            process will break a lock whose file modification time is older than ``lifetime`` seconds. ``None`` (the230            default) means locks never expire.231 232        """233        self._is_thread_local = thread_local234        self._is_singleton = is_singleton235 236        # Create the context. Note that external code should not work with the context directly and should instead use237        # properties of this class.238        kwargs: dict[str, Any] = {239            "lock_file": os.fspath(lock_file),240            "timeout": timeout,241            "mode": mode,242            "blocking": blocking,243            "poll_interval": poll_interval,244            "lifetime": lifetime,245        }246        self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs)247 248    def is_thread_local(self) -> bool:249        """:returns: a flag indicating if this lock is thread local or not"""250        return self._is_thread_local251 252    @property253    def is_singleton(self) -> bool:254        """255        :returns: a flag indicating if this lock is singleton or not256 257        .. versionadded:: 3.13.0258 259        """260        return self._is_singleton261 262    @property263    def lock_file(self) -> str:264        """:returns: path to the lock file"""265        return self._context.lock_file266 267    @property268    def timeout(self) -> float:269        """270        :returns: the default timeout value, in seconds271 272        .. versionadded:: 2.0.0273 274        """275        return self._context.timeout276 277    @timeout.setter278    def timeout(self, value: float | str) -> None:279        """280        Change the default timeout value.281 282        :param value: the new value, in seconds283 284        """285        self._context.timeout = float(value)286 287    @property288    def blocking(self) -> bool:289        """290        :returns: whether the locking is blocking or not291 292        .. versionadded:: 3.14.0293 294        """295        return self._context.blocking296 297    @blocking.setter298    def blocking(self, value: bool) -> None:299        """300        Change the default blocking value.301 302        :param value: the new value as bool303 304        """305        self._context.blocking = value306 307    @property308    def poll_interval(self) -> float:309        """310        :returns: the default polling interval, in seconds311 312        .. versionadded:: 3.24.0313 314        """315        return self._context.poll_interval316 317    @poll_interval.setter318    def poll_interval(self, value: float) -> None:319        """320        Change the default polling interval.321 322        :param value: the new value, in seconds323 324        """325        self._context.poll_interval = value326 327    @property328    def lifetime(self) -> float | None:329        """330        :returns: the lock lifetime in seconds, or ``None`` if the lock never expires331 332        .. versionadded:: 3.24.0333 334        """335        return self._context.lifetime336 337    @lifetime.setter338    def lifetime(self, value: float | None) -> None:339        """340        Change the lock lifetime.341 342        :param value: the new value in seconds, or ``None`` to disable expiration343 344        """345        self._context.lifetime = value346 347    @property348    def mode(self) -> int:349        """:returns: the file permissions for the lockfile"""350        return 0o644 if self._context.mode == _UNSET_FILE_MODE else self._context.mode351 352    @property353    def has_explicit_mode(self) -> bool:354        """:returns: whether the file permissions were explicitly set"""355        return self._context.mode != _UNSET_FILE_MODE356 357    def _open_mode(self) -> int:358        """:returns: the mode for os.open() — 0o666 when unset (let umask/ACLs decide), else the explicit mode"""359        return 0o666 if self._context.mode == _UNSET_FILE_MODE else self._context.mode360 361    def _try_break_expired_lock(self) -> None:362        """Remove the lock file if its modification time exceeds the configured :attr:`lifetime`."""363        if (lifetime := self._context.lifetime) is None:364            return365        with contextlib.suppress(OSError):366            if time.time() - pathlib.Path(self.lock_file).stat().st_mtime < lifetime:367                return368            break_path = f"{self.lock_file}.break.{os.getpid()}"369            pathlib.Path(self.lock_file).rename(break_path)370            pathlib.Path(break_path).unlink()371 372    @abstractmethod373    def _acquire(self) -> None:374        """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file."""375        raise NotImplementedError376 377    @abstractmethod378    def _release(self) -> None:379        """Releases the lock and sets self._context.lock_file_fd to None."""380        raise NotImplementedError381 382    @property383    def is_locked(self) -> bool:384        """385        :returns: A boolean indicating if the lock file is holding the lock currently.386 387        .. versionchanged:: 2.0.0388 389            This was previously a method and is now a property.390 391        """392        return self._context.lock_file_fd is not None393 394    @property395    def lock_counter(self) -> int:396        """:returns: The number of times this lock has been acquired (but not yet released)."""397        return self._context.lock_counter398 399    @staticmethod400    def _check_give_up(  # noqa: PLR0913401        lock_id: int,402        lock_filename: str,403        *,404        blocking: bool,405        cancel_check: Callable[[], bool] | None,406        timeout: float,407        start_time: float,408    ) -> bool:409        if blocking is False:410            _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)411            return True412        if cancel_check is not None and cancel_check():413            _LOGGER.debug("Cancellation requested for lock %s on %s", lock_id, lock_filename)414            return True415        if 0 <= timeout < time.perf_counter() - start_time:416            _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)417            return True418        return False419 420    def acquire(  # noqa: C901421        self,422        timeout: float | None = None,423        poll_interval: float | None = None,424        *,425        poll_intervall: float | None = None,426        blocking: bool | None = None,427        cancel_check: Callable[[], bool] | None = None,428    ) -> AcquireReturnProxy:429        """430        Try to acquire the file lock.431 432        :param timeout: maximum wait time for acquiring the lock, ``None`` means use the default :attr:`~timeout` is and433            if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired434        :param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default435            :attr:`~poll_interval`436        :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead437        :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the438            first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.439        :param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll440            iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.441 442        :returns: a context object that will unlock the file when the context is exited443 444        :raises Timeout: if fails to acquire lock within the timeout period445 446        .. code-block:: python447 448            # You can use this method in the context manager (recommended)449            with lock.acquire():450                pass451 452            # Or use an equivalent try-finally construct:453            lock.acquire()454            try:455                pass456            finally:457                lock.release()458 459        .. versionchanged:: 2.0.0460 461            This method returns now a *proxy* object instead of *self*, so that it can be used in a with statement462            without side effects.463 464        """465        # Use the default timeout, if no timeout is provided.466        if timeout is None:467            timeout = self._context.timeout468 469        if blocking is None:470            blocking = self._context.blocking471 472        if poll_intervall is not None:473            msg = "use poll_interval instead of poll_intervall"474            warnings.warn(msg, DeprecationWarning, stacklevel=2)475            poll_interval = poll_intervall476 477        poll_interval = poll_interval if poll_interval is not None else self._context.poll_interval478 479        # Increment the number right at the beginning. We can still undo it, if something fails.480        self._context.lock_counter += 1481 482        lock_id = id(self)483        lock_filename = self.lock_file484        canonical = _canonical(lock_filename)485 486        would_block = self._context.lock_counter == 1 and not self.is_locked and timeout < 0 and blocking487        if would_block and (existing := _registry.held.get(canonical)) is not None and existing != lock_id:488            self._context.lock_counter -= 1489            msg = (490                f"Deadlock: lock '{lock_filename}' is already held by a different "491                f"FileLock instance in this thread. Use is_singleton=True to "492                f"enable reentrant locking across instances."493            )494            raise RuntimeError(msg)495 496        start_time = time.perf_counter()497        try:498            while True:499                if not self.is_locked:500                    self._try_break_expired_lock()501                    _LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)502                    self._acquire()503                if self.is_locked:504                    _LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)505                    break506                if self._check_give_up(507                    lock_id,508                    lock_filename,509                    blocking=blocking,510                    cancel_check=cancel_check,511                    timeout=timeout,512                    start_time=start_time,513                ):514                    raise Timeout(lock_filename)  # noqa: TRY301515                msg = "Lock %s not acquired on %s, waiting %s seconds ..."516                _LOGGER.debug(msg, lock_id, lock_filename, poll_interval)517                time.sleep(poll_interval)518        except BaseException:519            self._context.lock_counter = max(0, self._context.lock_counter - 1)520            if self._context.lock_counter == 0:521                _registry.held.pop(canonical, None)522            raise523        if self._context.lock_counter == 1:524            _registry.held[canonical] = lock_id525        return AcquireReturnProxy(lock=self)526 527    def release(self, force: bool = False) -> None:  # noqa: FBT001, FBT002528        """529        Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file530        itself is not automatically deleted.531 532        :param force: If true, the lock counter is ignored and the lock is released in every case.533 534        """535        if self.is_locked:536            self._context.lock_counter -= 1537 538            if self._context.lock_counter == 0 or force:539                lock_id, lock_filename = id(self), self.lock_file540 541                _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)542                self._release()543                self._context.lock_counter = 0544                _registry.held.pop(_canonical(lock_filename), None)545                _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)546 547    def __enter__(self) -> Self:548        """549        Acquire the lock.550 551        :returns: the lock object552 553        """554        self.acquire()555        return self556 557    def __exit__(558        self,559        exc_type: type[BaseException] | None,560        exc_value: BaseException | None,561        traceback: TracebackType | None,562    ) -> None:563        """564        Release the lock.565 566        :param exc_type: the exception type if raised567        :param exc_value: the exception value if raised568        :param traceback: the exception traceback if raised569 570        """571        self.release()572 573    def __del__(self) -> None:574        """Called when the lock object is deleted."""575        self.release(force=True)576 577 578__all__ = [579    "_UNSET_FILE_MODE",580    "AcquireReturnProxy",581    "BaseFileLock",582]583