Preserve download integrity with lockfile

Introduce cross-process and asyncio lockfile with "fasterner"
to avoid damaging wheels downloads with partila overwrites,
reusing code from ScanCode Toolkit used for the license cache index

Reference: #215
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

requirements-dev.txt

@@ -7,7 +7,7 @@ cryptography==37.0.2
 dataclasses==0.8
 docutils==0.18.1
 et-xmlfile==1.1.0
-execnet==1.9.0
+execnet==2.1.1
 importlib-resources==5.4.0
 iniconfig==1.1.1
 isort==5.10.1
@@ -21,14 +21,15 @@ openpyxl==3.0.10
 pathspec==0.9.0
 pkginfo==1.8.3
 platformdirs==2.4.0
-pluggy==1.0.0
+pluggy==1.6.0
 py==1.11.0
 pycodestyle==2.8.0
 pycparser==2.21
 pygments==2.12.0
-pytest==7.0.1
-pytest-forked==1.4.0
-pytest-xdist==2.5.0
+pytest==8.4.0
+pytest-xdist==3.7.0
+pytest-forked==1.6.0
+pytest-rerunfailures==15.1
 readme-renderer==34.0
 requests-toolbelt==0.9.1
 rfc3986==1.5.0
@@ -39,4 +40,4 @@ tqdm==4.64.0
 twine==3.8.0
 typed-ast==1.5.4
 webencodings==0.5.1
-pytest-asyncio==0.21.1
+pytest-asyncio==1.0.0

requirements.txt

@@ -6,6 +6,7 @@ click==8.1.3
 colorama==0.4.5
 commoncode==30.2.0
 dparse2==0.7.0
+fasteners==0.17.3
 idna==3.3
 importlib-metadata==4.12.0
 intbitset==3.1.0
@@ -26,5 +27,5 @@ text-unidecode==1.3
 toml==0.10.2
 urllib3==1.26.11
 zipp==3.8.1
-aiohttp==3.11.14
-aiofiles==23.2.1
+aiohttp==3.12.7
+aiofiles==24.1.0

setup.cfg

@@ -59,6 +59,7 @@ install_requires =
     colorama >= 0.3.9
     commoncode >= 30.0.0
     dparse2 >= 0.7.0
+    fasteners >= 0.17.3
     importlib_metadata >= 4.12.0
     packageurl_python >= 0.9.0
     pkginfo2 >= 30.0.0

src/python_inspector/lockfile.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from contextlib import contextmanager
+
+import fasteners
+
+"""
+An interprocess lockfile with a timeout.
+"""
+
+
+class LockTimeout(Exception):
+    pass
+
+
+class FileLock(fasteners.InterProcessLock):
+    @contextmanager
+    def locked(self, timeout):
+        acquired = self.acquire(timeout=timeout)
+        if not acquired:
+            raise LockTimeout(timeout)
+        try:
+            yield
+        finally:
+            self.release()

src/python_inspector/utils_pypi.py

@@ -41,6 +41,7 @@
 from packvers import version as packaging_version
 from packvers.specifiers import SpecifierSet
 
+from python_inspector import lockfile
 from python_inspector import pyinspector_settings as settings
 from python_inspector import utils_pip_compatibility_tags
 
@@ -1650,6 +1651,8 @@ def resolve_relative_url(package_url, url):
 #
 ################################################################################
 
+PYINSP_CACHE_LOCK_TIMEOUT = 120  # in seconds
+
 
 @attr.attributes
 class Cache:
@@ -1681,6 +1684,7 @@ async def get(
         True otherwise as treat as binary. `path_or_url` can be a path or a URL
         to a file.
         """
+        # the cache key is a hash of the normalized path
         cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/")))
         cached = os.path.join(self.directory, cache_key)
 
@@ -1695,8 +1699,13 @@ async def get(
                 echo_func=echo_func,
             )
             wmode = "w" if as_text else "wb"
-            async with aiofiles.open(cached, mode=wmode) as fo:
-                await fo.write(content)
+
+            # acquire lock and wait until timeout to get a lock or die
+            lock_file = os.path.join(self.directory, f"{cache_key}.lockfile")
+
+            with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
+                async with aiofiles.open(cached, mode=wmode) as fo:
+                    await fo.write(content)
             return content, cached
         else:
             if TRACE_DEEP: