Some cleanups; replace final pypi data retrieval using asyncio.

Signed-off-by: Thomas Neidhart <thomas.neidhart@gmail.com>

Author: netomi

requirements.txt

@@ -1,10 +1,10 @@
-attrs==22.1.0
-beautifulsoup4==4.11.1
-certifi==2022.6.15
-charset-normalizer==2.1.0
-click==8.1.3
-colorama==0.4.5
-commoncode==30.2.0
+attrs==23.1.0
+beautifulsoup4==4.12.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+commoncode==31.0.3
 dparse2==0.7.0
 idna==3.3
 importlib-metadata==4.12.0
@@ -14,13 +14,14 @@ packaging==21.3
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
-pyparsing==3.0.9
-PyYAML==6.0
-requests==2.28.1
+pyparsing==3.1.1
+PyYAML==6.0.1
+requests==2.31.0
 resolvelib >= 1.0.0
-saneyaml==0.5.2
-soupsieve==2.3.2.post1
+saneyaml==0.6.0
+soupsieve==2.5
 text-unidecode==1.3
 toml==0.10.2
-urllib3==1.26.11
-zipp==3.8.1
+urllib3==2.1.0
+zipp==3.17.0
+aiohttp==3.9.1

setup.cfg

@@ -69,6 +69,8 @@ install_requires =
     toml >= 0.10.0
     mock >= 3.0.5
     packvers >= 21.5
+    aiohttp >= 3.9
+
 [options.packages.find]
 where = src
 

src/python_inspector/api.py

@@ -8,7 +8,7 @@
 # See https://aboutcode-orgnexB/python-inspector for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
+import asyncio
 import os
 from netrc import netrc
 from typing import Dict
@@ -26,7 +26,6 @@
 from _packagedcode.pypi import PipRequirementsFileHandler
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
-from python_inspector import DEFAULT_PYTHON_VERSION
 from python_inspector import dependencies
 from python_inspector import utils
 from python_inspector import utils_pypi
@@ -231,7 +230,7 @@ def resolve_dependencies(
     if not direct_dependencies:
         return Resolution(
             packages=[],
-            resolution=[],
+            resolution={},
             files=files,
         )
 
@@ -288,19 +287,21 @@ def resolve_dependencies(
         ignore_errors=ignore_errors,
     )
 
-    packages = []
+    async def gather_pypi_data():
+        async def get_pypi_data(package):
+            if verbose:
+                printer(f"  package '{package}'")
 
-    for package in purls:
-        packages.extend(
-            [
-                pkg.to_dict()
-                for pkg in list(
-                    get_pypi_data_from_purl(
-                        package, repos=repos, environment=environment, prefer_source=prefer_source
-                    )
-                )
-            ],
-        )
+            return await get_pypi_data_from_purl(
+                package, repos=repos, environment=environment, prefer_source=prefer_source
+            )
+
+        if verbose:
+            printer(f"retrieve data from pypi:")
+
+        return await asyncio.gather(*[get_pypi_data(package) for package in purls])
+
+    packages = [pkg.to_dict() for pkg in asyncio.run(gather_pypi_data()) if pkg is not None]
 
     if verbose:
         printer("done!")
@@ -316,14 +317,14 @@ def resolve_dependencies(
 
 
 def resolve(
-    direct_dependencies,
-    environment,
-    repos=tuple(),
-    as_tree=False,
-    max_rounds=200000,
-    pdt_output=False,
-    analyze_setup_py_insecurely=False,
-    ignore_errors=False,
+    direct_dependencies: List[DependentPackage],
+    environment: Environment,
+    repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(),
+    as_tree: bool = False,
+    max_rounds: int = 200000,
+    pdt_output: bool = False,
+    analyze_setup_py_insecurely: bool = False,
+    ignore_errors: bool = False,
 ):
     """
     Resolve dependencies given a ``direct_dependencies`` list of

src/python_inspector/package_data.py

@@ -9,7 +9,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from typing import List
+from typing import List, Iterable, Optional
 
 from packageurl import PackageURL
 
@@ -24,9 +24,9 @@
 from python_inspector.utils_pypi import PypiSimpleRepository
 
 
-def get_pypi_data_from_purl(
+async def get_pypi_data_from_purl(
     purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
-) -> PackageData:
+) -> Optional[PackageData]:
     """
     Generate `Package` object from the `purl` string of pypi type
 
@@ -36,18 +36,19 @@ def get_pypi_data_from_purl(
     ``prefer_source`` is a boolean value to prefer source distribution over wheel,
     if no source distribution is available then wheel is used
     """
-    purl = PackageURL.from_string(purl)
-    name = purl.name
-    version = purl.version
+    parsed_purl = PackageURL.from_string(purl)
+    name = parsed_purl.name
+    version = parsed_purl.version
     if not version:
         raise Exception("Version is not specified in the purl")
     base_path = "https://pypi.org/pypi"
     api_url = f"{base_path}/{name}/{version}/json"
-    from python_inspector.resolution import get_response
 
-    response = get_response(api_url)
+    from python_inspector.utils import get_response_async
+    response = await get_response_async(api_url)
     if not response:
-        return []
+        return None
+
     info = response.get("info") or {}
     homepage_url = info.get("home_page")
     project_urls = info.get("project_urls") or {}
@@ -56,13 +57,9 @@ def get_pypi_data_from_purl(
     python_version = get_python_version_from_env_tag(python_version=environment.python_version)
     valid_distribution_urls = []
 
-    valid_distribution_urls.append(
-        get_sdist_download_url(
-            purl=purl,
-            repos=repos,
-            python_version=python_version,
-        )
-    )
+    sdist_url = get_sdist_download_url(purl=parsed_purl, repos=repos, python_version=python_version)
+    if sdist_url:
+        valid_distribution_urls.append(sdist_url)
 
     valid_distribution_urls = [url for url in valid_distribution_urls if url]
 
@@ -71,24 +68,27 @@ def get_pypi_data_from_purl(
     if not valid_distribution_urls or not prefer_source:
         wheel_urls = list(
             get_wheel_download_urls(
-                purl=purl,
+                purl=parsed_purl,
                 repos=repos,
                 environment=environment,
                 python_version=python_version,
             )
         )
         wheel_url = choose_single_wheel(wheel_urls)
         if wheel_url:
-            valid_distribution_urls.append(wheel_url)
+            valid_distribution_urls.insert(0, wheel_url)
 
-    urls = response.get("urls") or []
-    for url in urls:
-        dist_url = url.get("url")
-        if dist_url not in valid_distribution_urls:
+    urls = {url.get("url"): url for url in response.get("urls", [])}
+    # iterate over the valid distribution urls and return the first
+    # one that is matching.
+    for dist_url in valid_distribution_urls:
+        if dist_url not in urls:
             continue
-        digests = url.get("digests") or {}
 
-        yield PackageData(
+        url_data = urls.get(dist_url)
+        digests = url_data.get("digests", {})
+
+        return PackageData(
             primary_language="Python",
             description=get_description(info),
             homepage_url=homepage_url,
@@ -98,10 +98,10 @@ def get_pypi_data_from_purl(
             license_expression=info.get("license_expression"),
             declared_license=get_declared_license(info),
             download_url=dist_url,
-            size=url.get("size"),
-            md5=digests.get("md5") or url.get("md5_digest"),
+            size=url_data.get("size"),
+            md5=digests.get("md5") or url_data.get("md5_digest"),
             sha256=digests.get("sha256"),
-            release_date=url.get("upload_time"),
+            release_date=url_data.get("upload_time"),
             keywords=get_keywords(info),
             parties=get_parties(
                 info,
@@ -110,9 +110,11 @@ def get_pypi_data_from_purl(
                 maintainer_key="maintainer",
                 maintainer_email_key="maintainer_email",
             ),
-            **purl.to_dict(),
+            **parsed_purl.to_dict(),
         )
 
+    return None
+
 
 def choose_single_wheel(wheel_urls):
     """
@@ -125,18 +127,18 @@ def choose_single_wheel(wheel_urls):
 
 def get_pypi_bugtracker_url(project_urls):
     bug_tracking_url = project_urls.get("Tracker")
-    if not (bug_tracking_url):
+    if not bug_tracking_url:
         bug_tracking_url = project_urls.get("Issue Tracker")
-    if not (bug_tracking_url):
+    if not bug_tracking_url:
         bug_tracking_url = project_urls.get("Bug Tracker")
     return bug_tracking_url
 
 
 def get_pypi_codeview_url(project_urls):
     code_view_url = project_urls.get("Source")
-    if not (code_view_url):
+    if not code_view_url:
         code_view_url = project_urls.get("Code")
-    if not (code_view_url):
+    if not code_view_url:
         code_view_url = project_urls.get("Source Code")
     return code_view_url
 
@@ -146,7 +148,7 @@ def get_wheel_download_urls(
     repos: List[PypiSimpleRepository],
     environment: Environment,
     python_version: str,
-) -> List[str]:
+) -> Iterable[str]:
     """
     Return a list of download urls for the given purl.
     """

src/python_inspector/resolution.py

@@ -10,7 +10,6 @@
 import ast
 import operator
 import os
-import re
 import tarfile
 from typing import Dict
 from typing import Generator

src/python_inspector/utils.py

@@ -11,10 +11,11 @@
 
 import json
 import os
-from typing import Dict
+from typing import Dict, Optional
 from typing import List
 from typing import NamedTuple
 
+import aiohttp
 import requests
 
 
@@ -67,13 +68,26 @@ class Candidate(NamedTuple):
 def get_response(url: str) -> Dict:
     """
     Return a mapping of the JSON response from fetching ``url``
-    or None if the ``url`` cannot be fetched..
+    or None if the ``url`` cannot be fetched.
     """
     resp = requests.get(url)
     if resp.status_code == 200:
         return resp.json()
 
 
+async def get_response_async(url: str) -> Optional[Dict]:
+    """
+    Return a mapping of the JSON response from fetching ``url``
+    or None if the ``url`` cannot be fetched.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as response:
+            if response.status == 200:
+                return await response.json()
+            else:
+                return None
+
+
 def remove_test_data_dir_variable_prefix(path, placeholder="<file>"):
     """
     Return a clean path, removing variable test path prefix or using a ``placeholder``.