Create settings class based on pydantic-settings

Settings can now be defined through environment variables, .env local
file or regular arguments to the script.

All variables need to be prefixed by PYTHON_INSPECTOR_ to be recognized.
Example: PYTHON_INSPECTOR_INDEX_URL="https://pypy1.org,https://foo.bar"
will add this two repositories overriding the public repository.

Raise the minimum python version to 3.9 as 3.8 is EOL.

Adapt base code to use new settings class

Adapt tests to use new settings class

Signed-off-by: Helio Chissini de Castro <helio.chissini.de.castro@cariad.technology>

Author: heliocastro

requirements.txt

@@ -14,6 +14,7 @@ packaging==21.3
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
+pydantic_settings >= 2.6.1
 pyparsing==3.0.9
 PyYAML==6.0
 requests==2.28.1

src/python_inspector/__init__.py

@@ -7,4 +7,12 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-DEFAULT_PYTHON_VERSION = "3.8"
+from pydantic import ValidationError
+
+from python_inspector.settings import Settings
+
+# Initialize global settings
+try:
+    settings = Settings()
+except ValidationError as e:
+    print(e)

src/python_inspector/api.py

@@ -26,8 +26,8 @@
 from _packagedcode.pypi import PipRequirementsFileHandler
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
-from python_inspector import DEFAULT_PYTHON_VERSION
 from python_inspector import dependencies
+from python_inspector import settings
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.package_data import get_pypi_data_from_purl
@@ -40,8 +40,8 @@
 from python_inspector.resolution import get_reqs_insecurely
 from python_inspector.resolution import get_requirements_from_python_manifest
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
-from python_inspector.utils_pypi import PYPI_SIMPLE_URL
 from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
 from python_inspector.utils_pypi import valid_python_versions
 
 
@@ -79,7 +79,7 @@ def resolve_dependencies(
     specifiers=tuple(),
     python_version=None,
     operating_system=None,
-    index_urls=tuple([PYPI_SIMPLE_URL]),
+    index_urls: tuple[str, ...] = settings.INDEX_URL,
     pdt_output=None,
     netrc_file=None,
     max_rounds=200000,
@@ -106,15 +106,15 @@ def resolve_dependencies(
     """
 
     if not operating_system:
-        raise Exception(f"No operating system provided.")
+        raise Exception("No operating system provided.")
     if operating_system not in PLATFORMS_BY_OS:
         raise ValueError(
             f"Invalid operating system: {operating_system}. "
             f"Must be one of: {', '.join(PLATFORMS_BY_OS.keys())}"
         )
 
     if not python_version:
-        raise Exception(f"No python version provided.")
+        raise Exception("No python version provided.")
     if python_version not in valid_python_versions:
         raise ValueError(
             f"Invalid python version: {python_version}. "
@@ -147,14 +147,13 @@ def resolve_dependencies(
 
     files = []
 
-    if PYPI_SIMPLE_URL not in index_urls:
-        index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls)
-
     # requirements
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(
             requirements_file=req_file)
-        for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
+        for extra_data in dependencies.get_extra_data_from_requirements(
+            requirements_file=req_file
+        ):
             index_urls = (
                 *index_urls, *tuple(extra_data.get("extra_index_urls") or []))
             index_urls = (
@@ -260,10 +259,8 @@ def resolve_dependencies(
         # Collect PyPI repos
         for index_url in index_urls:
             index_url = index_url.strip("/")
-            existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
-            if existing:
-                existing.use_cached_index = use_cached_index
-                repos.append(existing)
+            if index_url in settings.INDEX_URL:
+                repos.append(PypiSimpleRepository(index_url))
             else:
                 credentials = None
                 if parsed_netrc:
@@ -273,7 +270,7 @@ def resolve_dependencies(
                         dict(login=login,
                              password=password) if login and password else None
                     )
-                repo = utils_pypi.PypiSimpleRepository(
+                repo = PypiSimpleRepository(
                     index_url=index_url,
                     use_cached_index=use_cached_index,
                     credentials=credentials,
@@ -366,8 +363,8 @@ def resolve(
 
 def get_resolved_dependencies(
     requirements: List[Requirement],
-    environment: Environment = None,
-    repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(),
+    environment: Environment,
+    repos: Sequence[PypiSimpleRepository] = tuple(),
     as_tree: bool = False,
     max_rounds: int = 200000,
     pdt_output: bool = False,
@@ -382,6 +379,7 @@ def get_resolved_dependencies(
     Used the provided ``repos`` list of PypiSimpleRepository.
     If empty, use instead the PyPI.org JSON API exclusively instead
     """
+
     resolver = Resolver(
         provider=PythonInputProvider(
             environment=environment,
@@ -391,9 +389,12 @@ def get_resolved_dependencies(
         ),
         reporter=BaseReporter(),
     )
+
     resolver_results = resolver.resolve(
         requirements=requirements, max_rounds=max_rounds)
+
     package_list = get_package_list(results=resolver_results)
+
     if pdt_output:
         return (format_pdt_tree(resolver_results), package_list)
     return (

src/python_inspector/package_data.py

@@ -9,6 +9,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from collections.abc import Generator
 from typing import List
 
 from packageurl import PackageURL
@@ -25,7 +26,10 @@
 
 
 def get_pypi_data_from_purl(
-    purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
+    purl: str,
+    environment: Environment,
+    repos: list[PypiSimpleRepository],
+    prefer_source: bool,
 ) -> PackageData:
     """
     Generate `Package` object from the `purl` string of pypi type
@@ -36,9 +40,9 @@ def get_pypi_data_from_purl(
     ``prefer_source`` is a boolean value to prefer source distribution over wheel,
     if no source distribution is available then wheel is used
     """
-    purl = PackageURL.from_string(purl)
-    name = purl.name
-    version = purl.version
+    packageurl: PackageURL = PackageURL.from_string(purl)
+    name = packageurl.name
+    version = packageurl.version
     if not version:
         raise Exception("Version is not specified in the purl")
     base_path = "https://pypi.org/pypi"
@@ -54,12 +58,13 @@ def get_pypi_data_from_purl(
     code_view_url = get_pypi_codeview_url(project_urls)
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
     python_version = get_python_version_from_env_tag(
-        python_version=environment.python_version)
+        python_version=environment.python_version
+    )
     valid_distribution_urls = []
 
     valid_distribution_urls.append(
         get_sdist_download_url(
-            purl=purl,
+            purl=packageurl,
             repos=repos,
             python_version=python_version,
         )
@@ -70,7 +75,7 @@ def get_pypi_data_from_purl(
     if not valid_distribution_urls or not prefer_source:
         wheel_urls = list(
             get_wheel_download_urls(
-                purl=purl,
+                purl=packageurl,
                 repos=repos,
                 environment=environment,
                 python_version=python_version,
@@ -108,7 +113,7 @@ def get_pypi_data_from_purl(
                 maintainer_key="maintainer",
                 maintainer_email_key="maintainer_email",
             ),
-            **purl.to_dict(),
+            **packageurl.to_dict(),
         )
 
 
@@ -144,7 +149,7 @@ def get_wheel_download_urls(
     repos: List[PypiSimpleRepository],
     environment: Environment,
     python_version: str,
-) -> List[str]:
+) -> Generator[str, None, None]:
     """
     Return a list of download urls for the given purl.
     """