Support custom Artifactory repositories for package metadata

When using --index-url with a custom Artifactory repository, dependency resolution works but the packages array comes back empty. This happens because get_pypi_data_from_purl() hardcodes https://pypi.org/pypi for the JSON API endpoint. Internal packages that don't exist on PyPI.org return 404 and are silently skipped.

The fix includes deriving the JSON API base URL from the provided repository instead of hardcoding PyPI.org
It is also necessary to match distribution files by filename (standardized per PEP 427/491) instead of full URL, since URL paths can differ between Simple API and JSON API endpoints.

Signed-off-by: Kai Hodžić <hodzic.e.k@outlook.com>

Author: voidpetal

CHANGELOG.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+v0.15.1
+-----------
+
+- Fix package metadata fetch for Artifactory repositories
+
 v0.15.0
 -----------
 

src/python_inspector/package_data.py

@@ -9,9 +9,12 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import posixpath
 from typing import Dict
 from typing import List
 from typing import Optional
+from urllib.parse import urljoin
+from urllib.parse import urlparse
 
 from packageurl import PackageURL
 
@@ -30,67 +33,85 @@ async def get_pypi_data_from_purl(
     purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
 ) -> Optional[PackageData]:
     """
-    Generate `Package` object from the `purl` string of pypi type
+    Return a PackageData object from the ``purl`` string of pypi type.
 
-    ``purl`` is a package-url of pypi type
-    ``environment`` is a `Environment` object defaulting Python version 3.8 and linux OS
-    ``repos`` is a list of `PypiSimpleRepository` objects
-    ``prefer_source`` is a boolean value to prefer source distribution over wheel,
-    if no source distribution is available then wheel is used
+    ``environment`` is an Environment object with Python version and OS.
+    ``repos`` is a list of PypiSimpleRepository objects to try in order.
+    ``prefer_source`` prefers source distribution over wheel when True.
     """
+    from python_inspector.utils import get_response_async
+
     parsed_purl = PackageURL.from_string(purl)
     name = parsed_purl.name
     version = parsed_purl.version
     if not version:
         raise Exception("Version is not specified in the purl")
-    base_path = "https://pypi.org/pypi"
-    api_url = f"{base_path}/{name}/{version}/json"
-
-    from python_inspector.utils import get_response_async
-
-    response = await get_response_async(api_url)
-    if not response:
-        return None
 
-    info = response.get("info") or {}
-    homepage_url = info.get("home_page")
-    project_urls = info.get("project_urls") or {}
-    code_view_url = get_pypi_codeview_url(project_urls)
-    bug_tracking_url = get_pypi_bugtracker_url(project_urls)
     python_version = get_python_version_from_env_tag(python_version=environment.python_version)
+
+    # Collect distribution URLs from repos
     valid_distribution_urls = []
     sdist_url = await get_sdist_download_url(
         purl=parsed_purl, repos=repos, python_version=python_version
     )
     if sdist_url:
         valid_distribution_urls.append(sdist_url)
 
-    valid_distribution_urls = [url for url in valid_distribution_urls if url]
-
-    # if prefer_source is True then only source distribution is used
-    # in case of no source distribution available then wheel is used
     if not valid_distribution_urls or not prefer_source:
-        wheel_urls = [
-            item
-            for item in await get_wheel_download_urls(
+        wheel_urls = list(
+            await get_wheel_download_urls(
                 purl=parsed_purl,
                 repos=repos,
                 environment=environment,
                 python_version=python_version,
             )
-        ]
+        )
         wheel_url = choose_single_wheel(wheel_urls)
         if wheel_url:
             valid_distribution_urls.insert(0, wheel_url)
 
-    urls = {url.get("url"): url for url in response.get("urls") or []}
-    # iterate over the valid distribution urls and return the first
-    # one that is matching.
+    # Build list of JSON API URLs to try: each repo's /pypi endpoint, then PyPI.org as fallback
+    api_urls = []
+    for repo in repos:
+        # Convert /simple to /pypi for Artifactory-style JSON API
+        base_path = repo.index_url.replace("/simple", "/pypi")
+        api_urls.append(f"{base_path}/{name}/{version}/json")
+    api_urls.append(f"https://pypi.org/pypi/{name}/{version}/json")
+
+    # Try each API URL until one succeeds
+    response = None
+    api_url = None
+    for url in api_urls:
+        response = await get_response_async(url)
+        if response:
+            api_url = url
+            break
+
+    if not response:
+        return None
+
+    info = response.get("info") or {}
+    homepage_url = info.get("home_page")
+    project_urls = info.get("project_urls") or {}
+    code_view_url = get_pypi_codeview_url(project_urls)
+    bug_tracking_url = get_pypi_bugtracker_url(project_urls)
+
+    # Index by filename for matching (paths differ between /simple and /pypi endpoints)
+    urls_by_filename = {}
+    for url_entry in response.get("urls") or []:
+        url = url_entry.get("url")
+        if url:
+            absolute_url = urljoin(api_url, url)
+            filename = posixpath.basename(urlparse(absolute_url).path)
+            urls_by_filename[filename] = url_entry
+
+    # Match distribution URLs by filename
     for dist_url in valid_distribution_urls:
-        if dist_url not in urls:
+        filename = posixpath.basename(urlparse(dist_url).path)
+        url_data = urls_by_filename.get(filename)
+        if not url_data:
             continue
 
-        url_data = urls.get(dist_url)
         digests = url_data.get("digests") or {}
 
         return PackageData(

tests/test_package_data.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/python-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+from unittest import mock
+
+import pytest
+from commoncode.testcase import FileDrivenTesting
+
+from python_inspector.package_data import get_pypi_data_from_purl
+from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
+
+test_env = FileDrivenTesting()
+test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data")
+
+
+@pytest.mark.asyncio
+@mock.patch("python_inspector.package_data.get_sdist_download_url")
+@mock.patch("python_inspector.package_data.get_wheel_download_urls")
+@mock.patch("python_inspector.utils.get_response_async")
+async def test_get_pypi_data_from_purl_tries_repos_in_order(
+    mock_get_response, mock_get_wheels, mock_get_sdist
+):
+    mock_get_sdist.return_value = None
+    mock_get_wheels.return_value = []
+
+    call_urls = []
+
+    async def track_calls(url):
+        call_urls.append(url)
+        return None
+
+    mock_get_response.side_effect = track_calls
+
+    repo1 = PypiSimpleRepository(index_url="https://repo1.example.com/simple")
+    repo2 = PypiSimpleRepository(index_url="https://repo2.example.com/simple")
+    env = Environment(python_version="310", operating_system="linux")
+
+    await get_pypi_data_from_purl(
+        purl="pkg:pypi/requests@2.28.0",
+        environment=env,
+        repos=[repo1, repo2],
+        prefer_source=False,
+    )
+
+    assert call_urls == [
+        "https://repo1.example.com/pypi/requests/2.28.0/json",
+        "https://repo2.example.com/pypi/requests/2.28.0/json",
+        "https://pypi.org/pypi/requests/2.28.0/json",
+    ]
+
+
+@pytest.mark.asyncio
+@mock.patch("python_inspector.package_data.get_sdist_download_url")
+@mock.patch("python_inspector.package_data.get_wheel_download_urls")
+@mock.patch("python_inspector.utils.get_response_async")
+async def test_get_pypi_data_from_purl_stops_on_first_success(
+    mock_get_response, mock_get_wheels, mock_get_sdist
+):
+    mock_get_sdist.return_value = None
+    mock_get_wheels.return_value = []
+
+    call_urls = []
+
+    async def return_success_on_second(url):
+        call_urls.append(url)
+        if "repo2" in url:
+            return {"info": {}, "urls": []}
+        return None
+
+    mock_get_response.side_effect = return_success_on_second
+
+    repo1 = PypiSimpleRepository(index_url="https://repo1.example.com/simple")
+    repo2 = PypiSimpleRepository(index_url="https://repo2.example.com/simple")
+    env = Environment(python_version="310", operating_system="linux")
+
+    await get_pypi_data_from_purl(
+        purl="pkg:pypi/requests@2.28.0",
+        environment=env,
+        repos=[repo1, repo2],
+        prefer_source=False,
+    )
+
+    assert call_urls == [
+        "https://repo1.example.com/pypi/requests/2.28.0/json",
+        "https://repo2.example.com/pypi/requests/2.28.0/json",
+    ]
+
+
+@pytest.mark.asyncio
+@mock.patch("python_inspector.package_data.get_sdist_download_url")
+@mock.patch("python_inspector.package_data.get_wheel_download_urls")
+@mock.patch("python_inspector.utils.get_response_async")
+async def test_get_pypi_data_from_purl_falls_back_to_pypi_org(
+    mock_get_response, mock_get_wheels, mock_get_sdist
+):
+    mock_get_sdist.return_value = None
+    mock_get_wheels.return_value = []
+
+    call_urls = []
+
+    async def track_calls(url):
+        call_urls.append(url)
+        return None
+
+    mock_get_response.side_effect = track_calls
+
+    env = Environment(python_version="310", operating_system="linux")
+
+    await get_pypi_data_from_purl(
+        purl="pkg:pypi/requests@2.28.0",
+        environment=env,
+        repos=[],
+        prefer_source=False,
+    )
+
+    assert call_urls == ["https://pypi.org/pypi/requests/2.28.0/json"]
+
+
+@pytest.mark.asyncio
+@mock.patch("python_inspector.package_data.get_sdist_download_url")
+@mock.patch("python_inspector.package_data.get_wheel_download_urls")
+@mock.patch("python_inspector.utils.get_response_async")
+async def test_get_pypi_data_from_purl_matches_by_filename(
+    mock_get_response, mock_get_wheels, mock_get_sdist
+):
+    mock_get_sdist.return_value = None
+    mock_get_wheels.return_value = [
+        "https://repo.example.com/simple/../packages/ab/cd/requests-2.28.0-py3-none-any.whl"
+    ]
+
+    async def return_json_response(url):
+        if "pypi" in url:
+            return {
+                "info": {"name": "requests", "version": "2.28.0"},
+                "urls": [
+                    {
+                        "url": "../../packages/xy/zz/requests-2.28.0-py3-none-any.whl",
+                        "digests": {"sha256": "abc123", "md5": "def456"},
+                        "size": 12345,
+                    }
+                ],
+            }
+        return None
+
+    mock_get_response.side_effect = return_json_response
+
+    repo = PypiSimpleRepository(index_url="https://repo.example.com/simple")
+    env = Environment(python_version="310", operating_system="linux")
+
+    result = await get_pypi_data_from_purl(
+        purl="pkg:pypi/requests@2.28.0",
+        environment=env,
+        repos=[repo],
+        prefer_source=False,
+    )
+
+    assert result is not None
+    assert result.sha256 == "abc123"
+    assert result.md5 == "def456"
+    assert result.size == 12345