From 9c504e7eb7f5963234c49181aaee9d50719ba32b Mon Sep 17 00:00:00 2001 From: Oscar V Date: Tue, 1 Sep 2026 09:33:57 -0700 Subject: [PATCH] Search by name only when asked, and only when nothing else answered Run against the extracted express@4.18.2 tarball with no --enable-fuzzy, src2purl returned the right answer and ten pieces of npm spam: exact 0.85 pkg:npm/express@4.18.2 express fuzzy 0.83 None senam5jari-975 fuzzy 0.83 None voiti-1xbet-rech-idet-urxfhkrm ... eight more Two hundredths of confidence separate express from the noise, which is close enough that --confidence-threshold cannot tell them apart, and the noise carries no PURL, so nothing downstream can act on it either. It is expensive to distinguish from signal and nobody asked for it. Keyword search matches on the project name, so it answers with whatever else shares that name. There were two call sites. The one in the fuzzy fallback checked the flag. The one in phase 1 did not check anything, and ran before the manifest parsing in phase 2 that produces the exact match. With Software Heritage off, which is the default, phase 1 finds nothing, so the search fired every time and phase 2 then found express beside its results. The flag's help says fuzzy runs when exact matches fail; whether they failed was not known yet. Both are gone. identify_packages runs the search once, after everything else has had its turn, when nothing was identified and the flag was passed. That is what the help already described. express, no flag 1 exact express, --enable-fuzzy 1 exact unidentifiable dir, no flag nothing unidentifiable dir, fuzzy 10 keyword matches The issue also notes express coming back with "official": false. is_official_organization checks a hardcoded list of about thirty organisations that does not include expressjs, so the field means "in our known-official list" rather than "is the canonical repository". Widening that list, or replacing it, is a separate decision and is left alone here. Closes #59 --- CHANGELOG.md | 14 +++ src2id/core/orchestrator.py | 45 +++++---- tests/unit/test_fuzzy_gating.py | 165 ++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_fuzzy_gating.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a20984f..eedc4e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- Keyword search runs only when `--enable-fuzzy` was passed, and only when + nothing else identified the source (#59). It ran in phase 1, before the + manifest parsing in phase 2 that produces the exact match, so "when exact + matches fail", which is what the flag promises, was decided before the step + that finds the match had run. Against express@4.18.2 that returned the correct + match at 0.85 beside ten npm packages at 0.83, close enough that + `--confidence-threshold` cannot separate them, and carrying no PURL so nothing + downstream could act on them either. +- The same fallback inside the Software Heritage path is removed rather than + gated, so keyword search now happens once, in one place, for every route. + ## [1.3.5] - 2026-07-24 ### Fixed diff --git a/src2id/core/orchestrator.py b/src2id/core/orchestrator.py index cf3ce74..b979559 100644 --- a/src2id/core/orchestrator.py +++ b/src2id/core/orchestrator.py @@ -141,16 +141,6 @@ async def identify_packages(self, path: Path, enhance_licenses: bool = True) -> if self.config.verbose: console.print("[dim]Skipping Software Heritage (use --use-swh to enable)[/dim]") - if not all_matches: - # Try keyword search with GitHub and SCANOSS - if self.config.verbose: - console.print("[yellow]Trying web search (GitHub, SCANOSS)[/yellow]") - - # Try keyword search - keyword_matches = await self._find_keyword_matches(path) - if keyword_matches: - all_matches = keyword_matches - # Step 1b: Process hash-based matches if all_matches: hash_based_matches = await self._process_matches(all_matches) @@ -178,6 +168,27 @@ async def identify_packages(self, path: Path, enhance_licenses: bool = True) -> # Final deduplication and sorting final_matches = self._prioritize_and_deduplicate(enhanced_matches) + + # Keyword search is the last resort, and only when it was asked for. + # It searches on the project name, so it answers with whatever else + # shares that name: running it for express returned ten npm packages + # scoring 0.83 against the real one at 0.85, close enough that a + # confidence threshold cannot separate them, and carrying no PURL so + # nothing downstream can act on them either. + # + # It also belongs here rather than in phase 1. The flag says fuzzy + # runs when exact matches fail, and whether they failed is not known + # until the manifest parsing in phase 2 has had its turn. + if not final_matches and self.config.enable_fuzzy_matching: + if self.config.verbose: + console.print( + "[yellow]Nothing identified - trying keyword search[/yellow]" + ) + keyword_matches = await self._find_keyword_matches(path) + if keyword_matches: + final_matches = self._prioritize_and_deduplicate( + await self._process_matches(keyword_matches) + ) # Step 5: Optionally enhance with oslili license detection if enhance_licenses: @@ -588,14 +599,12 @@ async def _find_matches(self, dir_candidates: List[DirectoryCandidate], file_can for candidate in non_existing_dirs: console.print(f"[yellow]✗ No match for {candidate.path.name} ({candidate.swhid[:12]}...)[/yellow]") - # Try keyword search fallback if no exact matches and fuzzy is enabled - if not all_matches and self.config.enable_fuzzy_matching: - if self.config.verbose: - console.print("[yellow]No exact matches found - trying keyword search[/yellow]") - # Only try keyword search if fuzzy matching is enabled - keyword_matches = await self._find_keyword_matches(dir_candidates[0].path if dir_candidates else Path(".")) - all_matches.extend(keyword_matches) - + # Keyword search is not run here. It used to be, and that put it before + # the manifest parsing in phase 2, so "when exact matches fail", which + # is what the flag promises, was decided before the step that finds the + # exact match had run. identify_packages runs it once, after everything + # else has had its turn. + # Report match summary if self.config.verbose and (child_matches > 0 or parent_matches > 0): console.print("\n[bold]Match Summary:[/bold]") diff --git a/tests/unit/test_fuzzy_gating.py b/tests/unit/test_fuzzy_gating.py new file mode 100644 index 0000000..51c1cbf --- /dev/null +++ b/tests/unit/test_fuzzy_gating.py @@ -0,0 +1,165 @@ +"""Keyword search runs only when asked for, and only when nothing else answered. + +Issue #59. Run against the extracted express@4.18.2 tarball without +--enable-fuzzy, src2purl returned the correct express match at 0.85 alongside +ten npm packages at 0.83: + + exact 0.85 pkg:npm/express@4.18.2 express + fuzzy 0.83 None senam5jari-975 + fuzzy 0.83 None voiti-1xbet-rech-idet-urxfhkrm + ... + +Two things were wrong. The flag says fuzzy runs "when exact matches fail" and it +was never consulted at this call site, and the search ran in phase 1, before the +manifest parsing in phase 2 that produces the exact match, so "did the exact +match fail" could not have been known yet. + +The 0.02 gap between the right answer and npm spam is what makes it costly: +anything consuming by --confidence-threshold gets both, and the spam carries +purl: null so it cannot be acted on downstream either. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +from src2id.core.config import SWHPIConfig +from src2id.core.models import MatchType, PackageMatch +from src2id.core.orchestrator import SHPackageIdentifier + + +def _manifest_match(): + """What phase 2 finds for express, from package.json.""" + return PackageMatch( + name="express", + version="4.18.2", + confidence_score=0.85, + match_type=MatchType.EXACT, + download_url="https://www.npmjs.com/package/express", + purl="pkg:npm/express@4.18.2", + license="MIT", + is_official_org=False, + ) + + +def _spam_match(name): + """What keyword search finds: anything sharing the project name.""" + return PackageMatch( + name=name, + version=None, + confidence_score=0.83, + match_type=MatchType.FUZZY, + download_url=f"https://www.npmjs.com/package/{name}", + purl=None, + license="MIT", + is_official_org=False, + ) + + +def _identifier(enable_fuzzy, manifest, keyword): + config = SWHPIConfig(enable_fuzzy_matching=enable_fuzzy, use_swh=False) + identifier = SHPackageIdentifier(config) + identifier._scan_directories = AsyncMock(return_value=([MagicMock()], [])) + identifier._extract_with_upmex = MagicMock(return_value=list(manifest)) + identifier._find_keyword_matches = AsyncMock(return_value=[MagicMock() for _ in keyword]) + identifier._process_matches = AsyncMock(return_value=list(keyword)) + return identifier + + +def _run(identifier, tmp_path): + return asyncio.run(identifier.identify_packages(tmp_path, enhance_licenses=False)) + + +class TestFuzzyIsGatedOnTheFlag: + def test_not_requested_means_no_keyword_search(self, tmp_path): + """The reported case: an exact match, and ten fuzzy ones nobody asked for.""" + identifier = _identifier( + enable_fuzzy=False, + manifest=[_manifest_match()], + keyword=[_spam_match("senam5jari-975"), _spam_match("voiti-1xbet-rech-idet-urxfhkrm")], + ) + + result = _run(identifier, tmp_path) + + identifier._find_keyword_matches.assert_not_called() + assert [m.name for m in result] == ["express"] + assert all(m.match_type is not MatchType.FUZZY for m in result) + + def test_not_requested_and_nothing_found_still_means_no_keyword_search(self, tmp_path): + identifier = _identifier(enable_fuzzy=False, manifest=[], keyword=[_spam_match("whatever")]) + + result = _run(identifier, tmp_path) + + identifier._find_keyword_matches.assert_not_called() + assert result == [] + + +class TestFuzzyRunsOnlyWhenExactFailed: + def test_requested_but_something_answered_means_no_keyword_search(self, tmp_path): + """ "when exact matches fail" cannot be judged before phase 2 has run.""" + identifier = _identifier( + enable_fuzzy=True, + manifest=[_manifest_match()], + keyword=[_spam_match("senam5jari-975")], + ) + + result = _run(identifier, tmp_path) + + identifier._find_keyword_matches.assert_not_called() + assert [m.name for m in result] == ["express"] + + def test_requested_and_nothing_answered_means_keyword_search_runs(self, tmp_path): + identifier = _identifier( + enable_fuzzy=True, manifest=[], keyword=[_spam_match("something-alike")] + ) + + result = _run(identifier, tmp_path) + + identifier._find_keyword_matches.assert_called_once() + assert [m.name for m in result] == ["something-alike"] + + +class TestTheSoftwareHeritagePathIsGatedToo: + """--use-swh took a different route with its own ungated keyword search.""" + + def _identifier(self, enable_fuzzy, manifest, keyword, sh_matches=None): + config = SWHPIConfig(enable_fuzzy_matching=enable_fuzzy, use_swh=True) + identifier = SHPackageIdentifier(config) + identifier._scan_directories = AsyncMock(return_value=([MagicMock()], [])) + identifier._find_matches = AsyncMock(return_value=list(sh_matches or [])) + identifier._extract_with_upmex = MagicMock(return_value=list(manifest)) + identifier._find_keyword_matches = AsyncMock(return_value=[MagicMock() for _ in keyword]) + identifier._process_matches = AsyncMock(return_value=list(keyword)) + return identifier + + def test_no_keyword_search_when_the_manifest_answered(self, tmp_path): + """Software Heritage found nothing, but package.json did.""" + identifier = self._identifier( + enable_fuzzy=True, + manifest=[_manifest_match()], + keyword=[_spam_match("senam5jari-975")], + ) + + result = asyncio.run(identifier.identify_packages(tmp_path, enhance_licenses=False)) + + identifier._find_keyword_matches.assert_not_called() + assert [m.name for m in result] == ["express"] + + def test_no_keyword_search_without_the_flag(self, tmp_path): + identifier = self._identifier( + enable_fuzzy=False, manifest=[], keyword=[_spam_match("whatever")] + ) + + result = asyncio.run(identifier.identify_packages(tmp_path, enhance_licenses=False)) + + identifier._find_keyword_matches.assert_not_called() + assert result == [] + + def test_keyword_search_runs_when_asked_and_nothing_answered(self, tmp_path): + identifier = self._identifier( + enable_fuzzy=True, manifest=[], keyword=[_spam_match("something-alike")] + ) + + result = asyncio.run(identifier.identify_packages(tmp_path, enhance_licenses=False)) + + identifier._find_keyword_matches.assert_called_once() + assert [m.name for m in result] == ["something-alike"]