perf(vulnfeeds): make vulnfeeds tests go faster

vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go

@@ -8,6 +8,7 @@ import (
 	"flag"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"path/filepath"
 	"slices"
@@ -21,6 +22,7 @@ import (
 	"github.com/google/osv.dev/vulnfeeds/conversion/cve5"
 	"github.com/google/osv.dev/vulnfeeds/conversion/writer"
 	"github.com/google/osv.dev/vulnfeeds/gcs-tools"
+	"github.com/google/osv.dev/vulnfeeds/git"
 	"github.com/google/osv.dev/vulnfeeds/models"
 	"github.com/google/osv.dev/vulnfeeds/utility/logger"
 )
@@ -120,10 +122,11 @@ func main() {
 		logger.Info("GCS Upload Pool initialized", slog.String("bucket", *outputBucket))
 	}
 
+	repoTagsCache := git.NewRepoTagsCache()
 	// Start the worker pool.
 	for range *workers {
 		wg.Add(1)
-		go worker(&wg, jobs, gcsHelper, *localOutputDir, actualMetricsDir, cnaList, *rejectFailed, *outputMetrics, *gcsMetricsPrefix)
+		go worker(&wg, jobs, gcsHelper, *localOutputDir, actualMetricsDir, cnaList, *rejectFailed, *outputMetrics, *gcsMetricsPrefix, repoTagsCache)
 	}
 
 	// Discover files and send them to the workers.
@@ -180,7 +183,7 @@ func main() {
 }
 
 // worker is a function that processes CVE files from the jobs channel.
-func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDir string, metricsDir string, cnas []string, rejectFailed bool, outputMetrics bool, gcsMetricsPrefix string) {
+func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDir string, metricsDir string, cnas []string, rejectFailed bool, outputMetrics bool, gcsMetricsPrefix string, cache git.RepoTagsCache) {
 	defer wg.Done()
 	for path := range jobs {
 		data, err := os.ReadFile(path)
@@ -202,6 +205,8 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
 		logger.Info("Processing "+string(cveID), slog.String("cve", string(cveID)))
 		totalConversionsCount.Add(1)
 
+		httpClient := http.DefaultClient
+
 		sourceLink := ""
 		baseDirCVEList := "cves/" // The base folder for the CVEListV5 repository.
 		idx := strings.Index(path, baseDirCVEList)
@@ -211,7 +216,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
 		}
 
 		if gcsHelper != nil {
-			vuln, metrics := cve5.CVEToOSV(cve, sourceLink)
+			vuln, metrics := cve5.CVEToOSV(cve, sourceLink, cache, httpClient)
 			if metrics.Outcome == models.Successful {
 				successfulConversionsCount.Add(1)
 			}
@@ -263,7 +268,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
 			}
 
 			// Perform the conversion and export the results.
-			metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsSink, sourceLink)
+			metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsSink, sourceLink, cache, httpClient)
 			if err != nil {
 				logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 			} else {

vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go

@@ -9,8 +9,11 @@ import (
 
 	"github.com/google/osv.dev/vulnfeeds/conversion/cve5"
 	"github.com/google/osv.dev/vulnfeeds/conversion/writer"
+	"github.com/google/osv.dev/vulnfeeds/git"
 	"github.com/google/osv.dev/vulnfeeds/models"
 	"github.com/google/osv.dev/vulnfeeds/utility/logger"
+
+	"net/http"
 )
 
 var (
@@ -64,7 +67,9 @@ func main() {
 	}
 
 	// Perform the conversion and export the results.
-	if metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil {
+	cache := git.NewRepoTagsCache()
+	httpClient := http.DefaultClient
+	if metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, "", cache, httpClient); err != nil {
 		logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 	} else {
 		logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))

vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go

@@ -156,6 +156,7 @@ func main() {
 	}
 
 	repoTagsCache := git.NewRepoTagsCache()
+	httpClient := http.DefaultClient
 
 	var gcsHelper *gcs.Helper
 	ctx := context.Background()
@@ -174,7 +175,7 @@ func main() {
 
 	for range *workers {
 		wg.Add(1)
-		go worker(&wg, jobs, gcsHelper, *outDir, actualMetricsDir, *gcsMetricsPrefix, vpRepoCache, repoTagsCache)
+		go worker(&wg, jobs, gcsHelper, *outDir, actualMetricsDir, *gcsMetricsPrefix, vpRepoCache, repoTagsCache, httpClient)
 	}
 
 	for _, file := range files {
@@ -218,7 +219,7 @@ func main() {
 	logger.Info("NVD Conversion run complete")
 }
 
-func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache) (*vulns.Vulnerability, *models.ConversionMetrics, models.ConversionOutcome) {
+func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache, httpClient *http.Client) (*vulns.Vulnerability, *models.ConversionMetrics, models.ConversionOutcome) {
 	metrics := &models.ConversionMetrics{
 		CVEID: cve.ID,
 		CNA:   "nvd",
@@ -227,17 +228,17 @@ func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git
 	if nvd.IsLinuxKernelVulnerability(cve) {
 		metrics.AddNote("Linux kernel vulnerability detected, skipping repository processing")
 	} else {
-		repos = nvd.FindRepos(cve, vpRepoCache, repoTagsCache, metrics, http.DefaultClient)
+		repos = nvd.FindRepos(cve, vpRepoCache, repoTagsCache, metrics, httpClient)
 	}
 	metrics.Repos = repos
 
-	return nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics)
+	return nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics, httpClient)
 }
 
-func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, gcsHelper *gcs.Helper, outDir string, metricsDir string, gcsMetricsPrefix string, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache) {
+func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, gcsHelper *gcs.Helper, outDir string, metricsDir string, gcsMetricsPrefix string, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache, httpClient *http.Client) {
 	defer wg.Done()
 	for cve := range jobs {
-		vuln, metrics, outcome := processCVE(cve, vpRepoCache, repoTagsCache)
+		vuln, metrics, outcome := processCVE(cve, vpRepoCache, repoTagsCache, httpClient)
 		totalConversionsCount.Add(1)
 		cveID := string(cve.ID)
 		if outcome == models.Error {

vulnfeeds/cmd/mirrors/cpe-repo-gen/main.go

@@ -36,6 +36,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -317,7 +318,7 @@ func MaybeGetSourceRepoFromDebian(mdir string, pkg string) string {
 }
 
 // Analyze CPE Dictionary and return a product-to-repo map and a reference description frequency table.
-func analyzeCPEDictionary(cpes []CPE) (productToRepo c.VendorProductToRepoMap, descriptionFrequency map[string]int) {
+func analyzeCPEDictionary(cpes []CPE, httpClient *http.Client) (productToRepo c.VendorProductToRepoMap, descriptionFrequency map[string]int) {
 	productToRepo = make(c.VendorProductToRepoMap)
 	descriptionFrequency = make(map[string]int)
 	MaybeTryDebian := make(map[c.VendorProduct]bool)
@@ -385,7 +386,7 @@ func analyzeCPEDictionary(cpes []CPE) (productToRepo c.VendorProductToRepoMap, d
 					logger.Info("Disregarding derived repo", slog.String("repo", repo), slog.String("vendor", vp.Vendor), slog.String("product", vp.Product), slog.Any("err", err))
 					continue
 				}
-				if valid, _ := git.ValidRepoAndHasUsableRefs(repo); !valid {
+				if valid, _ := git.ValidRepoAndHasUsableRefs(repo, httpClient); !valid {
 					logger.Info("Disregarding derived repo as unusable", slog.String("repo", repo), slog.String("vendor", vp.Vendor), slog.String("product", vp.Product))
 					continue
 				}
@@ -398,7 +399,7 @@ func analyzeCPEDictionary(cpes []CPE) (productToRepo c.VendorProductToRepoMap, d
 }
 
 // validateRepos takes a VendorProductToRepoMap and removes any entries where the repository fails remote validation.
-func validateRepos(prm c.VendorProductToRepoMap) (validated c.VendorProductToRepoMap) {
+func validateRepos(prm c.VendorProductToRepoMap, httpClient *http.Client) (validated c.VendorProductToRepoMap) {
 	validated = make(c.VendorProductToRepoMap)
 	logger.Info("Validating repos", slog.Int("products", len(prm)))
 	// This is likely to be time consuming, so give an impatient log watcher something to gauge progress by.
@@ -407,7 +408,7 @@ func validateRepos(prm c.VendorProductToRepoMap) (validated c.VendorProductToRep
 		entryCount++
 		// As a side-effect, this also omits any with no repos.
 		for _, r := range prm[vp] {
-			if valid, _ := git.ValidRepoAndHasUsableRefs(r); !valid {
+			if valid, _ := git.ValidRepoAndHasUsableRefs(r, httpClient); !valid {
 				logger.Info("Invalid repo", slog.Int("count", entryCount), slog.Int("total", len(prm)), slog.String("repo", r), slog.String("vendor", vp.Vendor), slog.String("product", vp.Product))
 				continue
 			}
@@ -435,9 +436,10 @@ func main() {
 		logger.Fatal("Failed to load CPEs", slog.String("path", *CPEDictionaryDir), slog.Any("err", err))
 	}
 
-	productToRepo, descriptionFrequency := analyzeCPEDictionary(cpes)
+	httpClient := http.DefaultClient
+	productToRepo, descriptionFrequency := analyzeCPEDictionary(cpes, httpClient)
 	if *Validate {
-		productToRepo = validateRepos(productToRepo)
+		productToRepo = validateRepos(productToRepo, httpClient)
 	}
 
 	mappingFile, err := os.Create(filepath.Join(*OutputDir, "cpe_product_to_repo.json"))

vulnfeeds/conversion/common.go

@@ -171,7 +171,7 @@ func ConductAnalysisAndUpload(prefix string, year string, metricsDir string, csv
 
 // GitVersionsToCommits examines repos and tries to convert versions to commits by treating them as Git tags.
 // Returns the resolved ranges, unresolved ranges, and successful repos involved.
-func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
+func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache, httpClient *http.Client) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
 	var newVersionRanges []models.RangeWithMetadata
 	unresolvedRanges := versionRanges
 	var successfulRepos []string
@@ -183,7 +183,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 	for _, vr := range versionRanges {
 		if vr.Range.GetRepo() != "" {
 			claimedRepos[vr.Range.GetRepo()] = true // Always claim the raw repository URL.
-			canonicalRepo, err := git.FindCanonicalLink(vr.Range.GetRepo(), http.DefaultClient, cache)
+			canonicalRepo, err := git.FindCanonicalLink(vr.Range.GetRepo(), httpClient, cache)
 			if err != nil {
 				if git.IsRateLimit(err) {
 					metrics.Outcome = models.Error
@@ -204,7 +204,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 			continue
 		}
 
-		repo, err := git.FindCanonicalLink(repo, http.DefaultClient, cache)
+		repo, err := git.FindCanonicalLink(repo, httpClient, cache)
 		if err != nil {
 			metrics.AddNote("Failed to find canonical link - %s %v", repo, err)
 			if git.IsRateLimit(err) {
@@ -215,7 +215,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 			continue
 		}
 
-		normalizedTags, err := git.NormalizeRepoTags(repo, cache)
+		normalizedTags, err := git.NormalizeRepoTags(repo, cache, httpClient)
 		if err != nil {
 			if git.IsRateLimit(err) {
 				metrics.Outcome = models.Error
@@ -230,7 +230,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 		for _, vr := range unresolvedRanges {
 			vRepo := vr.Range.GetRepo()
 			if vRepo != "" {
-				canonicalVRepo, err := git.FindCanonicalLink(vRepo, http.DefaultClient, cache)
+				canonicalVRepo, err := git.FindCanonicalLink(vRepo, httpClient, cache)
 				if err != nil {
 					if git.IsRateLimit(err) {
 						metrics.Outcome = models.Error
@@ -729,12 +729,12 @@ func AddFieldToDatabaseSpecific(ds *structpb.Struct, field string, value any) er
 }
 
 // ProcessRanges attempts to resolve the given ranges to commits and updates the metrics accordingly.
-func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
+func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache, httpClient *http.Client) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
 	if len(ranges) == 0 {
 		return nil, nil, nil
 	}
 
-	r, un, sR := GitVersionsToCommits(ranges, repos, metrics, cache)
+	r, un, sR := GitVersionsToCommits(ranges, repos, metrics, cache, httpClient)
 	if len(r) > 0 {
 		metrics.ResolvedRangesCount += len(r)
 		metrics.SetOutcome(models.Successful)

vulnfeeds/conversion/common_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -729,25 +730,25 @@ func TestGitVersionsToCommits_Canonicalization(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			oldRedisHost := os.Getenv("REDISHOST")
-			os.Unsetenv("REDISHOST")
-			defer func() {
-				if oldRedisHost != "" {
-					t.Setenv("REDISHOST", oldRedisHost)
-				}
-			}()
-
-			cache := git.NewRepoTagsCache()
-			for k, v := range tt.canonicalLinks {
-				cache.SetCanonicalLink(k, v)
-			}
-			for k, v := range tt.cachedTags {
-				cache.Set(k, v)
+		oldRedisHost := os.Getenv("REDISHOST")
+		os.Unsetenv("REDISHOST")
+		defer func() {
+			if oldRedisHost != "" {
+				t.Setenv("REDISHOST", oldRedisHost)
 			}
+		}()
 
+		cache := git.NewRepoTagsCache()
+		for k, v := range tt.canonicalLinks {
+			cache.SetCanonicalLink(k, v)
+		}
+		for k, v := range tt.cachedTags {
+			cache.Set(k, v)
+		}
+		httpClient := http.DefaultClient
+		t.Run(tt.name, func(t *testing.T) {
 			metrics := &models.ConversionMetrics{}
-			gotResolved, gotUnresolved, gotSuccessful := GitVersionsToCommits(tt.versionRanges, tt.repos, metrics, cache)
+			gotResolved, gotUnresolved, gotSuccessful := GitVersionsToCommits(tt.versionRanges, tt.repos, metrics, cache, httpClient)
 
 			if len(gotResolved) != tt.wantResolved {
 				t.Errorf("GitVersionsToCommits() gotResolved count = %v, want %v", len(gotResolved), tt.wantResolved)