Merge branch 'main' into nginx-migration

Author: ziadhany

docs/source/contributing.rst

@@ -17,7 +17,7 @@ resources to help you get started.
 Do Your Homework
 ----------------
 
-Before adding a contribution or create a new issue, take a look at the project’s
+Before adding a contribution or create a new issue, take a look at the project's
 `README <https://github.com/aboutcode-org/vulnerablecode>`_, read through our
 `documentation <https://vulnerablecode.readthedocs.io/en/latest/>`_,
 and browse existing `issues <https://github.com/aboutcode-org/vulnerablecode/issues>`_,
@@ -73,7 +73,7 @@ overlooked. We value any suggestions to improve
 
 .. tip::
     Our documentation is treated like code. Make sure to check our
-    `writing guidelines <https://scancode-toolkit.readthedocs.io/en/latest/contribute/contrib_doc.html>`_
+    `writing guidelines <https://scancode-toolkit.readthedocs.io/en/stable/getting-started/contribute/contributing-docs.html>`_
     to help guide new users.
 
 Other Ways
@@ -87,7 +87,7 @@ questions, and interact with us and other community members on
 Helpful Resources
 -----------------
 
-- Review our `comprehensive guide <https://scancode-toolkit.readthedocs.io/en/latest/contribute/index.html>`_
+- Review our `comprehensive guide <https://scancode-toolkit.readthedocs.io/en/stable/getting-started/contribute/index.html>`_
   for more details on how to add quality contributions to our codebase and documentation
 - Check this free resource on `How to contribute to an open source project on github <https://egghead.io/lessons/javascript-identifying-how-to-contribute-to-an-open-source-project-on-github>`_
 - Follow `this wiki page <https://aboutcode.readthedocs.io/en/latest/contributing/writing_good_commit_messages.html>`_

docs/source/tutorial_add_importer_pipeline.rst

@@ -152,7 +152,7 @@ At this point, an example importer will look like this:
 .. code-block:: python
     :caption: vulnerabilities/pipelines/example_importer.py
     :linenos:
-    :emphasize-lines: 16-17, 20-21, 23-24
+    :emphasize-lines: 17-18, 21-22, 24-25
 
     from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
 
@@ -165,6 +165,7 @@ At this point, an example importer will look like this:
         license_url = "https://exmaple.org/license/"
         spdx_license_expression = "CC-BY-4.0"
         importer_name = "Example Importer"
+        run_once = False
 
         @classmethod
         def steps(cls):
@@ -196,7 +197,7 @@ version management from `univers <https://github.com/aboutcode-org/univers>`_.
 .. code-block:: python
     :caption: vulnerabilities/pipelines/example_importer.py
     :linenos:
-    :emphasize-lines: 34-35, 37-40
+    :emphasize-lines: 35-36, 38-41
 
     from datetime import datetime
     from datetime import timezone
@@ -223,6 +224,7 @@ version management from `univers <https://github.com/aboutcode-org/univers>`_.
         license_url = "https://example.org/license/"
         spdx_license_expression = "CC-BY-4.0"
         importer_name = "Example Importer"
+        run_once = False
 
         @classmethod
         def steps(cls):
@@ -303,6 +305,17 @@ version management from `univers <https://github.com/aboutcode-org/univers>`_.
    Implement ``on_failure`` to handle cleanup in case of pipeline failure.
    Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage.
 
+.. tip::
+
+   Set ``run_once`` to ``True`` if pipeline is meant to be run once.
+
+   - To rerun onetime pipeline, reset ``is_active`` to ``True`` via a migration, pipeline will
+     run one more time and then deactivate.
+
+   - To convert a onetime pipeline to a regular pipeline, set the ``run_once`` class variable
+     to ``False`` and reset ``is_active` field to ``True`` via a migration.
+
+
 .. note::
 
    | Use ``make valid`` to format your new code using black and isort automatically.

vulnerabilities/importer.py

@@ -512,24 +512,30 @@ def from_dict(cls, affected_pkg: dict):
         fixed_version_range = None
         affected_range = affected_pkg["affected_version_range"]
         fixed_range = affected_pkg["fixed_version_range"]
-        introduced_by_commit_patches = (
-            affected_pkg.get("introduced_by_package_commit_patches") or []
-        )
-        fixed_by_commit_patches = affected_pkg.get("fixed_by_package_commit_patches") or []
+        introduced_by_commit_patches = affected_pkg.get("introduced_by_commit_patches") or []
+        fixed_by_commit_patches = affected_pkg.get("fixed_by_commit_patches") or []
 
         try:
-            affected_version_range = VersionRange.from_string(affected_range)
-            fixed_version_range = VersionRange.from_string(fixed_range)
+            affected_version_range = (
+                VersionRange.from_string(affected_range) if affected_range else None
+            )
+            fixed_version_range = VersionRange.from_string(fixed_range) if fixed_range else None
         except:
             tb = traceback.format_exc()
             logger.error(
                 f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}"
             )
             return
 
-        if not fixed_version_range and not affected_version_range:
+        if (
+            not fixed_version_range
+            and not affected_version_range
+            and not introduced_by_commit_patches
+            and not fixed_by_commit_patches
+        ):
             logger.error(
-                f"Cannot create AffectedPackage without fixed or affected range: {affected_pkg!r}"
+                f"Cannot create an AffectedPackage for: {affected_pkg!r}, at least one of the following must be provided: "
+                "a fixed version range, an affected version range, introduced commit patches, or fixed commit patches"
             )
             return
 
@@ -575,6 +581,10 @@ class AdvisoryData:
     original_advisory_text: Optional[str] = None
 
     def __post_init__(self):
+        if self.advisory_id and self.advisory_id in self.aliases:
+            raise ValueError(
+                f"advisory_id {self.advisory_id} should not be present in aliases {self.aliases}"
+            )
         if self.summary:
             self.summary = self.clean_summary(self.summary)
 

vulnerabilities/importers/__init__.py

@@ -60,6 +60,12 @@
 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
 from vulnerabilities.pipelines.v2_importers import oss_fuzz as oss_fuzz_v2
 from vulnerabilities.pipelines.v2_importers import postgresql_importer as postgresql_importer_v2
+from vulnerabilities.pipelines.v2_importers import (
+    project_kb_msr2019_importer as project_kb_msr2019_importer_v2,
+)
+from vulnerabilities.pipelines.v2_importers import (
+    project_kb_statements_importer as project_kb_statements_importer_v2,
+)
 from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2
@@ -88,6 +94,8 @@
         github_osv_importer_v2.GithubOSVImporterPipeline,
         redhat_importer_v2.RedHatImporterPipeline,
         aosp_importer_v2.AospImporterPipeline,
+        project_kb_statements_importer_v2.ProjectKBStatementsPipeline,
+        project_kb_msr2019_importer_v2.ProjectKBMSR2019Pipeline,
         ruby_importer_v2.RubyImporterPipeline,
         epss_importer_v2.EPSSImporterPipeline,
         nginx_importer_v2.NginxImporterPipeline,

vulnerabilities/management/commands/run_scheduler.py

@@ -17,16 +17,21 @@
 
 
 def init_pipeline_scheduled():
-    """Initialize schedule jobs for active PipelineSchedule."""
-    active_pipeline_qs = models.PipelineSchedule.objects.filter(is_active=True).order_by(
-        "created_date"
-    )
-    for pipeline_schedule in active_pipeline_qs:
-        if scheduled_job_exists(pipeline_schedule.schedule_work_id):
-            continue
-        new_id = pipeline_schedule.create_new_job()
-        pipeline_schedule.schedule_work_id = new_id
-        pipeline_schedule.save(update_fields=["schedule_work_id"])
+    """
+    Initialize schedule jobs for active PipelineSchedule.
+        - Create new schedule if there is no schedule for active pipeline
+        - Create new schedule if schedule is corrupted for an active pipeline
+        - Delete schedule for inactive pipeline
+    """
+    pipeline_qs = models.PipelineSchedule.objects.order_by("created_date")
+    for pipeline in pipeline_qs:
+        reset_schedule = pipeline.is_active != bool(pipeline.schedule_work_id)
+        if not scheduled_job_exists(pipeline.schedule_work_id):
+            reset_schedule = True
+
+        if reset_schedule:
+            pipeline.schedule_work_id = pipeline.create_new_job()
+            pipeline.save(update_fields=["schedule_work_id"])
 
 
 class Command(rqscheduler.Command):

vulnerabilities/migrations/0110_pipelineschedule_is_run_once.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.22 on 2026-01-08 13:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0109_alter_advisoryseverity_scoring_elements_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pipelineschedule",
+            name="is_run_once",
+            field=models.BooleanField(
+                db_index=True,
+                default=False,
+                help_text="When set to True, this Pipeline will run only once.",
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -2273,6 +2273,13 @@ class PipelineSchedule(models.Model):
         ),
     )
 
+    is_run_once = models.BooleanField(
+        null=False,
+        db_index=True,
+        default=False,
+        help_text=("When set to True, this Pipeline will run only once."),
+    )
+
     live_logging = models.BooleanField(
         null=False,
         db_index=True,
@@ -2789,6 +2796,19 @@ class Meta:
             )
         ]
 
+    def to_dict(self):
+        return {
+            "patch_url": self.patch_url,
+            "patch_text": self.patch_text,
+            "patch_checksum": self.patch_checksum,
+        }
+
+    def to_patch_data(self):
+        """Return `PatchData` from the Patch."""
+        from vulnerabilities.importer import PatchData
+
+        return PatchData.from_dict(self.to_dict())
+
 
 class PackageCommitPatch(models.Model):
     """
@@ -2816,6 +2836,14 @@ def save(self, *args, **kwargs):
     class Meta:
         unique_together = ["commit_hash", "vcs_url"]
 
+    def to_dict(self):
+        return {
+            "vcs_url": self.vcs_url,
+            "commit_hash": self.commit_hash,
+            "patch_text": self.patch_text,
+            "patch_checksum": self.patch_checksum,
+        }
+
 
 class AdvisoryV2QuerySet(BaseQuerySet):
     def latest_for_avid(self, avid: str):
@@ -3009,6 +3037,7 @@ def to_advisory_data(self) -> "AdvisoryData":
                 impacted.to_affected_package_data() for impacted in self.impacted_packages.all()
             ],
             references_v2=[ref.to_reference_v2_data() for ref in self.references.all()],
+            patches=[patch.to_patch_data() for patch in self.patches.all()],
             date_published=self.date_published,
             weaknesses=[weak.cwe_id for weak in self.weaknesses.all()],
             severities=[sev.to_vulnerability_severity_data() for sev in self.severities.all()],
@@ -3092,6 +3121,12 @@ def to_dict(self):
             "package": purl_to_dict(self.base_purl),
             "affected_version_range": self.affecting_vers,
             "fixed_version_range": self.fixed_vers,
+            "introduced_by_commit_patches": [
+                commit.to_dict() for commit in self.introduced_by_package_commit_patches.all()
+            ],
+            "fixed_by_commit_patches": [
+                commit.to_dict() for commit in self.fixed_by_package_commit_patches.all()
+            ],
         }
 
     def to_affected_package_data(self):

vulnerabilities/pipelines/__init__.py

@@ -169,6 +169,10 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline):
     importer_name = None
     advisory_confidence = MAX_CONFIDENCE
 
+    # When set to true pipeline is run only once.
+    # To rerun onetime pipeline reset is_active field to True via migration.
+    run_once = False
+
     @classmethod
     def steps(cls):
         return (
@@ -262,6 +266,10 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     repo_url = None
     ignorable_versions = []
 
+    # When set to true pipeline is run only once.
+    # To rerun onetime pipeline reset is_active field to True via migration.
+    run_once = False
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -16,13 +16,9 @@
 from fetchcode.vcs import fetch_via_vcs
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackageV2
-from vulnerabilities.importer import PackageCommitPatchData
-from vulnerabilities.importer import PatchData
-from vulnerabilities.importer import ReferenceV2
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
-from vulnerabilities.pipes.advisory import classify_patch_source
+from vulnerabilities.pipes.advisory import append_patch_classifications
 from vulnerabilities.severity_systems import GENERIC
 
 
@@ -90,23 +86,14 @@ def collect_advisories(self):
                     patch_url = commit_data.get("patchUrl")
                     commit_id = commit_data.get("commitId")
 
-                    base_purl, patch_objs = classify_patch_source(
+                    append_patch_classifications(
                         url=patch_url,
                         commit_hash=commit_id,
                         patch_text=None,
+                        affected_packages=affected_packages,
+                        references=references,
+                        patches=patches,
                     )
-                    for patch_obj in patch_objs:
-                        if isinstance(patch_obj, PackageCommitPatchData):
-                            fixed_commit = patch_obj
-                            affected_package = AffectedPackageV2(
-                                package=base_purl,
-                                fixed_by_commit_patches=[fixed_commit],
-                            )
-                            affected_packages.append(affected_package)
-                        elif isinstance(patch_obj, PatchData):
-                            patches.append(patch_obj)
-                        elif isinstance(patch_obj, ReferenceV2):
-                            references.append(patch_obj)
 
                 url = (
                     "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/"